From 0a69ea02eeb9a58397e8ed259b797b4f96ab2ec7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Jun 2021 13:03:55 +0100 Subject: [PATCH 1/2] Adding some work towards supporting full data source integration. --- .../src/api/controllers/row/external.js | 82 ++++++++++++++----- .../server/src/api/controllers/row/index.js | 10 ++- .../src/api/controllers/row/internal.js | 4 +- packages/server/src/api/routes/row.js | 2 +- packages/server/src/integrations/base/sql.js | 20 +++-- .../server/src/integrations/tests/sql.spec.js | 16 ++++ 6 files changed, 99 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index d958c75e45..c9332b500f 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -2,15 +2,33 @@ const CouchDB = require("../../../db") const { makeExternalQuery } = require("./utils") const { DataSourceOperation, SortDirection } = require("../../../constants") -async function buildIDFilter(id) { - if (!id) { - return {} +async function getTable(appId, datasourceId, tableName) { + const db = new CouchDB(appId) + const datasource = await db.get(datasourceId) + if (!datasource || !datasource.entities) { + throw "Datasource is not configured fully." + } + return Object.values(datasource.entities).find( + entity => entity.name === tableName + ) +} + +function buildIDFilter(id, table) { + if (!id || !table) { + return null + } + // if used as URL parameter it will have been joined + if (typeof id === "string") { + id = id.split(",") + } + const primary = table.primary + const equal = {} + for (let field of primary) { + // work through the ID and get the parts + equal[field] = id.shift() } - // TODO: work out how to use the schema to get filter return { - equal: { - id: id, - }, + equal, } } @@ -18,20 +36,22 @@ async function handleRequest( appId, operation, tableId, - { id, row, filters, sort, paginate } + { id, row, filters, sort, paginate } = {} ) { - let [datasourceId, tableName] = tableId.split("/") - let idFilter = buildIDFilter(id) + let [datasourceId, tableName] = tableId.split("_") + const table = await getTable(appId, datasourceId, tableName) + if (!table) { + throw `Unable to process query, table "${tableName}" not defined.` + } + // try and build an id filter if required + let idFilters = buildIDFilter(id) let json = { endpoint: { datasourceId, entityId: tableName, operation, }, - filters: { - ...filters, - ...idFilter, - }, + filters: idFilters != null ? idFilters : filters, sort, paginate, body: row, @@ -65,15 +85,23 @@ exports.save = async ctx => { } exports.fetchView = async ctx => { - // TODO: don't know what this does for external + // there are no views in external data sources, shouldn't ever be called + ctx.throw(501, "Not implemented") } -exports.fetchTableRows = async ctx => { - // TODO: this is a basic read? +exports.fetch = async ctx => { + const appId = ctx.appId + const tableId = ctx.params.tableId + ctx.body = await handleRequest(appId, DataSourceOperation.READ, tableId) } exports.find = async ctx => { - // TODO: single find + const appId = ctx.appId + const id = ctx.params.rowId + const tableId = ctx.params.tableId + ctx.body = await handleRequest(appId, DataSourceOperation.READ, tableId, { + id, + }) } exports.destroy = async ctx => { @@ -85,7 +113,18 @@ exports.destroy = async ctx => { } exports.bulkDestroy = async ctx => { - // TODO: iterate through rows, build a large OR filter? + const appId = ctx.appId + const { rows } = ctx.request.body + const tableId = ctx.params.tableId + // TODO: this can probably be optimised to a single SQL statement in the future + let promises = [] + for (let row of rows) { + promises.push(handleRequest(appId, DataSourceOperation.DELETE, tableId, { + id: row._id, + })) + } + await Promise.all(promises) + ctx.body = { response: { ok: true }, rows } } exports.search = async ctx => { @@ -123,7 +162,6 @@ exports.validate = async ctx => { } exports.fetchEnrichedRow = async ctx => { - // TODO: should this join? - const appId = ctx.appId - ctx.body = {} + // TODO: How does this work + ctx.throw(501, "Not implemented") } diff --git a/packages/server/src/api/controllers/row/index.js b/packages/server/src/api/controllers/row/index.js index 6eaac88119..64ae88bf21 100644 --- a/packages/server/src/api/controllers/row/index.js +++ b/packages/server/src/api/controllers/row/index.js @@ -1,8 +1,11 @@ const internal = require("./internal") const external = require("./external") +const { DocumentTypes } = require("../../../db/utils") function pickApi(tableId) { - // TODO: go to external + if (tableId.includes(DocumentTypes.DATASOURCE)) { + return external + } return internal } @@ -33,7 +36,6 @@ exports.patch = async ctx => { } exports.save = async function (ctx) { - // TODO: this used to handle bulk delete, need to update builder/client const appId = ctx.appId const tableId = getTableId(ctx) try { @@ -55,10 +57,10 @@ exports.fetchView = async function (ctx) { } } -exports.fetchTableRows = async function (ctx) { +exports.fetch = async function (ctx) { const tableId = getTableId(ctx) try { - ctx.body = await pickApi(tableId).fetchTableRows(ctx) + ctx.body = await pickApi(tableId).fetch(ctx) } catch (err) { ctx.throw(400, err) } diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 5982259eec..5ebc032f3f 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -142,7 +142,7 @@ exports.fetchView = async ctx => { // if this is a table view being looked for just transfer to that if (viewName.startsWith(TABLE_VIEW_BEGINS_WITH)) { ctx.params.tableId = viewName.substring(4) - return exports.fetchTableRows(ctx) + return exports.fetch(ctx) } const db = new CouchDB(appId) @@ -195,7 +195,7 @@ exports.fetchView = async ctx => { return rows } -exports.fetchTableRows = async ctx => { +exports.fetch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) diff --git a/packages/server/src/api/routes/row.js b/packages/server/src/api/routes/row.js index dc60a14112..290df0ecf5 100644 --- a/packages/server/src/api/routes/row.js +++ b/packages/server/src/api/routes/row.js @@ -24,7 +24,7 @@ router "/api/:tableId/rows", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.READ), - rowController.fetchTableRows + rowController.fetch ) .get( "/api/:tableId/rows/:rowId", diff --git a/packages/server/src/integrations/base/sql.js b/packages/server/src/integrations/base/sql.js index b22ea50bc3..4901574ffc 100644 --- a/packages/server/src/integrations/base/sql.js +++ b/packages/server/src/integrations/base/sql.js @@ -3,6 +3,8 @@ const { DataSourceOperation, SortDirection } = require("../../constants") const BASE_LIMIT = 5000 function addFilters(query, filters) { + // if all or specified in filters, then everything is an or + const allOr = !!filters.allOr function iterate(structure, fn) { for (let [key, value] of Object.entries(structure)) { fn(key, value) @@ -13,7 +15,8 @@ function addFilters(query, filters) { } if (filters.string) { iterate(filters.string, (key, value) => { - query = query.where(key, "like", `${value}%`) + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, "like", `${value}%`) }) } if (filters.range) { @@ -21,27 +24,32 @@ function addFilters(query, filters) { if (!value.high || !value.low) { return } - query = query.whereBetween(key, [value.low, value.high]) + const fnc = allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [value.low, value.high]) }) } if (filters.equal) { iterate(filters.equal, (key, value) => { - query = query.where({ [key]: value }) + const fnc = allOr ? "orWhere" : "where" + query = query[fnc]({ [key]: value }) }) } if (filters.notEqual) { iterate(filters.notEqual, (key, value) => { - query = query.whereNot({ [key]: value }) + const fnc = allOr ? "orWhereNot" : "whereNot" + query = query[fnc]({ [key]: value }) }) } if (filters.empty) { iterate(filters.empty, key => { - query = query.whereNull(key) + const fnc = allOr ? "orWhereNull" : "whereNull" + query = query[fnc](key) }) } if (filters.notEmpty) { iterate(filters.notEmpty, key => { - query = query.whereNotNull(key) + const fnc = allOr ? "orWhereNotNull" : "whereNotNull" + query = query[fnc](key) }) } return query diff --git a/packages/server/src/integrations/tests/sql.spec.js b/packages/server/src/integrations/tests/sql.spec.js index 8bfa6f765c..80df7fe5ab 100644 --- a/packages/server/src/integrations/tests/sql.spec.js +++ b/packages/server/src/integrations/tests/sql.spec.js @@ -102,6 +102,22 @@ describe("SQL query builder", () => { }) }) + it("should test for multiple IDs with OR", () => { + const query = sql._query(generateReadJson({ + filters: { + equal: { + age: 10, + name: "John", + }, + allOr: true, + } + })) + expect(query).toEqual({ + bindings: [10, "John", limit], + sql: `select * from "${TABLE_NAME}" where ("age" = $1) or ("name" = $2) limit $3` + }) + }) + it("should test an create statement", () => { const query = sql._query(generateCreateJson(TABLE_NAME, { name: "Michael", From 7e9b9f2180031de869b05b61ae56db4cc77c2705 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Jun 2021 13:20:25 +0100 Subject: [PATCH 2/2] Some quick work to make it function as required. --- .../server/src/api/controllers/datasource.js | 2 +- .../src/api/controllers/row/external.js | 8 ++++++-- packages/server/src/api/routes/row.js | 1 - .../server/src/integrations/plus/postgres.js | 19 ++++++++++++------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index e79834b7c5..0aff868bb6 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -35,7 +35,7 @@ exports.save = async function (ctx) { const PlusConnector = plusIntegrations[datasource.source].integration const connector = new PlusConnector(ctx.request.body.config) - await connector.init() + await connector.init(datasource._id) datasource.entities = connector.tables } diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index c9332b500f..d91ea7cdaf 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -38,7 +38,9 @@ async function handleRequest( tableId, { id, row, filters, sort, paginate } = {} ) { - let [datasourceId, tableName] = tableId.split("_") + const parts = tableId.split("_") + let tableName = parts.pop() + let datasourceId = parts.join("_") const table = await getTable(appId, datasourceId, tableName) if (!table) { throw `Unable to process query, table "${tableName}" not defined.` @@ -86,7 +88,9 @@ exports.save = async ctx => { exports.fetchView = async ctx => { // there are no views in external data sources, shouldn't ever be called - ctx.throw(501, "Not implemented") + // for now just fetch + ctx.params.tableId = ctx.params.viewName.split("all_")[1] + return exports.fetch(ctx) } exports.fetch = async ctx => { diff --git a/packages/server/src/api/routes/row.js b/packages/server/src/api/routes/row.js index 290df0ecf5..c48f492909 100644 --- a/packages/server/src/api/routes/row.js +++ b/packages/server/src/api/routes/row.js @@ -38,7 +38,6 @@ router authorized(PermissionTypes.TABLE, PermissionLevels.READ), rowController.search ) - .post( "/api/:tableId/rows", paramResource("tableId"), diff --git a/packages/server/src/integrations/plus/postgres.js b/packages/server/src/integrations/plus/postgres.js index 3ac49dc925..56a490f1a1 100644 --- a/packages/server/src/integrations/plus/postgres.js +++ b/packages/server/src/integrations/plus/postgres.js @@ -79,12 +79,16 @@ class PostgresPlus extends Sql { this.client = this.pool } - async init() { - const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL) - const primaryKeys = {} - - for (let table of primaryKeysResponse.rows) { - primaryKeys[table.primary_key] = table.column_name + async init(datasourceId) { + let keys = [] + try { + const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL) + for (let table of primaryKeysResponse.rows) { + keys.push(table.column_name) + } + } catch (err) { + // TODO: this try catch method isn't right + keys = ["id"] } const columnsResponse = await this.client.query(this.COLUMNS_SQL) @@ -97,7 +101,8 @@ class PostgresPlus extends Sql { // table key doesn't exist yet if (!tables[tableName]) { tables[tableName] = { - _id: primaryKeys[tableName], + _id: `${datasourceId}_${tableName}`, + primary: keys, name: tableName, schema: {}, }