From fd4403037d40f655294f932c08de4acfe95608e3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 23 Jun 2021 19:05:32 +0100 Subject: [PATCH 1/2] WIP - basic override of foreign keys. --- .../src/api/controllers/row/external.js | 85 ++++++++++++++++--- .../server/src/api/controllers/table/utils.js | 11 ++- packages/server/src/integrations/base/sql.js | 26 +++++- packages/server/src/integrations/postgres.js | 14 +++ 4 files changed, 117 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 896f5a78e2..79a347750f 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,6 +1,6 @@ const { makeExternalQuery } = require("./utils") -const { DataSourceOperation, SortDirection } = require("../../../constants") -const { getExternalTable } = require("../table/utils") +const { DataSourceOperation, SortDirection, FieldTypes, RelationshipTypes } = require("../../../constants") +const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, generateRowIdField, @@ -35,17 +35,56 @@ function generateIdForRow(row, table) { return generateRowIdField(idParts) } -function outputProcessing(rows, table) { +function updateRelationshipColumns(rows, row, relationships, allTables) { + const columns = {} + for (let relationship of relationships) { + const linkedTable = allTables[relationship.tableName] + if (!linkedTable) { + continue + } + const display = linkedTable.primaryDisplay + const related = {} + if (display && row[display]) { + related.primaryDisplay = row[display] + } + related._id = row[relationship.to] + columns[relationship.from] = related + } + for (let [column, related] of Object.entries(columns)) { + if (!Array.isArray(rows[row._id][column])) { + rows[row._id][column] = [] + } + rows[row._id][column].push(related) + } + return rows +} + +function outputProcessing(rows, table, relationships, allTables) { // if no rows this is what is returned? Might be PG only if (rows[0].read === true) { return [] } + let finalRows = {} for (let row of rows) { row._id = generateIdForRow(row, table) - row.tableId = table._id - row._rev = "rev" + // this is a relationship of some sort + if (finalRows[row._id]) { + finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) + continue + } + const thisRow = {} + // filter the row down to what is actually the row (not joined) + for (let fieldName of Object.keys(table.schema)) { + thisRow[fieldName] = row[fieldName] + } + thisRow._id = row._id + thisRow.tableId = table._id + thisRow._rev = "rev" + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) } - return rows + return Object.values(finalRows) } function buildFilters(id, filters, table) { @@ -83,6 +122,26 @@ function buildFilters(id, filters, table) { } } +function buildRelationships(table) { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + // TODO: through field + if (field.relationshipType === RelationshipTypes.MANY_TO_MANY) { + continue + } + const broken = breakExternalTableId(field.tableId) + relationships.push({ + from: fieldName, + to: field.fieldName, + tableName: broken.tableName, + }) + } + return relationships +} + async function handleRequest( appId, operation, @@ -90,12 +149,14 @@ async function handleRequest( { id, row, filters, sort, paginate } = {} ) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const table = await getExternalTable(appId, datasourceId, tableName) + const tables = await getAllExternalTables(appId, datasourceId) + const table = tables[tableName] if (!table) { throw `Unable to process query, table "${tableName}" not defined.` } // clean up row on ingress using schema filters = buildFilters(id, filters, table) + const relationships = buildRelationships(table) row = inputProcessing(row, table) if ( operation === DataSourceOperation.DELETE && @@ -116,6 +177,7 @@ async function handleRequest( filters, sort, paginate, + relationships, body: row, // pass an id filter into extra, purely for mysql/returning extra: { @@ -126,9 +188,9 @@ async function handleRequest( const response = await makeExternalQuery(appId, json) // we searched for rows in someway if (operation === DataSourceOperation.READ && Array.isArray(response)) { - return outputProcessing(response, table) + return outputProcessing(response, table, relationships, tables) } else { - row = outputProcessing(response, table)[0] + row = outputProcessing(response, table, relationships, tables)[0] return { row, table } } } @@ -270,7 +332,4 @@ exports.validate = async () => { return { valid: true } } -exports.fetchEnrichedRow = async () => { - // TODO: How does this work - throw "Not Implemented" -} +exports.fetchEnrichedRow = async () => {} diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index cdfd390027..78dae60ab1 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -204,15 +204,18 @@ class TableSaveFunctions { } } -exports.getExternalTable = async (appId, datasourceId, tableName) => { +exports.getAllExternalTables = async (appId, datasourceId) => { const db = new CouchDB(appId) const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." } - return Object.values(datasource.entities).find( - entity => entity.name === tableName - ) + return datasource.entities +} + +exports.getExternalTable = async (appId, datasourceId, tableName) => { + const entities = await exports.getAllExternalTables(appId, datasourceId) + return entities[tableName] } exports.TableSaveFunctions = TableSaveFunctions diff --git a/packages/server/src/integrations/base/sql.js b/packages/server/src/integrations/base/sql.js index c7cc95fc3e..08eac09f0b 100644 --- a/packages/server/src/integrations/base/sql.js +++ b/packages/server/src/integrations/base/sql.js @@ -55,6 +55,25 @@ function addFilters(query, filters) { return query } +function addRelationships(query, fromTable, relationships) { + if (!relationships) { + return query + } + for (let relationship of relationships) { + const from = `${fromTable}.${relationship.from}` + const to = `${relationship.tableName}.${relationship.to}` + if (!relationship.through) { + query = query.innerJoin(relationship.tableName, from, to) + } else { + const through = relationship + query = query + .innerJoin(through.tableName, from, through.from) + .innerJoin(relationship.tableName, to, through.to) + } + } + return query +} + function buildCreate(knex, json, opts) { const { endpoint, body } = json let query = knex(endpoint.entityId) @@ -67,8 +86,9 @@ function buildCreate(knex, json, opts) { } function buildRead(knex, json, limit) { - let { endpoint, resource, filters, sort, paginate } = json - let query = knex(endpoint.entityId) + let { endpoint, resource, filters, sort, paginate, relationships } = json + const tableName = endpoint.entityId + let query = knex(tableName) // select all if not specified if (!resource) { resource = { fields: [] } @@ -81,6 +101,8 @@ function buildRead(knex, json, limit) { } // handle where query = addFilters(query, filters) + // handle join + query = addRelationships(query, tableName, relationships) // handle sorting if (sort) { for (let [key, value] of Object.entries(sort)) { diff --git a/packages/server/src/integrations/postgres.js b/packages/server/src/integrations/postgres.js index 6266e6ca64..f2d1f08162 100644 --- a/packages/server/src/integrations/postgres.js +++ b/packages/server/src/integrations/postgres.js @@ -153,6 +153,20 @@ class PostgresIntegration extends Sql { name: columnName, type: convertType(column.data_type, TYPE_MAP), } + + // // TODO: hack for testing + // if (tableName === "persons") { + // tables[tableName].primaryDisplay = "firstname" + // } + // if (columnName.toLowerCase() === "personid" && tableName === "tasks") { + // tables[tableName].schema[columnName] = { + // name: columnName, + // type: "link", + // tableId: buildExternalTableId(datasourceId, "persons"), + // relationshipType: "one-to-many", + // fieldName: "personid", + // } + // } } this.tables = tables } From 84ea655fd9639be801c3d3efa2965ad9ba425aac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 25 Jun 2021 18:34:21 +0100 Subject: [PATCH 2/2] Linting. --- .../src/api/controllers/row/external.js | 21 +++- .../src/integrations/base/definitions.ts | 114 +++++++++--------- packages/server/src/integrations/base/sql.ts | 24 +++- 3 files changed, 95 insertions(+), 64 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 79a347750f..a5180a1436 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,5 +1,10 @@ const { makeExternalQuery } = require("./utils") -const { DataSourceOperation, SortDirection, FieldTypes, RelationshipTypes } = require("../../../constants") +const { + DataSourceOperation, + SortDirection, + FieldTypes, + RelationshipTypes, +} = require("../../../constants") const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, @@ -69,7 +74,12 @@ function outputProcessing(rows, table, relationships, allTables) { row._id = generateIdForRow(row, table) // this is a relationship of some sort if (finalRows[row._id]) { - finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) + finalRows = updateRelationshipColumns( + finalRows, + row, + relationships, + allTables + ) continue } const thisRow = {} @@ -82,7 +92,12 @@ function outputProcessing(rows, table, relationships, allTables) { thisRow._rev = "rev" finalRows[thisRow._id] = thisRow // do this at end once its been added to the final rows - finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) + finalRows = updateRelationshipColumns( + finalRows, + row, + relationships, + allTables + ) } return Object.values(finalRows) } diff --git a/packages/server/src/integrations/base/definitions.ts b/packages/server/src/integrations/base/definitions.ts index 64663d073b..e8a5bfe10a 100644 --- a/packages/server/src/integrations/base/definitions.ts +++ b/packages/server/src/integrations/base/definitions.ts @@ -27,95 +27,95 @@ export enum DatasourceFieldTypes { } export interface QueryDefinition { - type: QueryTypes, - displayName?: string, - readable?: boolean, - customisable?: boolean, - fields?: object, - urlDisplay?: boolean, + type: QueryTypes + displayName?: string + readable?: boolean + customisable?: boolean + fields?: object + urlDisplay?: boolean } export interface Integration { - docs: string, - plus?: boolean, - description: string, - friendlyName: string, - datasource: {}, + docs: string + plus?: boolean + description: string + friendlyName: string + datasource: {} query: { - [key: string]: QueryDefinition, - }, + [key: string]: QueryDefinition + } } export interface SearchFilters { - allOr: boolean, + allOr: boolean string?: { - [key: string]: string, - }, + [key: string]: string + } fuzzy?: { - [key: string]: string, - }, + [key: string]: string + } range?: { [key: string]: { - high: number | string, - low: number | string, - }, - }, + high: number | string + low: number | string + } + } equal?: { - [key: string]: any, - }, + [key: string]: any + } notEqual?: { - [key: string]: any, - }, + [key: string]: any + } empty?: { - [key: string]: any, - }, + [key: string]: any + } notEmpty?: { - [key: string]: any, - }, + [key: string]: any + } } export interface RelationshipsJson { through?: { - from: string, - to: string, - tableName: string, - }, - from: string, - to: string, - tableName: string, + from: string + to: string + tableName: string + } + from: string + to: string + tableName: string } export interface QueryJson { endpoint: { - datasourceId: string, - entityId: string, - operation: Operation, - }, + datasourceId: string + entityId: string + operation: Operation + } resource: { - fields: string[], - }, - filters?: SearchFilters, + fields: string[] + } + filters?: SearchFilters sort?: { - [key: string]: SortDirection, - }, + [key: string]: SortDirection + } paginate?: { - limit: number, - page: string | number, - }, - body?: object, + limit: number + page: string | number + } + body?: object extra?: { - idFilter?: SearchFilters, - }, - relationships?: RelationshipsJson[], + idFilter?: SearchFilters + } + relationships?: RelationshipsJson[] } export interface SqlQuery { - sql: string, + sql: string bindings?: { - [key: string]: any, - }, + [key: string]: any + } } export interface QueryOptions { - disableReturning?: boolean, + disableReturning?: boolean } diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index dafbc2e6e1..60dfa84862 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -70,7 +70,11 @@ function addFilters( return query } -function addRelationships(query: KnexQuery, fromTable: string, relationships: RelationshipsJson[] | undefined): KnexQuery { +function addRelationships( + query: KnexQuery, + fromTable: string, + relationships: RelationshipsJson[] | undefined +): KnexQuery { if (!relationships) { return query } @@ -91,7 +95,11 @@ function addRelationships(query: KnexQuery, fromTable: string, relationships: Re return query } -function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { +function buildCreate( + knex: Knex, + json: QueryJson, + opts: QueryOptions +): KnexQuery { const { endpoint, body } = json let query: KnexQuery = knex(endpoint.entityId) // mysql can't use returning @@ -141,7 +149,11 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { return query } -function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { +function buildUpdate( + knex: Knex, + json: QueryJson, + opts: QueryOptions +): KnexQuery { const { endpoint, body, filters } = json let query: KnexQuery = knex(endpoint.entityId) query = addFilters(query, filters) @@ -153,7 +165,11 @@ function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery } } -function buildDelete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { +function buildDelete( + knex: Knex, + json: QueryJson, + opts: QueryOptions +): KnexQuery { const { endpoint, filters } = json let query: KnexQuery = knex(endpoint.entityId) query = addFilters(query, filters)