From a13b5111bbaab7f6337eb0cc3721e0737900e122 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Jul 2021 14:10:44 +0100 Subject: [PATCH] First version of enrichment. --- .../src/api/controllers/row/external.js | 45 +++++++++++++++---- .../src/api/controllers/row/externalUtils.js | 22 ++++----- .../server/src/api/controllers/table/utils.js | 2 + packages/server/src/definitions/datasource.ts | 3 ++ packages/server/src/integrations/base/sql.ts | 6 +++ 5 files changed, 56 insertions(+), 22 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index e3e46ae527..143aa8a6b2 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,5 +1,5 @@ const { makeExternalQuery } = require("./utils") -const { DataSourceOperation, SortDirection } = require("../../../constants") +const { DataSourceOperation, SortDirection, FieldTypes } = require("../../../constants") const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, @@ -19,10 +19,12 @@ async function handleRequest( appId, operation, tableId, - { id, row, filters, sort, paginate, fullDocs } = {} + { id, row, filters, sort, paginate, tables } = {} ) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const tables = await getAllExternalTables(appId, datasourceId) + if (!tables) { + tables = await getAllExternalTables(appId, datasourceId) + } const table = tables[tableName] if (!table) { throw `Unable to process query, table "${tableName}" not defined.` @@ -83,8 +85,7 @@ async function handleRequest( response, table, relationships, - tables, - fullDocs + tables ) // if reading it'll just be an array of rows, return whole thing return operation === DataSourceOperation.READ && Array.isArray(response) @@ -239,15 +240,43 @@ exports.fetchEnrichedRow = async ctx => { const appId = ctx.appId const id = ctx.params.rowId const tableId = ctx.params.tableId - // TODO: this only enriches the full docs 1 layer deep, need to join those as well + const { datasourceId, tableName } = breakExternalTableId(tableId) + const tables = await getAllExternalTables(appId, datasourceId) const response = await handleRequest( appId, DataSourceOperation.READ, tableId, { id, - fullDocs: true, + tables, } ) - return response ? response[0] : response + const table = tables[tableName] + const row = response[0] + // this seems like a lot of work, but basically we need to dig deeper for the enrich + // for a single row, there is probably a better way to do this with some smart multi-layer joins + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK || !row[fieldName] || row[fieldName].length === 0) { + continue + } + const links = row[fieldName] + const linkedTableId = field.tableId + const linkedTable = tables[breakExternalTableId(linkedTableId).tableName] + // don't support composite keys right now + const linkedIds = links.map(link => breakRowIdField(link._id)[0]) + row[fieldName] = await handleRequest( + appId, + DataSourceOperation.READ, + linkedTableId, + { + tables, + filters: { + oneOf: { + [linkedTable.primary]: linkedIds, + }, + }, + }, + ) + } + return row } diff --git a/packages/server/src/api/controllers/row/externalUtils.js b/packages/server/src/api/controllers/row/externalUtils.js index 6074bdb429..85cc11b603 100644 --- a/packages/server/src/api/controllers/row/externalUtils.js +++ b/packages/server/src/api/controllers/row/externalUtils.js @@ -80,8 +80,7 @@ exports.updateRelationshipColumns = ( row, rows, relationships, - allTables, - fullDocs + allTables ) => { const columns = {} for (let relationship of relationships) { @@ -91,12 +90,10 @@ exports.updateRelationshipColumns = ( } let linked = basicProcessing(row, linkedTable) // if not returning full docs then get the minimal links out - if (!fullDocs) { - const display = linkedTable.primaryDisplay - linked = { - primaryDisplay: display ? linked[display] : undefined, - _id: linked._id, - } + const display = linkedTable.primaryDisplay + linked = { + primaryDisplay: display ? linked[display] : undefined, + _id: linked._id, } columns[relationship.column] = linked } @@ -116,8 +113,7 @@ exports.outputProcessing = ( rows, table, relationships, - allTables, - fullDocs + allTables ) => { // if no rows this is what is returned? Might be PG only if (rows[0].read === true) { @@ -132,8 +128,7 @@ exports.outputProcessing = ( row, finalRows, relationships, - allTables, - fullDocs + allTables ) continue } @@ -144,8 +139,7 @@ exports.outputProcessing = ( row, finalRows, relationships, - allTables, - fullDocs + allTables ) } return Object.values(finalRows) diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 78dae60ab1..760cc54b3b 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -1,3 +1,5 @@ +import {breakExternalTableId} from "../../../integrations/utils" + const CouchDB = require("../../../db") const csvParser = require("../../../utilities/csvParser") const { diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 733ad679cb..b7a3d47fff 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -72,6 +72,9 @@ export interface SearchFilters { notEmpty?: { [key: string]: any } + oneOf?: { + [key: string]: any[] + } } export interface RelationshipsJson { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 83631b4b98..e1c065dd26 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -30,6 +30,12 @@ function addFilters( } // if all or specified in filters, then everything is an or const allOr = filters.allOr + if (filters.oneOf) { + iterate(filters.oneOf, (key, array) => { + const fnc = allOr ? "orWhereIn" : "whereIn" + query = query[fnc](key, array) + }) + } if (filters.string) { iterate(filters.string, (key, value) => { const fnc = allOr ? "orWhere" : "where"