From ebb79c16fe7d626d9d201fe5d1090da6e4ab078f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 12 Apr 2024 16:15:36 +0100 Subject: [PATCH] Aliasing support for SQS. --- .../api/controllers/row/ExternalRequest.ts | 3 +- packages/server/src/db/utils.ts | 1 + .../src/integrations/tests/sqlAlias.spec.ts | 4 +- packages/server/src/sdk/app/rows/index.ts | 2 + .../server/src/sdk/app/rows/search/sqs.ts | 99 +++++++++++-------- .../row/alias.ts => sdk/app/rows/sqlAlias.ts} | 28 ++++-- 6 files changed, 86 insertions(+), 51 deletions(-) rename packages/server/src/{api/controllers/row/alias.ts => sdk/app/rows/sqlAlias.ts} (87%) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7fc0333de1..4adbb72c7a 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { db as dbCore } from "@budibase/backend-core" -import AliasTables from "./alias" import sdk from "../../../sdk" import env from "../../../environment" @@ -618,7 +617,7 @@ export class ExternalRequest { if (env.SQL_ALIASING_DISABLE) { response = await getDatasourceAndQuery(json) } else { - const aliasing = new AliasTables(Object.keys(this.tables)) + const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) response = await aliasing.queryWithAliasing(json) } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index b1c02b1764..ce8d0accbb 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}` +export const SQS_DATASOURCE_INTERNAL = "internal" export const ViewName = dbCore.ViewName export const InternalTables = dbCore.InternalTable export const UNICODE_MAX = dbCore.UNICODE_MAX diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index bfca24ff7d..58c3a05245 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -8,8 +8,10 @@ import { import { join } from "path" import Sql from "../base/sql" import { SqlClient } from "../utils" -import AliasTables from "../../api/controllers/row/alias" import { generator } from "@budibase/backend-core/tests" +import sdk from "../../sdk" + +const AliasTables = sdk.rows.AliasTables function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index ea501e93d9..c117941419 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,6 +3,7 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" +import AliasTables from "./sqlAlias" export default { ...attachments, @@ -10,4 +11,5 @@ export default { ...search, utils, external, + AliasTables, } diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 5b0b6e3bc7..20edb988d3 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -20,7 +20,12 @@ import { } from "../../../../api/controllers/row/utils" import sdk from "../../../index" import { context } from "@budibase/backend-core" -import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" +import { + CONSTANT_INTERNAL_ROW_COLS, + SQS_DATASOURCE_INTERNAL, +} from "../../../../db/utils" +import AliasTables from "../sqlAlias" +import { outputProcessing } from "../../../../utilities/rowProcessor" function buildInternalFieldList( table: Table, @@ -31,19 +36,19 @@ function buildInternalFieldList( fieldList = fieldList.concat( CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`) ) - if (opts.relationships) { - for (let col of Object.values(table.schema)) { - if (col.type === FieldType.LINK) { - const linkCol = col as RelationshipFieldMetadata - const relatedTable = tables.find( - table => table._id === linkCol.tableId - )! - fieldList = fieldList.concat( - buildInternalFieldList(relatedTable, tables, { relationships: false }) - ) - } else { - fieldList.push(`${table._id}.${col.name}`) - } + for (let col of Object.values(table.schema)) { + const isRelationship = col.type === FieldType.LINK + if (!opts.relationships && isRelationship) { + continue + } + if (isRelationship) { + const linkCol = col as RelationshipFieldMetadata + const relatedTable = tables.find(table => table._id === linkCol.tableId)! + fieldList = fieldList.concat( + buildInternalFieldList(relatedTable, tables, { relationships: false }) + ) + } else { + fieldList.push(`${table._id}.${col.name}`) } } return fieldList @@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) { } export async function search( - options: RowSearchParams + options: RowSearchParams, + table: Table ): Promise> { - const { tableId, paginate, query, ...params } = options + const { paginate, query, ...params } = options const builder = new SqlQueryBuilder(SqlClient.SQL_LITE) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) - const table = allTables.find(table => table._id === tableId) if (!table) { throw new Error("Unable to find table") } @@ -111,7 +116,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasourceId: "internal", + datasourceId: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, @@ -154,34 +159,44 @@ export async function search( } } try { - const query = builder._query(request, { - disableReturning: true, + const alias = new AliasTables(allTables.map(table => table.name)) + const rows = await alias.queryWithAliasing(request, async json => { + const query = builder._query(json, { + disableReturning: true, + }) + + if (Array.isArray(query)) { + throw new Error("SQS cannot currently handle multiple queries") + } + + let sql = query.sql, + bindings = query.bindings + + // quick hack for docIds + sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") + sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") + + const db = context.getAppDB() + return await db.sql(sql, bindings) }) - if (Array.isArray(query)) { - throw new Error("SQS cannot currently handle multiple queries") - } - - let sql = query.sql, - bindings = query.bindings - - // quick hack for docIds - sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") - sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") - - const db = context.getAppDB() - const rows = await db.sql(sql, bindings) + // process from the format of tableId.column to expected format + const processed = await sqlOutputProcessing( + rows, + table!, + allTablesMap, + relationships, + { + sqs: true, + } + ) return { - rows: await sqlOutputProcessing( - rows, - table!, - allTablesMap, - relationships, - { - sqs: true, - } - ), + // final row processing for response + rows: await outputProcessing(table, processed, { + preserveLinks: true, + squash: true, + }), } } catch (err: any) { const msg = typeof err === "string" ? err : err.message diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts similarity index 87% rename from packages/server/src/api/controllers/row/alias.ts rename to packages/server/src/sdk/app/rows/sqlAlias.ts index 0ec9d1a09c..0fc338ecbe 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -6,11 +6,12 @@ import { Row, SearchFilters, } from "@budibase/types" -import { getSQLClient } from "../../../sdk/app/rows/utils" +import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" -import sdk from "../../../sdk" +import datasources from "../datasources" import { makeExternalQuery } from "../../../integrations/base/query" import { SqlClient } from "../../../integrations/utils" +import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils" const WRITE_OPERATIONS: Operation[] = [ Operation.CREATE, @@ -156,12 +157,19 @@ export default class AliasTables { } async queryWithAliasing( - json: QueryJson + json: QueryJson, + queryFn?: (json: QueryJson) => Promise ): Promise { const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) + const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL + let aliasingEnabled: boolean, datasource: Datasource | undefined + if (isSqs) { + aliasingEnabled = true + } else { + datasource = await datasources.get(datasourceId) + aliasingEnabled = this.isAliasingEnabled(json, datasource) + } - const aliasingEnabled = this.isAliasingEnabled(json, datasource) if (aliasingEnabled) { json = cloneDeep(json) // run through the query json to update anywhere a table may be used @@ -207,7 +215,15 @@ export default class AliasTables { } json.tableAliases = invertedTableAliases } - const response = await makeExternalQuery(datasource, json) + + let response: DatasourcePlusQueryResponse + if (datasource && !isSqs) { + response = await makeExternalQuery(datasource, json) + } else if (queryFn) { + response = await queryFn(json) + } else { + throw new Error("No supplied method to perform aliased query") + } if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else {