Aliasing support for SQS.

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { db as dbCore } from "@budibase/backend-core"
-import AliasTables from "./alias"
 import sdk from "../../../sdk"
 import env from "../../../environment"
 
@@ -618,7 +617,7 @@ export class ExternalRequest<T extends Operation> {
     if (env.SQL_ALIASING_DISABLE) {
       response = await getDatasourceAndQuery(json)
     } else {
-      const aliasing = new AliasTables(Object.keys(this.tables))
+      const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
       response = await aliasing.queryWithAliasing(json)
     }
 

packages/server/src/db/utils.ts

@@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
+export const SQS_DATASOURCE_INTERNAL = "internal"
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -8,8 +8,10 @@ import {
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
-import AliasTables from "../../api/controllers/row/alias"
 import { generator } from "@budibase/backend-core/tests"
+import sdk from "../../sdk"
+
+const AliasTables = sdk.rows.AliasTables
 
 function multiline(sql: string) {
   return sql.replace(/\n/g, "").replace(/ +/g, " ")

packages/server/src/sdk/app/rows/index.ts

@@ -3,6 +3,7 @@ import * as rows from "./rows"
 import * as search from "./search"
 import * as utils from "./utils"
 import * as external from "./external"
+import AliasTables from "./sqlAlias"
 
 export default {
   ...attachments,
@@ -10,4 +11,5 @@ export default {
   ...search,
   utils,
   external,
+  AliasTables,
 }

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -20,7 +20,12 @@ import {
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
 import { context } from "@budibase/backend-core"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  SQS_DATASOURCE_INTERNAL,
+} from "../../../../db/utils"
+import AliasTables from "../sqlAlias"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
 
 function buildInternalFieldList(
   table: Table,
@@ -31,19 +36,19 @@ function buildInternalFieldList(
   fieldList = fieldList.concat(
     CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
   )
-  if (opts.relationships) {
-    for (let col of Object.values(table.schema)) {
-      if (col.type === FieldType.LINK) {
-        const linkCol = col as RelationshipFieldMetadata
-        const relatedTable = tables.find(
-          table => table._id === linkCol.tableId
-        )!
-        fieldList = fieldList.concat(
-          buildInternalFieldList(relatedTable, tables, { relationships: false })
-        )
-      } else {
-        fieldList.push(`${table._id}.${col.name}`)
-      }
+  for (let col of Object.values(table.schema)) {
+    const isRelationship = col.type === FieldType.LINK
+    if (!opts.relationships && isRelationship) {
+      continue
+    }
+    if (isRelationship) {
+      const linkCol = col as RelationshipFieldMetadata
+      const relatedTable = tables.find(table => table._id === linkCol.tableId)!
+      fieldList = fieldList.concat(
+        buildInternalFieldList(relatedTable, tables, { relationships: false })
+      )
+    } else {
+      fieldList.push(`${table._id}.${col.name}`)
     }
   }
   return fieldList
@@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) {
 }
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
-  const { tableId, paginate, query, ...params } = options
+  const { paginate, query, ...params } = options
 
   const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
-  const table = allTables.find(table => table._id === tableId)
   if (!table) {
     throw new Error("Unable to find table")
   }
@@ -111,7 +116,7 @@ export async function search(
   const request: QueryJson = {
     endpoint: {
       // not important, we query ourselves
-      datasourceId: "internal",
+      datasourceId: SQS_DATASOURCE_INTERNAL,
       entityId: table._id!,
       operation: Operation.READ,
     },
@@ -154,34 +159,44 @@ export async function search(
     }
   }
   try {
-    const query = builder._query(request, {
-      disableReturning: true,
+    const alias = new AliasTables(allTables.map(table => table.name))
+    const rows = await alias.queryWithAliasing(request, async json => {
+      const query = builder._query(json, {
+        disableReturning: true,
+      })
+
+      if (Array.isArray(query)) {
+        throw new Error("SQS cannot currently handle multiple queries")
+      }
+
+      let sql = query.sql,
+        bindings = query.bindings
+
+      // quick hack for docIds
+      sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+      sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+      const db = context.getAppDB()
+      return await db.sql<Row>(sql, bindings)
     })
 
-    if (Array.isArray(query)) {
-      throw new Error("SQS cannot currently handle multiple queries")
-    }
-
-    let sql = query.sql,
-      bindings = query.bindings
-
-    // quick hack for docIds
-    sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
-    sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
-
-    const db = context.getAppDB()
-    const rows = await db.sql<Row>(sql, bindings)
+    // process from the format of tableId.column to expected format
+    const processed = await sqlOutputProcessing(
+      rows,
+      table!,
+      allTablesMap,
+      relationships,
+      {
+        sqs: true,
+      }
+    )
 
     return {
-      rows: await sqlOutputProcessing(
-        rows,
-        table!,
-        allTablesMap,
-        relationships,
-        {
-          sqs: true,
-        }
-      ),
+      // final row processing for response
+      rows: await outputProcessing<Row[]>(table, processed, {
+        preserveLinks: true,
+        squash: true,
+      }),
     }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -6,11 +6,12 @@ import {
   Row,
   SearchFilters,
 } from "@budibase/types"
-import { getSQLClient } from "../../../sdk/app/rows/utils"
+import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
-import sdk from "../../../sdk"
+import datasources from "../datasources"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { SqlClient } from "../../../integrations/utils"
+import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
 
 const WRITE_OPERATIONS: Operation[] = [
   Operation.CREATE,
@@ -156,12 +157,19 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(
-    json: QueryJson
+    json: QueryJson,
+    queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
   ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
-    const datasource = await sdk.datasources.get(datasourceId)
+    const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
+    let aliasingEnabled: boolean, datasource: Datasource | undefined
+    if (isSqs) {
+      aliasingEnabled = true
+    } else {
+      datasource = await datasources.get(datasourceId)
+      aliasingEnabled = this.isAliasingEnabled(json, datasource)
+    }
 
-    const aliasingEnabled = this.isAliasingEnabled(json, datasource)
     if (aliasingEnabled) {
       json = cloneDeep(json)
       // run through the query json to update anywhere a table may be used
@@ -207,7 +215,15 @@ export default class AliasTables {
       }
       json.tableAliases = invertedTableAliases
     }
-    const response = await makeExternalQuery(datasource, json)
+
+    let response: DatasourcePlusQueryResponse
+    if (datasource && !isSqs) {
+      response = await makeExternalQuery(datasource, json)
+    } else if (queryFn) {
+      response = await queryFn(json)
+    } else {
+      throw new Error("No supplied method to perform aliased query")
+    }
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {