wip

packages/backend-core/src/sql/sql.ts

@@ -18,6 +18,7 @@ import {
   BasicOperator,
   BBReferenceFieldMetadata,
   CalculationType,
+  EnrichedQueryJson,
   FieldSchema,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
@@ -27,7 +28,6 @@ import {
   LogicalOperator,
   Operation,
   prefixed,
-  QueryJson,
   QueryOptions,
   RangeOperator,
   RelationshipsJson,
@@ -134,18 +134,18 @@ const allowEmptyRelationships: Record<SearchFilterKey, boolean> = {
 
 class InternalBuilder {
   private readonly client: SqlClient
-  private readonly query: QueryJson
+  private readonly query: EnrichedQueryJson
   private readonly splitter: dataFilters.ColumnSplitter
   private readonly knex: Knex
 
-  constructor(client: SqlClient, knex: Knex, query: QueryJson) {
+  constructor(client: SqlClient, knex: Knex, query: EnrichedQueryJson) {
     this.client = client
     this.query = query
     this.knex = knex
 
     this.splitter = new dataFilters.ColumnSplitter([this.table], {
       aliases: this.query.tableAliases,
-      columnPrefix: this.query.meta.columnPrefix,
+      columnPrefix: this.query.meta?.columnPrefix,
     })
   }
 
@@ -167,7 +167,7 @@ class InternalBuilder {
   }
 
   get table(): Table {
-    return this.query.meta.table
+    return this.query.table
   }
 
   get knexClient(): Knex.Client {
@@ -273,8 +273,7 @@ class InternalBuilder {
   }
 
   private isFullSelectStatementRequired(): boolean {
-    const { meta } = this.query
+    for (let column of Object.values(this.table.schema)) {
-    for (let column of Object.values(meta.table.schema)) {
       if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
         return true
       } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
@@ -285,14 +284,14 @@ class InternalBuilder {
   }
 
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
-    const { meta, endpoint, resource } = this.query
+    const { endpoint, resource } = this.query
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
     }
 
     const alias = this.getTableName(endpoint.entityId)
-    const schema = meta.table.schema
+    const schema = this.table.schema
     if (!this.isFullSelectStatementRequired()) {
       return [this.knex.raw("??", [`${alias}.*`])]
     }
@@ -1018,15 +1017,15 @@ class InternalBuilder {
       const name = tableOrName
       if (this.query.table?.name === name) {
         table = this.query.table
-      } else if (this.query.meta.table?.name === name) {
+      } else if (this.query.table.name === name) {
-        table = this.query.meta.table
+        table = this.query.table
-      } else if (!this.query.meta.tables?.[name]) {
+      } else if (!this.query.tables[name]) {
         // This can legitimately happen in custom queries, where the user is
         // querying against a table that may not have been imported into
         // Budibase.
         return name
       } else {
-        table = this.query.meta.tables[name]
+        table = this.query.tables[name]
       }
     } else if (tableOrName) {
       table = tableOrName
@@ -1238,7 +1237,7 @@ class InternalBuilder {
   ): Knex.QueryBuilder {
     const sqlClient = this.client
     const knex = this.knex
-    const { resource, tableAliases: aliases, endpoint, meta } = this.query
+    const { resource, tableAliases: aliases, endpoint, tables } = this.query
     const fields = resource?.fields || []
     for (let relationship of relationships) {
       const {
@@ -1253,7 +1252,7 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
-      const relatedTable = meta.tables?.[toTable]
+      const relatedTable = tables[toTable]
       const toAlias = aliases?.[toTable] || toTable,
         fromAlias = aliases?.[fromTable] || fromTable,
         throughAlias = (throughTable && aliases?.[throughTable]) || throughTable
@@ -1276,7 +1275,7 @@ class InternalBuilder {
         Math.floor(this.maxFunctionParameters() / 2)
       )
       const fieldList = relationshipFields.map(field =>
-        this.buildJsonField(relatedTable!, field)
+        this.buildJsonField(relatedTable, field)
       )
       const fieldListFormatted = fieldList
         .map(f => {
@@ -1472,9 +1471,7 @@ class InternalBuilder {
     if (this.client === SqlClient.ORACLE) {
       // Oracle doesn't seem to automatically insert nulls
       // if we don't specify them, so we need to do that here
-      for (const [column, schema] of Object.entries(
+      for (const [column, schema] of Object.entries(this.query.table.schema)) {
-        this.query.meta.table.schema
-      )) {
         if (
           schema.constraints?.presence === true ||
           schema.type === FieldType.FORMULA ||
@@ -1678,7 +1675,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes.
    * @return the query ready to be passed to the driver.
    */
-  _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
+  _query(
+    json: EnrichedQueryJson,
+    opts: QueryOptions = {}
+  ): SqlQuery | SqlQuery[] {
     const sqlClient = this.getSqlClient()
     const config: Knex.Config = {
       client: this.getBaseSqlClient(),
@@ -1728,34 +1728,30 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return this.convertToNative(query, opts)
   }
 
-  async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
+  async getReturningRow(queryFn: QueryFunction, json: EnrichedQueryJson) {
     if (!json.extra || !json.extra.idFilter) {
       return {}
     }
     const input = this._query({
+      ...json,
       endpoint: {
         ...json.endpoint,
         operation: Operation.READ,
       },
-      resource: {
+      resource: { fields: [] },
-        fields: [],
-      },
       filters: json.extra?.idFilter,
-      paginate: {
+      paginate: { limit: 1 },
-        limit: 1,
-      },
-      meta: json.meta,
     })
     return queryFn(input, Operation.READ)
   }
 
   // when creating if an ID has been inserted need to make sure
   // the id filter is enriched with it before trying to retrieve the row
-  checkLookupKeys(id: any, json: QueryJson) {
+  checkLookupKeys(id: any, json: EnrichedQueryJson) {
-    if (!id || !json.meta.table || !json.meta.table.primary) {
+    if (!id || !json.table.primary) {
       return json
     }
-    const primaryKey = json.meta.table.primary?.[0]
+    const primaryKey = json.table.primary[0]
     json.extra = {
       idFilter: {
         equal: {
@@ -1768,7 +1764,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 
   // this function recreates the returning functionality of postgres
   async queryWithReturning(
-    json: QueryJson,
+    json: EnrichedQueryJson,
     queryFn: QueryFunction,
     processFn: Function = (result: any) => result
   ) {

packages/backend-core/src/sql/sqlTable.ts

@@ -3,13 +3,13 @@ import {
   FieldType,
   NumberFieldMetadata,
   Operation,
-  QueryJson,
   RelationshipType,
   RenameColumn,
   SqlQuery,
   Table,
   TableSourceType,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import { breakExternalTableId, getNativeSql } from "./utils"
 import { helpers, utils } from "@budibase/shared-core"
@@ -200,7 +200,7 @@ function buildUpdateTable(
   table: Table,
   tables: Record<string, Table>,
   oldTable: Table,
-  renamed: RenameColumn
+  renamed?: RenameColumn
 ): SchemaBuilder {
   return knex.alterTable(table.name, schema => {
     generateSchema(schema, table, tables, oldTable, renamed)
@@ -238,11 +238,11 @@ class SqlTableQueryBuilder {
    * @param json the input JSON structure from which an SQL query will be built.
    * @return the operation that was found in the JSON.
    */
-  _operation(json: QueryJson): Operation {
+  _operation(json: EnrichedQueryJson): Operation {
     return json.endpoint.operation
   }
 
-  _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] {
+  _tableQuery(json: EnrichedQueryJson): SqlQuery | SqlQuery[] {
     let client = knex({ client: this.sqlClient }).schema
     let schemaName = json?.endpoint?.schema
     if (schemaName) {
@@ -250,7 +250,7 @@ class SqlTableQueryBuilder {
     }
 
     let query: Knex.SchemaBuilder
-    if (!json.table || !json.meta || !json.meta.tables) {
+    if (!json.table || !json.tables) {
       throw new Error("Cannot execute without table being specified")
     }
     if (json.table.sourceType === TableSourceType.INTERNAL) {
@@ -259,14 +259,14 @@ class SqlTableQueryBuilder {
 
     switch (this._operation(json)) {
       case Operation.CREATE_TABLE:
-        query = buildCreateTable(client, json.table, json.meta.tables)
+        query = buildCreateTable(client, json.table, json.tables)
         break
       case Operation.UPDATE_TABLE:
-        if (!json.meta || !json.meta.table) {
+        if (!json.table) {
           throw new Error("Must specify old table for update")
         }
         // renameColumn does not work for MySQL, so return a raw query
-        if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
+        if (this.sqlClient === SqlClient.MY_SQL && json.meta?.renamed) {
           const updatedColumn = json.meta.renamed.updated
           const tableName = schemaName
             ? `\`${schemaName}\`.\`${json.table.name}\``
@@ -280,14 +280,14 @@ class SqlTableQueryBuilder {
         query = buildUpdateTable(
           client,
           json.table,
-          json.meta.tables,
+          json.tables,
-          json.meta.table,
+          json.table,
-          json.meta.renamed!
+          json.meta?.renamed
         )
 
         // renameColumn for SQL Server returns a parameterised `sp_rename` query,
         // which is not supported by SQL Server and gives a syntax error.
-        if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) {
+        if (this.sqlClient === SqlClient.MS_SQL && json.meta?.renamed) {
           const oldColumn = json.meta.renamed.old
           const updatedColumn = json.meta.renamed.updated
           const tableName = schemaName

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -42,7 +42,7 @@ import {
   sqlOutputProcessing,
 } from "./utils"
 import {
-  getDatasourceAndQuery,
+  enrichQueryJson,
   processRowCountResponse,
 } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
@@ -135,16 +135,9 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   return config
 }
 
-function getEndpoint(tableId: string | undefined, operation: string) {
+function getEndpoint(tableId: string, operation: Operation) {
-  if (!tableId) {
-    throw new Error("Cannot get endpoint information - no table ID specified")
-  }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  return {
+  return { datasourceId, entityId: tableName, operation }
-    datasourceId: datasourceId,
-    entityId: tableName,
-    operation: operation as Operation,
-  }
 }
 
 function isOneSide(
@@ -268,12 +261,9 @@ export class ExternalRequest<T extends Operation> {
     const filters = this.prepareFilters(rowId, {}, table)
     // safety check, if there are no filters on deletion bad things happen
     if (Object.keys(filters).length !== 0) {
-      return getDatasourceAndQuery({
+      return makeExternalQuery({
         endpoint: getEndpoint(tableId, Operation.DELETE),
         filters,
-        meta: {
-          table,
-        },
       })
     } else {
       return []
@@ -289,13 +279,10 @@ export class ExternalRequest<T extends Operation> {
     const filters = this.prepareFilters(rowId, {}, table)
     // safety check, if there are no filters on deletion bad things happen
     if (Object.keys(filters).length !== 0) {
-      return getDatasourceAndQuery({
+      return makeExternalQuery({
         endpoint: getEndpoint(tableId, Operation.UPDATE),
         body: { [colName]: null },
         filters,
-        meta: {
-          table,
-        },
       })
     } else {
       return []
@@ -311,12 +298,9 @@ export class ExternalRequest<T extends Operation> {
   }
 
   async getRow(table: Table, rowId: string): Promise<Row> {
-    const response = await getDatasourceAndQuery({
+    const response = await makeExternalQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
       filters: this.prepareFilters(rowId, {}, table),
-      meta: {
-        table,
-      },
     })
     if (Array.isArray(response) && response.length > 0) {
       return response[0]
@@ -490,16 +474,13 @@ export class ExternalRequest<T extends Operation> {
       if (!relatedTable) {
         throw new Error("unable to find related table")
       }
-      const response = await getDatasourceAndQuery({
+      const response = await makeExternalQuery({
-        endpoint: endpoint,
+        endpoint,
         filters: {
           equal: {
             [fieldName]: row[lookupField],
           },
         },
-        meta: {
-          table: relatedTable,
-        },
       })
       // this is the response from knex if no rows found
       const rows: Row[] =
@@ -537,6 +518,11 @@ export class ExternalRequest<T extends Operation> {
     for (let relationship of relationships) {
       const { key, tableId, isUpdate, id, relationshipType, ...rest } =
         relationship
+
+      if (!tableId) {
+        throw new Error("Table ID is unknown, cannot find table")
+      }
+
       const body: { [key: string]: any } = processObjectSync(rest, row, {})
       const linkTable = this.getTable(tableId)
       const relationshipPrimary = linkTable?.primary || []
@@ -583,14 +569,11 @@ export class ExternalRequest<T extends Operation> {
       const operation = isUpdate ? Operation.UPDATE : Operation.CREATE
       if (!existingRelationship) {
         promises.push(
-          getDatasourceAndQuery({
+          makeExternalQuery({
             endpoint: getEndpoint(tableId, operation),
             // if we're doing many relationships then we're writing, only one response
             body,
             filters: this.prepareFilters(id, {}, linkTable),
-            meta: {
-              table: linkTable,
-            },
           })
         )
       } else {
@@ -749,10 +732,6 @@ export class ExternalRequest<T extends Operation> {
           table
         ),
       },
-      meta: {
-        table,
-        tables: this.tables,
-      },
     }
 
     // remove any relationships that could block deletion
@@ -773,8 +752,11 @@ export class ExternalRequest<T extends Operation> {
       response = [unprocessedRow]
     } else {
       response = env.SQL_ALIASING_DISABLE
-        ? await getDatasourceAndQuery(json)
+        ? await makeExternalQuery(json)
-        : await aliasing.queryWithAliasing(json, makeExternalQuery)
+        : await aliasing.queryWithAliasing(
+            await enrichQueryJson(json),
+            makeExternalQuery
+          )
     }
 
     // if it's a counting operation there will be no more processing, just return the number

packages/server/src/integrations/base/query.ts

@@ -1,37 +1,27 @@
-import {
+import { DatasourcePlusQueryResponse, QueryJson } from "@budibase/types"
-  QueryJson,
-  Datasource,
-  DatasourcePlusQueryResponse,
-  RowOperations,
-} from "@budibase/types"
 import { getIntegration } from "../index"
 import sdk from "../../sdk"
+import { enrichQueryJson } from "../../sdk/app/rows/utils"
 
 export async function makeExternalQuery(
-  datasource: Datasource,
   json: QueryJson
 ): Promise<DatasourcePlusQueryResponse> {
-  const entityId = json.endpoint.entityId,
+  const enrichedJson = await enrichQueryJson(json)
-    tableName = json.meta.table.name,
+  if (!enrichedJson.datasource) {
-    tableId = json.meta.table._id
-  // case found during testing - make sure this doesn't happen again
-  if (
-    RowOperations.includes(json.endpoint.operation) &&
-    entityId !== tableId &&
-    entityId !== tableName
-  ) {
-    throw new Error("Entity ID and table metadata do not align")
-  }
-  if (!datasource) {
     throw new Error("No datasource provided for external query")
   }
-  datasource = await sdk.datasources.enrich(datasource)
+
-  const Integration = await getIntegration(datasource.source)
+  enrichedJson.datasource = await sdk.datasources.enrich(
+    enrichedJson.datasource
+  )
+
+  const Integration = await getIntegration(enrichedJson.datasource.source)
+
   // query is the opinionated function
-  if (Integration.prototype.query) {
+  if (!Integration.prototype.query) {
-    const integration = new Integration(datasource.config)
-    return integration.query(json)
-  } else {
     throw "Datasource does not support query."
   }
+
+  const integration = new Integration(enrichedJson.datasource.config)
+  return integration.query(json)
 }

packages/server/src/integrations/microsoftSqlServer.ts

@@ -4,9 +4,9 @@ import {
   DatasourceFieldType,
   DatasourcePlus,
   DatasourcePlusQueryResponse,
+  EnrichedQueryJson,
   Integration,
   Operation,
-  QueryJson,
   QueryType,
   Schema,
   SourceName,
@@ -506,7 +506,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     return response.recordset || [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const schema = this.config.schema
     await this.connect()
     if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {
@@ -515,14 +515,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     const operation = this._operation(json)
     const queryFn = (query: any, op: string) => this.internalQuery(query, op)
     const processFn = (result: any) => {
-      if (json?.meta?.table && result.recordset) {
+      if (result.recordset) {
         return this.convertJsonStringColumns(
-          json.meta.table,
+          json.table,
           result.recordset,
           json.tableAliases
         )
-      } else if (result.recordset) {
-        return result.recordset
       }
       return [{ [operation]: true }]
     }

packages/server/src/integrations/mysql.ts

@@ -2,7 +2,6 @@ import {
   Integration,
   DatasourceFieldType,
   QueryType,
-  QueryJson,
   SqlQuery,
   Table,
   TableSchema,
@@ -15,6 +14,7 @@ import {
   DatasourcePlusQueryResponse,
   SqlQueryBinding,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -390,15 +390,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     return results.length ? results : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     await this.connect()
     try {
       const queryFn = (query: any) =>
         this.internalQuery(query, { connect: false, disableCoercion: true })
       const processFn = (result: any) => {
-        if (json?.meta?.table && Array.isArray(result)) {
+        if (Array.isArray(result)) {
           return this.convertJsonStringColumns(
-            json.meta.table,
+            json.table,
             result,
             json.tableAliases
           )

packages/server/src/integrations/oracle.ts

@@ -3,7 +3,6 @@ import {
   DatasourceFieldType,
   Integration,
   Operation,
-  QueryJson,
   QueryType,
   SqlQuery,
   Table,
@@ -15,6 +14,7 @@ import {
   Row,
   DatasourcePlusQueryResponse,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -545,7 +545,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const operation = this._operation(json)
     const input = this._query(json, { disableReturning: true }) as SqlQuery
     if (Array.isArray(input)) {

packages/server/src/integrations/postgres.ts

@@ -3,7 +3,6 @@ import {
   Integration,
   DatasourceFieldType,
   QueryType,
-  QueryJson,
   SqlQuery,
   Table,
   DatasourcePlus,
@@ -14,6 +13,7 @@ import {
   TableSourceType,
   DatasourcePlusQueryResponse,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -419,7 +419,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     return response.rows.length ? response.rows : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const operation = this._operation(json).toLowerCase()
     const input = this._query(json) as SqlQuery
     if (Array.isArray(input)) {

packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json

@@ -61,7 +61,6 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
@@ -167,7 +166,6 @@
     "sourceType": "external",
     "primaryDisplay": "firstname",
     "views": {}
-    }
   },
   "tableAliases": {
     "persons": "a",

packages/server/src/integrations/tests/sqlQueryJson/createSimple.json

@@ -5,10 +5,7 @@
     "operation": "CREATE"
   },
   "resource": {
-    "fields": [
+    "fields": ["a.name", "a.age"]
-      "a.name",
-      "a.age"
-    ]
   },
   "filters": {},
   "relationships": [],
@@ -24,16 +21,12 @@
       }
     }
   },
-  "meta": {
   "table": {
     "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people",
     "type": "table",
     "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae",
     "sourceType": "external",
-      "primary": [
+    "primary": ["name", "age"],
-        "name",
-        "age"
-      ],
     "name": "people",
     "schema": {
       "name": {
@@ -56,7 +49,6 @@
       }
     },
     "primaryDisplay": "name"
-    }
   },
   "tableAliases": {
     "people": "a"

packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json

@@ -52,12 +52,9 @@
     "idFilter": {}
   },
   "meta": {
-    "table": {
     "type": "table",
     "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "primary": [
+    "primary": ["personid"],
-        "personid"
-      ],
     "name": "persons",
     "schema": {
       "year": {
@@ -112,12 +109,7 @@
         "name": "type",
         "constraints": {
           "presence": false,
-            "inclusion": [
+          "inclusion": ["support", "designer", "programmer", "qa"]
-              "support",
-              "designer",
-              "programmer",
-              "qa"
-            ]
         }
       },
       "city": {
@@ -164,7 +156,6 @@
     "sourceType": "external",
     "primaryDisplay": "firstname",
     "views": {}
-    }
   },
   "tableAliases": {
     "persons": "a",

packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json

@@ -5,11 +5,7 @@
     "operation": "DELETE"
   },
   "resource": {
-    "fields": [
+    "fields": ["a.keyparttwo", "a.keypartone", "a.name"]
-      "a.keyparttwo",
-      "a.keypartone",
-      "a.name"
-    ]
   },
   "filters": {
     "equal": {
@@ -26,14 +22,10 @@
       }
     }
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable",
-      "primary": [
+    "primary": ["keypartone", "keyparttwo"],
-        "keypartone",
-        "keyparttwo"
-      ],
     "name": "compositetable",
     "schema": {
       "keyparttwo": {
@@ -67,7 +59,6 @@
     "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
     "sourceType": "external",
     "primaryDisplay": "keypartone"
-    }
   },
   "tableAliases": {
     "compositetable": "a"

packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json

@@ -17,10 +17,7 @@
   },
   "filters": {
     "oneOf": {
-      "taskid": [
+      "taskid": [1, 2]
-        1,
-        2
-      ]
     }
   },
   "relationships": [
@@ -42,13 +39,10 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
-      "primary": [
+    "primary": ["taskid"],
-        "taskid"
-      ],
     "name": "tasks",
     "schema": {
       "executorid": {
@@ -113,7 +107,6 @@
     "primaryDisplay": "taskname",
     "sql": true,
     "views": {}
-    }
   },
   "tableAliases": {
     "tasks": "a",

packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json

@@ -56,7 +56,6 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
@@ -97,7 +96,6 @@
     "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
     "sourceType": "external",
     "primaryDisplay": "productname"
-    }
   },
   "tableAliases": {
     "products": "a",

packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json

@@ -46,7 +46,6 @@
     "tasks": "b",
     "products": "a"
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products",
@@ -88,5 +87,4 @@
     "sourceType": "external",
     "primaryDisplay": "productname"
   }
-  }
 }

packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json

@@ -102,7 +102,6 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
@@ -189,7 +188,6 @@
     "primaryDisplay": "taskname",
     "sql": true,
     "views": {}
-    }
   },
   "tableAliases": {
     "tasks": "a",

packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json

@@ -59,13 +59,10 @@
       }
     }
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "primary": [
+    "primary": ["personid"],
-        "personid"
-      ],
     "name": "persons",
     "schema": {
       "year": {
@@ -120,12 +117,7 @@
         "name": "type",
         "constraints": {
           "presence": false,
-            "inclusion": [
+          "inclusion": ["support", "designer", "programmer", "qa"]
-              "support",
-              "designer",
-              "programmer",
-              "qa"
-            ]
         }
       },
       "city": {
@@ -172,7 +164,6 @@
     "sourceType": "external",
     "primaryDisplay": "firstname",
     "views": {}
-    }
   },
   "tableAliases": {
     "persons": "a",

packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json

@@ -59,13 +59,10 @@
       }
     }
   },
-  "meta": {
   "table": {
     "type": "table",
     "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "primary": [
+    "primary": ["personid"],
-        "personid"
-      ],
     "name": "persons",
     "schema": {
       "year": {
@@ -120,12 +117,7 @@
         "name": "type",
         "constraints": {
           "presence": false,
-            "inclusion": [
+          "inclusion": ["support", "designer", "programmer", "qa"]
-              "support",
-              "designer",
-              "programmer",
-              "qa"
-            ]
         }
       },
       "city": {
@@ -172,7 +164,6 @@
     "sourceType": "external",
     "primaryDisplay": "firstname",
     "views": {}
-    }
   },
   "tableAliases": {
     "persons": "a",

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -1,8 +1,8 @@
 import {
   Aggregation,
   CalculationType,
-  Datasource,
   DocumentType,
+  EnrichedQueryJson,
   FieldType,
   isLogicalSearchOperator,
   Operation,
@@ -38,7 +38,7 @@ import { generateJunctionTableID } from "../../../../../db/utils"
 import AliasTables from "../../sqlAlias"
 import { outputProcessing } from "../../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
-import { processRowCountResponse } from "../../utils"
+import { enrichQueryJson, processRowCountResponse } from "../../utils"
 import {
   dataFilters,
   helpers,
@@ -223,18 +223,18 @@ function reverseUserColumnMapping(rows: Row[], table?: Table) {
 }
 
 function runSqlQuery(
-  json: QueryJson,
+  json: EnrichedQueryJson,
   tables: Table[],
   relationships: RelationshipsJson[]
 ): Promise<Row[]>
 function runSqlQuery(
-  json: QueryJson,
+  json: EnrichedQueryJson,
   tables: Table[],
   relationships: RelationshipsJson[],
   opts: { countTotalRows: true }
 ): Promise<number>
 async function runSqlQuery(
-  json: QueryJson,
+  json: EnrichedQueryJson,
   tables: Table[],
   relationships: RelationshipsJson[],
   opts?: { countTotalRows?: boolean }
@@ -246,7 +246,7 @@ async function runSqlQuery(
   if (opts?.countTotalRows) {
     json.endpoint.operation = Operation.COUNT
   }
-  const processSQLQuery = async (_: Datasource, json: QueryJson) => {
+  const processSQLQuery = async (json: EnrichedQueryJson) => {
     const query = builder._query(json, {
       disableReturning: true,
     })
@@ -281,7 +281,7 @@ async function runSqlQuery(
   if (opts?.countTotalRows) {
     return processRowCountResponse(response)
   } else if (Array.isArray(response)) {
-    return reverseUserColumnMapping(response, json.meta.table)
+    return reverseUserColumnMapping(response, json.table)
   }
   return response
 }
@@ -372,10 +372,7 @@ export async function search(
       operation: Operation.READ,
     },
     filters: searchFilters,
-    table,
     meta: {
-      table,
-      tables: allTablesMap,
       columnPrefix: USER_COLUMN_PREFIX,
     },
     resource: {
@@ -427,11 +424,13 @@ export async function search(
     }
   }
 
+  const enrichedRequest = await enrichQueryJson(request)
+
   try {
     const [rows, totalRows] = await Promise.all([
-      runSqlQuery(request, allTables, relationships),
+      runSqlQuery(enrichedRequest, allTables, relationships),
       options.countRows
-        ? runSqlQuery(request, allTables, relationships, {
+        ? runSqlQuery(enrichedRequest, allTables, relationships, {
             countTotalRows: true,
           })
         : Promise.resolve(undefined),

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -1,11 +1,13 @@
 import {
   Datasource,
   DatasourcePlusQueryResponse,
+  EnrichedQueryJson,
   Operation,
   QueryJson,
   Row,
   SearchFilters,
   SqlClient,
+  Table,
 } from "@budibase/types"
 import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getSQLClient } from "./utils"
@@ -15,7 +17,6 @@ import { BudibaseInternalDB } from "../../../db/utils"
 import { dataFilters } from "@budibase/shared-core"
 
 type PerformQueryFunction = (
-  datasource: Datasource,
   json: QueryJson
 ) => Promise<DatasourcePlusQueryResponse>
 
@@ -99,7 +100,11 @@ export default class AliasTables {
     return true
   }
 
-  getAlias(tableName: string) {
+  getAlias(tableName: string | Table) {
+    if (typeof tableName === "object") {
+      tableName = tableName.name
+    }
+
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
     }
@@ -177,7 +182,7 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(
-    json: QueryJson,
+    json: EnrichedQueryJson,
     queryFn: PerformQueryFunction
   ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
@@ -215,14 +220,7 @@ export default class AliasTables {
         }
         json.filters = aliasFilters(json.filters)
       }
-      if (json.meta?.table) {
+
-        this.getAlias(json.meta.table.name)
-      }
-      if (json.meta?.tables) {
-        Object.keys(json.meta.tables).forEach(tableName =>
-          this.getAlias(tableName)
-        )
-      }
       if (json.relationships) {
         json.relationships = json.relationships.map(relationship => ({
           ...relationship,
@@ -233,6 +231,12 @@ export default class AliasTables {
           ]),
         }))
       }
+
+      this.getAlias(json.table)
+      for (const tableName of Object.keys(json.tables)) {
+        this.getAlias(tableName)
+      }
+
       // invert and return
       const invertedTableAliases: Record<string, string> = {}
       for (let [key, value] of Object.entries(this.tableAliases)) {
@@ -241,7 +245,7 @@ export default class AliasTables {
       json.tableAliases = invertedTableAliases
     }
 
-    let response: DatasourcePlusQueryResponse = await queryFn(datasource, json)
+    let response = await queryFn(json)
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {

packages/server/src/sdk/app/rows/utils.ts

@@ -14,13 +14,13 @@ import {
   SqlClient,
   ArrayOperator,
   ViewV2,
+  EnrichedQueryJson,
 } from "@budibase/types"
-import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
-import { docIds, sql } from "@budibase/backend-core"
+import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
 import env from "../../../environment"
 
@@ -73,18 +73,32 @@ export function processRowCountResponse(
   }
 }
 
-export async function getDatasourceAndQuery(
+export async function enrichQueryJson(
   json: QueryJson
-): Promise<DatasourcePlusQueryResponse> {
+): Promise<EnrichedQueryJson> {
-  const datasourceId = json.endpoint.datasourceId
+  let datasource: Datasource | undefined = undefined
-  const datasource = await sdk.datasources.get(datasourceId)
+  if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) {
-  const table = datasource.entities?.[json.endpoint.entityId]
+    datasource = await sdk.datasources.get(json.endpoint.datasourceId)
-  if (!json.meta && table) {
+  }
-    json.meta = {
+
+  let tables: Record<string, Table>
+  if (datasource) {
+    tables = datasource.entities || {}
+  } else {
+    tables = {}
+    for (const table of await sdk.tables.getAllInternalTables()) {
+      tables[table._id!] = table
+    }
+  }
+
+  const table = tables[json.endpoint.entityId]
+
+  return {
     table,
+    tables,
+    datasource,
+    ...json,
   }
-  }
-  return makeExternalQuery(datasource, json)
 }
 
 export function cleanExportRows(

packages/types/src/sdk/datasources.ts

@@ -1,5 +1,5 @@
 import { Table, Row } from "../documents"
-import { QueryJson } from "./search"
+import { EnrichedQueryJson } from "./search"
 
 export const PASSWORD_REPLACEMENT = "--secret-value--"
 
@@ -207,7 +207,7 @@ export interface DatasourcePlus extends IntegrationBase {
   // this returns the format of the identifier
   getBindingIdentifier(): string
   getStringConcat(parts: string[]): string
-  query(json: QueryJson): Promise<DatasourcePlusQueryResponse>
+  query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse>
   buildSchema(
     datasourceId: string,
     entities: Record<string, Table>

packages/types/src/sdk/search.ts

@@ -1,5 +1,5 @@
 import { Operation } from "./datasources"
-import { Row, Table, DocumentType } from "../documents"
+import { Row, DocumentType, Table, Datasource } from "../documents"
 import { SortOrder, SortType } from "../api"
 import { Knex } from "knex"
 import { Aggregation } from "./row"
@@ -171,10 +171,7 @@ export interface QueryJson {
   sort?: SortJson
   paginate?: PaginationJson
   body?: Row | Row[]
-  table?: Table
+  meta?: {
-  meta: {
-    table: Table
-    tables?: Record<string, Table>
     renamed?: RenameColumn
     // can specify something that columns could be prefixed with
     columnPrefix?: string
@@ -186,6 +183,12 @@ export interface QueryJson {
   tableAliases?: Record<string, string>
 }
 
+export interface EnrichedQueryJson extends QueryJson {
+  table: Table
+  tables: Record<string, Table>
+  datasource?: Datasource
+}
+
 export interface QueryOptions {
   disableReturning?: boolean
   disableBindings?: boolean