Merge branch 'master' of github.com:budibase/budibase into budi-8220-support-updating-existing-rows-for-external-databases-using-2

packages/backend-core/src/constants/db.ts

@@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
 export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
 export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
+export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core"

packages/backend-core/src/sql/sqlTable.ts

@@ -109,8 +109,10 @@ function generateSchema(
           const { tableName } = breakExternalTableId(column.tableId)
           // @ts-ignore
           const relatedTable = tables[tableName]
-          if (!relatedTable) {
-            throw new Error("Referenced table doesn't exist")
+          if (!relatedTable || !relatedTable.primary) {
+            throw new Error(
+              "Referenced table doesn't exist or has no primary keys"
+            )
           }
           const relatedPrimary = relatedTable.primary[0]
           const externalType = relatedTable.schema[relatedPrimary].externalType

packages/backend-core/src/sql/utils.ts

@@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
-export function breakExternalTableId(tableId: string | undefined) {
-  if (!tableId) {
-    return {}
-  }
+export function breakExternalTableId(tableId: string) {
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let datasourceId = parts.shift()
   // if they need joined
@@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) {
   if (tableName.includes(ENCODED_SPACE)) {
     tableName = decodeURIComponent(tableName)
   }
+  if (!datasourceId || !tableName) {
+    throw new Error("Unable to get datasource/table name from table ID")
+  }
   return { datasourceId, tableName }
 }
 

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -18,7 +18,7 @@
   import FilterUsers from "./FilterUsers.svelte"
   import { getFields } from "../utils/searchFields"
 
-  const { OperatorOptions } = Constants
+  const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants
 
   export let schemaFields
   export let filters = []
@@ -28,6 +28,23 @@
   export let allowBindings = false
   export let filtersLabel = "Filters"
 
+  $: {
+    if (
+      tables.find(
+        table =>
+          table._id === datasource.tableId &&
+          table.sourceId === DEFAULT_BB_DATASOURCE_ID
+      ) &&
+      !schemaFields.some(field => field.name === "_id")
+    ) {
+      schemaFields = [
+        ...schemaFields,
+        { name: "_id", type: "string" },
+        { name: "_rev", type: "string" },
+      ]
+    }
+  }
+
   $: matchAny = filters?.find(filter => filter.operator === "allOr") != null
   $: onEmptyFilter =
     filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
@@ -35,7 +52,6 @@
   $: fieldFilters = filters.filter(
     filter => filter.operator !== "allOr" && !filter.onEmptyFilter
   )
-
   const behaviourOptions = [
     { value: "and", label: "Match all filters" },
     { value: "or", label: "Match any filter" },
@@ -44,7 +60,6 @@
     { value: "all", label: "Return all table rows" },
     { value: "none", label: "Return no rows" },
   ]
-
   const context = getContext("context")
 
   $: fieldOptions = getFields(tables, schemaFields || [], {

packages/frontend-core/src/constants.js

@@ -1,7 +1,11 @@
 /**
  * Operator options for lucene queries
  */
-export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
+export {
+  OperatorOptions,
+  SqlNumberTypeRangeMap,
+  DEFAULT_BB_DATASOURCE_ID,
+} from "@budibase/shared-core"
 export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
 import { FieldType, BBReferenceFieldSubType } from "@budibase/types"

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -72,92 +72,6 @@ export type ExternalRequestReturnType<T extends Operation> =
     ? number
     : { row: Row; table: Table }
 
-function buildFilters(
-  id: string | undefined | string[],
-  filters: SearchFilters,
-  table: Table
-) {
-  const primary = table.primary
-  // if passed in array need to copy for shifting etc
-  let idCopy: undefined | string | any[] = cloneDeep(id)
-  if (filters) {
-    // need to map over the filters and make sure the _id field isn't present
-    let prefix = 1
-    for (let operator of Object.values(filters)) {
-      for (let field of Object.keys(operator || {})) {
-        if (dbCore.removeKeyNumbering(field) === "_id") {
-          if (primary) {
-            const parts = breakRowIdField(operator[field])
-            for (let field of primary) {
-              operator[`${prefix}:${field}`] = parts.shift()
-            }
-            prefix++
-          }
-          // make sure this field doesn't exist on any filter
-          delete operator[field]
-        }
-      }
-    }
-  }
-  // there is no id, just use the user provided filters
-  if (!idCopy || !table) {
-    return filters
-  }
-  // if used as URL parameter it will have been joined
-  if (!Array.isArray(idCopy)) {
-    idCopy = breakRowIdField(idCopy)
-  }
-  const equal: any = {}
-  if (primary && idCopy) {
-    for (let field of primary) {
-      // work through the ID and get the parts
-      equal[field] = idCopy.shift()
-    }
-  }
-  return {
-    equal,
-  }
-}
-
-async function removeManyToManyRelationships(
-  rowId: string,
-  table: Table,
-  colName: string
-) {
-  const tableId = table._id!
-  const filters = buildFilters(rowId, {}, table)
-  // safety check, if there are no filters on deletion bad things happen
-  if (Object.keys(filters).length !== 0) {
-    return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, Operation.DELETE),
-      body: { [colName]: null },
-      filters,
-      meta: {
-        table,
-      },
-    })
-  } else {
-    return []
-  }
-}
-
-async function removeOneToManyRelationships(rowId: string, table: Table) {
-  const tableId = table._id!
-  const filters = buildFilters(rowId, {}, table)
-  // safety check, if there are no filters on deletion bad things happen
-  if (Object.keys(filters).length !== 0) {
-    return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, Operation.UPDATE),
-      filters,
-      meta: {
-        table,
-      },
-    })
-  } else {
-    return []
-  }
-}
-
 /**
  * This function checks the incoming parameters to make sure all the inputs are
  * valid based on on the table schema. The main thing this is looking for is when a
@@ -212,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) {
   }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   return {
-    datasourceId: datasourceId!,
-    entityId: tableName!,
+    datasourceId: datasourceId,
+    entityId: tableName,
     operation: operation as Operation,
   }
 }
@@ -240,6 +154,7 @@ export class ExternalRequest<T extends Operation> {
   private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
+  private tableList: Table[]
 
   constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
@@ -248,22 +163,134 @@ export class ExternalRequest<T extends Operation> {
     if (datasource && datasource.entities) {
       this.tables = datasource.entities
     }
+    this.tableList = Object.values(this.tables)
+  }
+
+  private prepareFilters(
+    id: string | undefined | string[],
+    filters: SearchFilters,
+    table: Table
+  ): SearchFilters {
+    // replace any relationship columns initially, table names and relationship column names are acceptable
+    const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table)
+    filters = sdk.rows.filters.updateFilterKeys(
+      filters,
+      relationshipColumns.map(({ name, definition }) => {
+        const { tableName } = breakExternalTableId(definition.tableId)
+        return {
+          original: name,
+          updated: tableName,
+        }
+      })
+    )
+    const primary = table.primary
+    // if passed in array need to copy for shifting etc
+    let idCopy: undefined | string | any[] = cloneDeep(id)
+    if (filters) {
+      // need to map over the filters and make sure the _id field isn't present
+      let prefix = 1
+      for (let operator of Object.values(filters)) {
+        for (let field of Object.keys(operator || {})) {
+          if (dbCore.removeKeyNumbering(field) === "_id") {
+            if (primary) {
+              const parts = breakRowIdField(operator[field])
+              for (let field of primary) {
+                operator[`${prefix}:${field}`] = parts.shift()
+              }
+              prefix++
+            }
+            // make sure this field doesn't exist on any filter
+            delete operator[field]
+          }
+        }
+      }
+    }
+    // there is no id, just use the user provided filters
+    if (!idCopy || !table) {
+      return filters
+    }
+    // if used as URL parameter it will have been joined
+    if (!Array.isArray(idCopy)) {
+      idCopy = breakRowIdField(idCopy)
+    }
+    const equal: SearchFilters["equal"] = {}
+    if (primary && idCopy) {
+      for (let field of primary) {
+        // work through the ID and get the parts
+        equal[field] = idCopy.shift()
+      }
+    }
+    return {
+      equal,
+    }
+  }
+
+  private async removeManyToManyRelationships(
+    rowId: string,
+    table: Table,
+    colName: string
+  ) {
+    const tableId = table._id!
+    const filters = this.prepareFilters(rowId, {}, table)
+    // safety check, if there are no filters on deletion bad things happen
+    if (Object.keys(filters).length !== 0) {
+      return getDatasourceAndQuery({
+        endpoint: getEndpoint(tableId, Operation.DELETE),
+        body: { [colName]: null },
+        filters,
+        meta: {
+          table,
+        },
+      })
+    } else {
+      return []
+    }
+  }
+
+  private async removeOneToManyRelationships(rowId: string, table: Table) {
+    const tableId = table._id!
+    const filters = this.prepareFilters(rowId, {}, table)
+    // safety check, if there are no filters on deletion bad things happen
+    if (Object.keys(filters).length !== 0) {
+      return getDatasourceAndQuery({
+        endpoint: getEndpoint(tableId, Operation.UPDATE),
+        filters,
+        meta: {
+          table,
+        },
+      })
+    } else {
+      return []
+    }
   }
 
   getTable(tableId: string | undefined): Table | undefined {
     if (!tableId) {
-      throw "Table ID is unknown, cannot find table"
+      throw new Error("Table ID is unknown, cannot find table")
     }
     const { tableName } = breakExternalTableId(tableId)
-    if (tableName) {
-      return this.tables[tableName]
+    return this.tables[tableName]
+  }
+
+  // seeds the object with table and datasource information
+  async retrieveMetadata(
+    datasourceId: string
+  ): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
+    if (!this.datasource) {
+      this.datasource = await sdk.datasources.get(datasourceId)
+      if (!this.datasource || !this.datasource.entities) {
+        throw "No tables found, fetch tables before query."
+      }
+      this.tables = this.datasource.entities
+      this.tableList = Object.values(this.tables)
     }
+    return { tables: this.tables, datasource: this.datasource }
   }
 
   async getRow(table: Table, rowId: string): Promise<Row> {
     const response = await getDatasourceAndQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
-      filters: buildFilters(rowId, {}, table),
+      filters: this.prepareFilters(rowId, {}, table),
       meta: {
         table,
       },
@@ -302,9 +329,7 @@ export class ExternalRequest<T extends Operation> {
       if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
         newRow[key] = parseFloat(row[key])
       } else if (field.type === FieldType.LINK) {
-        const { tableName: linkTableName } = breakExternalTableId(
-          field?.tableId
-        )
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
         // table has to exist for many to many
         if (!linkTableName || !this.tables[linkTableName]) {
           continue
@@ -385,9 +410,6 @@ export class ExternalRequest<T extends Operation> {
       [key: string]: { rows: Row[]; isMany: boolean; tableId: string }
     } = {}
     const { tableName } = breakExternalTableId(tableId)
-    if (!tableName) {
-      return related
-    }
     const table = this.tables[tableName]
     // @ts-ignore
     const primaryKey = table.primary[0]
@@ -520,7 +542,7 @@ export class ExternalRequest<T extends Operation> {
             endpoint: getEndpoint(tableId, operation),
             // if we're doing many relationships then we're writing, only one response
             body,
-            filters: buildFilters(id, {}, linkTable),
+            filters: this.prepareFilters(id, {}, linkTable),
             meta: {
               table: linkTable,
             },
@@ -544,8 +566,8 @@ export class ExternalRequest<T extends Operation> {
       for (let row of rows) {
         const rowId = generateIdForRow(row, table)
         const promise: Promise<any> = isMany
-          ? removeManyToManyRelationships(rowId, table, colName)
-          : removeOneToManyRelationships(rowId, table)
+          ? this.removeManyToManyRelationships(rowId, table, colName)
+          : this.removeOneToManyRelationships(rowId, table)
         if (promise) {
           promises.push(promise)
         }
@@ -568,12 +590,12 @@ export class ExternalRequest<T extends Operation> {
         rows.map(row => {
           const rowId = generateIdForRow(row, table)
           return isMany
-            ? removeManyToManyRelationships(
+            ? this.removeManyToManyRelationships(
                 rowId,
                 table,
                 relationshipColumn.fieldName
               )
-            : removeOneToManyRelationships(rowId, table)
+            : this.removeOneToManyRelationships(rowId, table)
         })
       )
     }
@@ -581,21 +603,21 @@ export class ExternalRequest<T extends Operation> {
 
   async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    if (!tableName) {
-      throw "Unable to run without a table name"
+    if (!tableId) {
+      throw new Error("Unable to run without a table ID")
     }
-    if (!this.datasource) {
-      this.datasource = await sdk.datasources.get(datasourceId!)
-      if (!this.datasource || !this.datasource.entities) {
-        throw "No tables found, fetch tables before query."
-      }
-      this.tables = this.datasource.entities
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    let datasource = this.datasource
+    if (!datasource) {
+      const { datasource: ds } = await this.retrieveMetadata(datasourceId)
+      datasource = ds
     }
     const table = this.tables[tableName]
-    let isSql = isSQL(this.datasource)
+    let isSql = isSQL(datasource)
     if (!table) {
-      throw `Unable to process query, table "${tableName}" not defined.`
+      throw new Error(
+        `Unable to process query, table "${tableName}" not defined.`
+      )
     }
     // look for specific components of config which may not be considered acceptable
     let { id, row, filters, sort, paginate, rows } = cleanupConfig(
@@ -618,7 +640,7 @@ export class ExternalRequest<T extends Operation> {
           break
       }
     }
-    filters = buildFilters(id, filters || {}, table)
+    filters = this.prepareFilters(id, filters || {}, table)
     const relationships = buildExternalRelationships(table, this.tables)
 
     const incRelationships =
@@ -666,7 +688,11 @@ export class ExternalRequest<T extends Operation> {
       body: row || rows,
       // pass an id filter into extra, purely for mysql/returning
       extra: {
-        idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+        idFilter: this.prepareFilters(
+          id || generateIdForRow(row, table),
+          {},
+          table
+        ),
       },
       meta: {
         table,

packages/server/src/api/controllers/row/external.ts

@@ -136,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
-  if (!tableName) {
-    ctx.throw(400, "Unable to find table.")
-  }
+  const datasource: Datasource = await sdk.datasources.get(datasourceId)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
@@ -163,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     }
     const links = row[fieldName]
     const linkedTableId = field.tableId
-    const linkedTableName = breakExternalTableId(linkedTableId).tableName!
+    const linkedTableName = breakExternalTableId(linkedTableId).tableName
     const linkedTable = tables[linkedTableName]
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])

packages/server/src/api/controllers/row/utils/basic.ts

@@ -99,7 +99,7 @@ export function basicProcessing({
         row,
         tableName: table._id!,
         fieldName: internalColumn,
-        isLinked: false,
+        isLinked,
       })
     }
   }

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -2,6 +2,8 @@ import {
   DatasourcePlusQueryResponse,
   DSPlusOperation,
   FieldType,
+  isManyToOne,
+  isOneToMany,
   ManyToManyRelationshipFieldMetadata,
   RelationshipFieldMetadata,
   RelationshipsJson,
@@ -93,12 +95,12 @@ export function buildExternalRelationships(
 ): RelationshipsJson[] {
   const relationships = []
   for (let [fieldName, field] of Object.entries(table.schema)) {
-    if (field.type !== FieldType.LINK) {
+    if (field.type !== FieldType.LINK || !field.tableId) {
       continue
     }
     const { tableName: linkTableName } = breakExternalTableId(field.tableId)
     // no table to link to, this is not a valid relationships
-    if (!linkTableName || !tables[linkTableName]) {
+    if (!tables[linkTableName]) {
       continue
     }
     const linkTable = tables[linkTableName]
@@ -110,7 +112,7 @@ export function buildExternalRelationships(
       // need to specify where to put this back into
       column: fieldName,
     }
-    if (isManyToMany(field)) {
+    if (isManyToMany(field) && field.through) {
       const { tableName: throughTableName } = breakExternalTableId(
         field.through
       )
@@ -120,7 +122,7 @@ export function buildExternalRelationships(
       definition.to = field.throughFrom || linkTable.primary[0]
       definition.fromPrimary = table.primary[0]
       definition.toPrimary = linkTable.primary[0]
-    } else {
+    } else if (isManyToOne(field) || isOneToMany(field)) {
       // if no foreign key specified then use the name of the field in other table
       definition.from = field.foreignKey || table.primary[0]
       definition.to = field.fieldName
@@ -180,16 +182,18 @@ export function buildSqlFieldList(
   }
   let fields = extractRealFields(table)
   for (let field of Object.values(table.schema)) {
-    if (field.type !== FieldType.LINK || !opts?.relationships) {
+    if (
+      field.type !== FieldType.LINK ||
+      !opts?.relationships ||
+      !field.tableId
+    ) {
       continue
     }
     const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-    if (linkTableName) {
-      const linkTable = tables[linkTableName]
-      if (linkTable) {
-        const linkedFields = extractRealFields(linkTable, fields)
-        fields = fields.concat(linkedFields)
-      }
+    const linkTable = tables[linkTableName]
+    if (linkTable) {
+      const linkedFields = extractRealFields(linkTable, fields)
+      fields = fields.concat(linkedFields)
     }
   }
   return fields

packages/server/src/api/controllers/table/external.ts

@@ -20,11 +20,14 @@ import { isEqual } from "lodash"
 
 function getDatasourceId(table: Table) {
   if (!table) {
-    throw "No table supplied"
+    throw new Error("No table supplied")
   }
   if (table.sourceId) {
     return table.sourceId
   }
+  if (!table._id) {
+    throw new Error("No table ID supplied")
+  }
   return breakExternalTableId(table._id).datasourceId
 }
 

packages/server/src/api/routes/tests/search.spec.ts

@@ -76,9 +76,9 @@ describe.each([
     }
   })
 
-  async function createTable(schema: TableSchema) {
+  async function createTable(schema: TableSchema, name?: string) {
     return await config.api.table.save(
-      tableForDatasource(datasource, { schema })
+      tableForDatasource(datasource, { schema, name })
     )
   }
 
@@ -1956,53 +1956,74 @@ describe.each([
     // isn't available.
     !isInMemory &&
     describe("relations", () => {
-      let otherTable: Table
-      let otherRows: Row[]
+      let productCategoryTable: Table, productCatRows: Row[]
 
       beforeAll(async () => {
-        otherTable = await createTable({
-          one: { name: "one", type: FieldType.STRING },
-        })
-        table = await createTable({
-          two: { name: "two", type: FieldType.STRING },
-          other: {
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-            name: "other",
-            fieldName: "other",
-            tableId: otherTable._id!,
-            constraints: {
-              type: "array",
+        productCategoryTable = await createTable(
+          {
+            name: { name: "name", type: FieldType.STRING },
+          },
+          "productCategory"
+        )
+        table = await createTable(
+          {
+            name: { name: "name", type: FieldType.STRING },
+            productCat: {
+              type: FieldType.LINK,
+              relationshipType: RelationshipType.ONE_TO_MANY,
+              name: "productCat",
+              fieldName: "product",
+              tableId: productCategoryTable._id!,
+              constraints: {
+                type: "array",
+              },
             },
           },
-        })
+          "product"
+        )
 
-        otherRows = await Promise.all([
-          config.api.row.save(otherTable._id!, { one: "foo" }),
-          config.api.row.save(otherTable._id!, { one: "bar" }),
+        productCatRows = await Promise.all([
+          config.api.row.save(productCategoryTable._id!, { name: "foo" }),
+          config.api.row.save(productCategoryTable._id!, { name: "bar" }),
         ])
 
         await Promise.all([
           config.api.row.save(table._id!, {
-            two: "foo",
-            other: [otherRows[0]._id],
+            name: "foo",
+            productCat: [productCatRows[0]._id],
           }),
           config.api.row.save(table._id!, {
-            two: "bar",
-            other: [otherRows[1]._id],
+            name: "bar",
+            productCat: [productCatRows[1]._id],
+          }),
+          config.api.row.save(table._id!, {
+            name: "baz",
+            productCat: [],
           }),
         ])
-
-        rows = await config.api.row.fetch(table._id!)
       })
 
-      it("can search through relations", async () => {
+      it("should be able to filter by relationship using column name", async () => {
         await expectQuery({
-          equal: { [`${otherTable.name}.one`]: "foo" },
+          equal: { ["productCat.name"]: "foo" },
         }).toContainExactly([
-          { two: "foo", other: [{ _id: otherRows[0]._id }] },
+          { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
         ])
       })
+
+      it("should be able to filter by relationship using table name", async () => {
+        await expectQuery({
+          equal: { ["productCategory.name"]: "foo" },
+        }).toContainExactly([
+          { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+        ])
+      })
+
+      it("shouldn't return any relationship for last row", async () => {
+        await expectQuery({
+          equal: { ["name"]: "baz" },
+        }).toContainExactly([{ name: "baz", productCat: undefined }])
+      })
     })
 
   // lucene can't count the total rows

packages/server/src/sdk/app/rows/index.ts

@@ -3,12 +3,14 @@ import * as rows from "./rows"
 import * as search from "./search"
 import * as utils from "./utils"
 import * as external from "./external"
+import * as filters from "./search/filters"
 import AliasTables from "./sqlAlias"
 
 export default {
   ...attachments,
   ...rows,
   ...search,
+  filters,
   utils,
   external,
   AliasTables,

packages/server/src/sdk/app/rows/search/external.ts

@@ -145,6 +145,10 @@ export async function exportRows(
     delimiter,
     customHeaders,
   } = options
+
+  if (!tableId) {
+    throw new HTTPError("No table ID for search provided.", 400)
+  }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
 
   let requestQuery: SearchFilters = {}
@@ -167,7 +171,7 @@ export async function exportRows(
     requestQuery = query || {}
   }
 
-  const datasource = await sdk.datasources.get(datasourceId!)
+  const datasource = await sdk.datasources.get(datasourceId)
   const table = await sdk.tables.getTable(tableId)
   if (!datasource || !datasource.entities) {
     throw new HTTPError("Datasource has not been configured for plus API.", 400)
@@ -180,10 +184,6 @@ export async function exportRows(
   let rows: Row[] = []
   let headers
 
-  if (!tableName) {
-    throw new HTTPError("Could not find table name.", 400)
-  }
-
   // Filter data to only specified columns if required
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {

packages/server/src/sdk/app/rows/search/filters.ts

@@ -0,0 +1,62 @@
+import {
+  FieldType,
+  RelationshipFieldMetadata,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+import { isPlainObject } from "lodash"
+
+export function getRelationshipColumns(table: Table): {
+  name: string
+  definition: RelationshipFieldMetadata
+}[] {
+  // performing this with a for loop rather than an array filter improves
+  // type guarding, as no casts are required
+  const linkEntries: [string, RelationshipFieldMetadata][] = []
+  for (let entry of Object.entries(table.schema)) {
+    if (entry[1].type === FieldType.LINK) {
+      const linkColumn: RelationshipFieldMetadata = entry[1]
+      linkEntries.push([entry[0], linkColumn])
+    }
+  }
+  return linkEntries.map(entry => ({
+    name: entry[0],
+    definition: entry[1],
+  }))
+}
+
+export function getTableIDList(
+  tables: Table[]
+): { name: string; id: string }[] {
+  return tables
+    .filter(table => table.originalName && table._id)
+    .map(table => ({ id: table._id!, name: table.originalName! }))
+}
+
+export function updateFilterKeys(
+  filters: SearchFilters,
+  updates: { original: string; updated: string }[]
+): SearchFilters {
+  const makeFilterKeyRegex = (str: string) =>
+    new RegExp(`^${str}\\.|:${str}\\.`)
+  for (let filter of Object.values(filters)) {
+    if (!isPlainObject(filter)) {
+      continue
+    }
+    for (let [key, keyFilter] of Object.entries(filter)) {
+      if (keyFilter === "") {
+        delete filter[key]
+      }
+      const possibleKey = updates.find(({ original }) =>
+        key.match(makeFilterKeyRegex(original))
+      )
+      if (possibleKey && possibleKey.original !== possibleKey.updated) {
+        // only replace the first, not replaceAll
+        filter[key.replace(possibleKey.original, possibleKey.updated)] =
+          filter[key]
+        delete filter[key]
+      }
+    }
+  }
+  return filters
+}

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -1,4 +1,5 @@
 import {
+  Datasource,
   DocumentType,
   FieldType,
   Operation,
@@ -12,7 +13,6 @@ import {
   SortType,
   SqlClient,
   Table,
-  Datasource,
 } from "@budibase/types"
 import {
   buildInternalRelationships,
@@ -30,6 +30,11 @@ import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
 import { processRowCountResponse } from "../utils"
+import {
+  updateFilterKeys,
+  getRelationshipColumns,
+  getTableIDList,
+} from "./filters"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 
@@ -60,34 +65,31 @@ function buildInternalFieldList(
   return fieldList
 }
 
-function tableNameInFieldRegex(tableName: string) {
-  return new RegExp(`^${tableName}.|:${tableName}.`, "g")
-}
-
-function cleanupFilters(filters: SearchFilters, tables: Table[]) {
-  for (let filter of Object.values(filters)) {
-    if (typeof filter !== "object") {
-      continue
-    }
-    for (let [key, keyFilter] of Object.entries(filter)) {
-      if (keyFilter === "") {
-        delete filter[key]
-      }
-
-      // relationship, switch to table ID
-      const tableRelated = tables.find(
-        table =>
-          table.originalName &&
-          key.match(tableNameInFieldRegex(table.originalName))
+function cleanupFilters(
+  filters: SearchFilters,
+  table: Table,
+  allTables: Table[]
+) {
+  // get a list of all relationship columns in the table for updating
+  const relationshipColumns = getRelationshipColumns(table)
+  // get table names to ID map for relationships
+  const tableNameToID = getTableIDList(allTables)
+  // all should be applied at once
+  filters = updateFilterKeys(
+    filters,
+    relationshipColumns
+      .map(({ name, definition }) => ({
+        original: name,
+        updated: definition.tableId,
+      }))
+      .concat(
+        tableNameToID.map(({ name, id }) => ({
+          original: name,
+          updated: id,
+        }))
       )
-      if (tableRelated && tableRelated.originalName) {
-        // only replace the first, not replaceAll
-        filter[key.replace(tableRelated.originalName, tableRelated._id!)] =
-          filter[key]
-        delete filter[key]
-      }
-    }
-  }
+  )
+
   return filters
 }
 
@@ -176,7 +178,7 @@ export async function search(
       operation: Operation.READ,
     },
     filters: {
-      ...cleanupFilters(query, allTables),
+      ...cleanupFilters(query, table, allTables),
       documentType: DocumentType.ROW,
     },
     table,

packages/server/src/sdk/app/tables/getters.ts

@@ -90,10 +90,10 @@ export async function getExternalTable(
 export async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
   let output: Table
-  if (isExternalTableID(tableId)) {
+  if (tableId && isExternalTableID(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await datasources.get(datasourceId!)
-    const table = await getExternalTable(datasourceId!, tableName!)
+    const datasource = await datasources.get(datasourceId)
+    const table = await getExternalTable(datasourceId, tableName)
     output = { ...table, sql: isSQL(datasource) }
   } else {
     output = await db.get<Table>(tableId)

packages/server/src/sdk/app/views/external.ts

@@ -10,9 +10,9 @@ export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
 
-  const table = ds.entities![tableName!]
+  const table = ds.entities![tableName]
   const views = Object.values(table.views!).filter(isV2)
   const found = views.find(v => v.id === viewId)
   if (!found) {
@@ -25,9 +25,9 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
 
-  const table = ds.entities![tableName!]
+  const table = ds.entities![tableName]
   const views = Object.values(table.views!).filter(isV2)
   const found = views.find(v => v.id === viewId)
   if (!found) {
@@ -49,9 +49,9 @@ export async function create(
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
-  ds.entities![tableName!].views ??= {}
-  ds.entities![tableName!].views![view.name] = view
+  const ds = await sdk.datasources.get(datasourceId)
+  ds.entities![tableName].views ??= {}
+  ds.entities![tableName].views![view.name] = view
   await db.put(ds)
   return view
 }
@@ -60,9 +60,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
-  ds.entities![tableName!].views ??= {}
-  const views = ds.entities![tableName!].views!
+  const ds = await sdk.datasources.get(datasourceId)
+  ds.entities![tableName].views ??= {}
+  const views = ds.entities![tableName].views!
 
   const existingView = Object.values(views).find(
     v => isV2(v) && v.id === view.id
@@ -87,9 +87,9 @@ export async function remove(viewId: string): Promise<ViewV2> {
   }
 
   const { datasourceId, tableName } = breakExternalTableId(view.tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
 
-  delete ds.entities![tableName!].views![view?.name]
+  delete ds.entities![tableName].views![view?.name]
   await db.put(ds)
   return view
 }

packages/shared-core/src/constants/index.ts

@@ -180,3 +180,5 @@ export enum BpmStatusValue {
   VERIFYING_EMAIL = "verifying_email",
   COMPLETED = "completed",
 }
+
+export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"