Merge branch 'master' of github.com:Budibase/budibase into table-improvements

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.29.0",
+  "version": "2.29.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 247f56d455abbd64da17d865275ed978f577549f
+Subproject commit b600cca314a5cc9971e44d46047d1a0019b46b08

packages/backend-core/src/constants/db.ts

@@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
 export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
 export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
+export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core"

packages/backend-core/src/db/constants.ts

@@ -1,14 +1,5 @@
-export const CONSTANT_INTERNAL_ROW_COLS = [
+export {
-  "_id",
+  CONSTANT_INTERNAL_ROW_COLS,
-  "_rev",
+  CONSTANT_EXTERNAL_ROW_COLS,
-  "type",
+  isInternalColumnName,
-  "createdAt",
+} from "@budibase/shared-core"
-  "updatedAt",
-  "tableId",
-] as const
-
-export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
-
-export function isInternalColumnName(name: string): boolean {
-  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
-}

packages/backend-core/src/sql/sqlTable.ts

@@ -109,8 +109,10 @@ function generateSchema(
           const { tableName } = breakExternalTableId(column.tableId)
           // @ts-ignore
           const relatedTable = tables[tableName]
-          if (!relatedTable) {
+          if (!relatedTable || !relatedTable.primary) {
-            throw new Error("Referenced table doesn't exist")
+            throw new Error(
+              "Referenced table doesn't exist or has no primary keys"
+            )
           }
           const relatedPrimary = relatedTable.primary[0]
           const externalType = relatedTable.schema[relatedPrimary].externalType

packages/backend-core/src/sql/utils.ts

@@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
-export function breakExternalTableId(tableId: string | undefined) {
+export function breakExternalTableId(tableId: string) {
-  if (!tableId) {
-    return {}
-  }
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let datasourceId = parts.shift()
   // if they need joined
@@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) {
   if (tableName.includes(ENCODED_SPACE)) {
     tableName = decodeURIComponent(tableName)
   }
+  if (!datasourceId || !tableName) {
+    throw new Error("Unable to get datasource/table name from table ID")
+  }
   return { datasourceId, tableName }
 }
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -17,6 +17,8 @@
     SWITCHABLE_TYPES,
     ValidColumnNameRegex,
     helpers,
+    CONSTANT_INTERNAL_ROW_COLS,
+    CONSTANT_EXTERNAL_ROW_COLS,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -52,7 +54,6 @@
   const DATE_TYPE = FieldType.DATETIME
 
   const dispatch = createEventDispatcher()
-  const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
   const { dispatch: gridDispatch, rows } = getContext("grid")
 
   export let field
@@ -487,20 +488,27 @@
       })
     }
     const newError = {}
+    const prohibited = externalTable
+      ? CONSTANT_EXTERNAL_ROW_COLS
+      : CONSTANT_INTERNAL_ROW_COLS
     if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
-    } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
+    } else if (
-      newError.name = `${PROHIBITED_COLUMN_NAMES.join(
+      prohibited.some(
+        name => fieldInfo?.name?.toLowerCase() === name.toLowerCase()
+      )
+    ) {
+      newError.name = `${prohibited.join(
         ", "
-      )} are not allowed as column names`
+      )} are not allowed as column names - case insensitive.`
     } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
 
     if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) {
-      newError.subtype = `Auto Column requires a type`
+      newError.subtype = `Auto Column requires a type.`
     }
 
     if (fieldInfo.fieldName && fieldInfo.tableId) {

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte

@@ -53,6 +53,12 @@
         placeholder="Are you sure you want to delete?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 </div>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -83,6 +83,12 @@
         placeholder="Are you sure you want to duplicate this row?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte

@@ -74,6 +74,18 @@
           placeholder="Are you sure you want to execute this query?"
           bind:value={parameters.confirmText}
         />
+
+        <Input
+          label="Confirm Text"
+          placeholder="Confirm"
+          bind:value={parameters.confirmButtonText}
+        />
+
+        <Input
+          label="Cancel Text"
+          placeholder="Cancel"
+          bind:value={parameters.cancelButtonText}
+        />
       {/if}
 
       {#if query?.parameters?.length > 0}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -80,6 +80,12 @@
         placeholder="Are you sure you want to save this row?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 

packages/client/src/components/overlay/ConfirmationDisplay.svelte

@@ -8,6 +8,8 @@
     <ModalContent
       title={$confirmationStore.title}
       onConfirm={confirmationStore.actions.confirm}
+      confirmText={$confirmationStore.confirmButtonText}
+      cancelText={$confirmationStore.cancelButtonText}
     >
       {$confirmationStore.text}
     </ModalContent>

packages/client/src/stores/confirmation.js

@@ -4,6 +4,8 @@ const initialState = {
   showConfirmation: false,
   title: null,
   text: null,
+  confirmButtonText: null,
+  cancelButtonText: null,
   onConfirm: null,
   onCancel: null,
 }
@@ -11,11 +13,20 @@ const initialState = {
 const createConfirmationStore = () => {
   const store = writable(initialState)
 
-  const showConfirmation = (title, text, onConfirm, onCancel) => {
+  const showConfirmation = (
+    title,
+    text,
+    onConfirm,
+    onCancel,
+    confirmButtonText,
+    cancelButtonText
+  ) => {
     store.set({
       showConfirmation: true,
       title,
       text,
+      confirmButtonText,
+      cancelButtonText,
       onConfirm,
       onCancel,
     })

packages/client/src/utils/buttonActions.js

@@ -522,6 +522,7 @@ const confirmTextMap = {
   ["Execute Query"]: "Are you sure you want to execute this query?",
   ["Trigger Automation"]: "Are you sure you want to trigger this automation?",
   ["Prompt User"]: "Are you sure you want to continue?",
+  ["Duplicate Row"]: "Are you sure you want to duplicate this row?",
 }
 
 /**
@@ -582,6 +583,11 @@ export const enrichButtonActions = (actions, context) => {
             const defaultTitleText = action["##eventHandlerType"]
             const customTitleText =
               action.parameters?.customTitleText || defaultTitleText
+            const cancelButtonText =
+              action.parameters?.cancelButtonText || "Cancel"
+            const confirmButtonText =
+              action.parameters?.confirmButtonText || "Confirm"
+
             confirmationStore.actions.showConfirmation(
               customTitleText,
               confirmText,
@@ -612,7 +618,9 @@ export const enrichButtonActions = (actions, context) => {
               },
               () => {
                 resolve(false)
-              }
+              },
+              confirmButtonText,
+              cancelButtonText
             )
           })
         }

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -18,7 +18,7 @@
   import FilterUsers from "./FilterUsers.svelte"
   import { getFields } from "../utils/searchFields"
 
-  const { OperatorOptions } = Constants
+  const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants
 
   export let schemaFields
   export let filters = []
@@ -28,6 +28,23 @@
   export let allowBindings = false
   export let filtersLabel = "Filters"
 
+  $: {
+    if (
+      tables.find(
+        table =>
+          table._id === datasource.tableId &&
+          table.sourceId === DEFAULT_BB_DATASOURCE_ID
+      ) &&
+      !schemaFields.some(field => field.name === "_id")
+    ) {
+      schemaFields = [
+        ...schemaFields,
+        { name: "_id", type: "string" },
+        { name: "_rev", type: "string" },
+      ]
+    }
+  }
+
   $: matchAny = filters?.find(filter => filter.operator === "allOr") != null
   $: onEmptyFilter =
     filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
@@ -35,7 +52,6 @@
   $: fieldFilters = filters.filter(
     filter => filter.operator !== "allOr" && !filter.onEmptyFilter
   )
-
   const behaviourOptions = [
     { value: "and", label: "Match all filters" },
     { value: "or", label: "Match any filter" },
@@ -44,7 +60,6 @@
     { value: "all", label: "Return all table rows" },
     { value: "none", label: "Return no rows" },
   ]
-
   const context = getContext("context")
 
   $: fieldOptions = getFields(tables, schemaFields || [], {

packages/frontend-core/src/constants.js

@@ -1,7 +1,11 @@
 /**
  * Operator options for lucene queries
  */
-export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
+export {
+  OperatorOptions,
+  SqlNumberTypeRangeMap,
+  DEFAULT_BB_DATASOURCE_ID,
+} from "@budibase/shared-core"
 export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
 import { FieldType, BBReferenceFieldSubType } from "@budibase/types"

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -72,92 +72,6 @@ export type ExternalRequestReturnType<T extends Operation> =
     ? number
     : { row: Row; table: Table }
 
-function buildFilters(
-  id: string | undefined | string[],
-  filters: SearchFilters,
-  table: Table
-) {
-  const primary = table.primary
-  // if passed in array need to copy for shifting etc
-  let idCopy: undefined | string | any[] = cloneDeep(id)
-  if (filters) {
-    // need to map over the filters and make sure the _id field isn't present
-    let prefix = 1
-    for (let operator of Object.values(filters)) {
-      for (let field of Object.keys(operator || {})) {
-        if (dbCore.removeKeyNumbering(field) === "_id") {
-          if (primary) {
-            const parts = breakRowIdField(operator[field])
-            for (let field of primary) {
-              operator[`${prefix}:${field}`] = parts.shift()
-            }
-            prefix++
-          }
-          // make sure this field doesn't exist on any filter
-          delete operator[field]
-        }
-      }
-    }
-  }
-  // there is no id, just use the user provided filters
-  if (!idCopy || !table) {
-    return filters
-  }
-  // if used as URL parameter it will have been joined
-  if (!Array.isArray(idCopy)) {
-    idCopy = breakRowIdField(idCopy)
-  }
-  const equal: any = {}
-  if (primary && idCopy) {
-    for (let field of primary) {
-      // work through the ID and get the parts
-      equal[field] = idCopy.shift()
-    }
-  }
-  return {
-    equal,
-  }
-}
-
-async function removeManyToManyRelationships(
-  rowId: string,
-  table: Table,
-  colName: string
-) {
-  const tableId = table._id!
-  const filters = buildFilters(rowId, {}, table)
-  // safety check, if there are no filters on deletion bad things happen
-  if (Object.keys(filters).length !== 0) {
-    return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, Operation.DELETE),
-      body: { [colName]: null },
-      filters,
-      meta: {
-        table,
-      },
-    })
-  } else {
-    return []
-  }
-}
-
-async function removeOneToManyRelationships(rowId: string, table: Table) {
-  const tableId = table._id!
-  const filters = buildFilters(rowId, {}, table)
-  // safety check, if there are no filters on deletion bad things happen
-  if (Object.keys(filters).length !== 0) {
-    return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, Operation.UPDATE),
-      filters,
-      meta: {
-        table,
-      },
-    })
-  } else {
-    return []
-  }
-}
-
 /**
  * This function checks the incoming parameters to make sure all the inputs are
  * valid based on on the table schema. The main thing this is looking for is when a
@@ -212,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) {
   }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   return {
-    datasourceId: datasourceId!,
+    datasourceId: datasourceId,
-    entityId: tableName!,
+    entityId: tableName,
     operation: operation as Operation,
   }
 }
@@ -240,6 +154,7 @@ export class ExternalRequest<T extends Operation> {
   private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
+  private tableList: Table[]
 
   constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
@@ -248,22 +163,134 @@ export class ExternalRequest<T extends Operation> {
     if (datasource && datasource.entities) {
       this.tables = datasource.entities
     }
+    this.tableList = Object.values(this.tables)
+  }
+
+  private prepareFilters(
+    id: string | undefined | string[],
+    filters: SearchFilters,
+    table: Table
+  ): SearchFilters {
+    // replace any relationship columns initially, table names and relationship column names are acceptable
+    const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table)
+    filters = sdk.rows.filters.updateFilterKeys(
+      filters,
+      relationshipColumns.map(({ name, definition }) => {
+        const { tableName } = breakExternalTableId(definition.tableId)
+        return {
+          original: name,
+          updated: tableName,
+        }
+      })
+    )
+    const primary = table.primary
+    // if passed in array need to copy for shifting etc
+    let idCopy: undefined | string | any[] = cloneDeep(id)
+    if (filters) {
+      // need to map over the filters and make sure the _id field isn't present
+      let prefix = 1
+      for (let operator of Object.values(filters)) {
+        for (let field of Object.keys(operator || {})) {
+          if (dbCore.removeKeyNumbering(field) === "_id") {
+            if (primary) {
+              const parts = breakRowIdField(operator[field])
+              for (let field of primary) {
+                operator[`${prefix}:${field}`] = parts.shift()
+              }
+              prefix++
+            }
+            // make sure this field doesn't exist on any filter
+            delete operator[field]
+          }
+        }
+      }
+    }
+    // there is no id, just use the user provided filters
+    if (!idCopy || !table) {
+      return filters
+    }
+    // if used as URL parameter it will have been joined
+    if (!Array.isArray(idCopy)) {
+      idCopy = breakRowIdField(idCopy)
+    }
+    const equal: SearchFilters["equal"] = {}
+    if (primary && idCopy) {
+      for (let field of primary) {
+        // work through the ID and get the parts
+        equal[field] = idCopy.shift()
+      }
+    }
+    return {
+      equal,
+    }
+  }
+
+  private async removeManyToManyRelationships(
+    rowId: string,
+    table: Table,
+    colName: string
+  ) {
+    const tableId = table._id!
+    const filters = this.prepareFilters(rowId, {}, table)
+    // safety check, if there are no filters on deletion bad things happen
+    if (Object.keys(filters).length !== 0) {
+      return getDatasourceAndQuery({
+        endpoint: getEndpoint(tableId, Operation.DELETE),
+        body: { [colName]: null },
+        filters,
+        meta: {
+          table,
+        },
+      })
+    } else {
+      return []
+    }
+  }
+
+  private async removeOneToManyRelationships(rowId: string, table: Table) {
+    const tableId = table._id!
+    const filters = this.prepareFilters(rowId, {}, table)
+    // safety check, if there are no filters on deletion bad things happen
+    if (Object.keys(filters).length !== 0) {
+      return getDatasourceAndQuery({
+        endpoint: getEndpoint(tableId, Operation.UPDATE),
+        filters,
+        meta: {
+          table,
+        },
+      })
+    } else {
+      return []
+    }
   }
 
   getTable(tableId: string | undefined): Table | undefined {
     if (!tableId) {
-      throw "Table ID is unknown, cannot find table"
+      throw new Error("Table ID is unknown, cannot find table")
     }
     const { tableName } = breakExternalTableId(tableId)
-    if (tableName) {
+    return this.tables[tableName]
-      return this.tables[tableName]
+  }
+
+  // seeds the object with table and datasource information
+  async retrieveMetadata(
+    datasourceId: string
+  ): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
+    if (!this.datasource) {
+      this.datasource = await sdk.datasources.get(datasourceId)
+      if (!this.datasource || !this.datasource.entities) {
+        throw "No tables found, fetch tables before query."
+      }
+      this.tables = this.datasource.entities
+      this.tableList = Object.values(this.tables)
     }
+    return { tables: this.tables, datasource: this.datasource }
   }
 
   async getRow(table: Table, rowId: string): Promise<Row> {
     const response = await getDatasourceAndQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
-      filters: buildFilters(rowId, {}, table),
+      filters: this.prepareFilters(rowId, {}, table),
       meta: {
         table,
       },
@@ -296,9 +323,7 @@ export class ExternalRequest<T extends Operation> {
       if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
         newRow[key] = parseFloat(row[key])
       } else if (field.type === FieldType.LINK) {
-        const { tableName: linkTableName } = breakExternalTableId(
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-          field?.tableId
-        )
         // table has to exist for many to many
         if (!linkTableName || !this.tables[linkTableName]) {
           continue
@@ -379,9 +404,6 @@ export class ExternalRequest<T extends Operation> {
       [key: string]: { rows: Row[]; isMany: boolean; tableId: string }
     } = {}
     const { tableName } = breakExternalTableId(tableId)
-    if (!tableName) {
-      return related
-    }
     const table = this.tables[tableName]
     // @ts-ignore
     const primaryKey = table.primary[0]
@@ -514,7 +536,7 @@ export class ExternalRequest<T extends Operation> {
             endpoint: getEndpoint(tableId, operation),
             // if we're doing many relationships then we're writing, only one response
             body,
-            filters: buildFilters(id, {}, linkTable),
+            filters: this.prepareFilters(id, {}, linkTable),
             meta: {
               table: linkTable,
             },
@@ -538,8 +560,8 @@ export class ExternalRequest<T extends Operation> {
       for (let row of rows) {
         const rowId = generateIdForRow(row, table)
         const promise: Promise<any> = isMany
-          ? removeManyToManyRelationships(rowId, table, colName)
+          ? this.removeManyToManyRelationships(rowId, table, colName)
-          : removeOneToManyRelationships(rowId, table)
+          : this.removeOneToManyRelationships(rowId, table)
         if (promise) {
           promises.push(promise)
         }
@@ -562,12 +584,12 @@ export class ExternalRequest<T extends Operation> {
         rows.map(row => {
           const rowId = generateIdForRow(row, table)
           return isMany
-            ? removeManyToManyRelationships(
+            ? this.removeManyToManyRelationships(
                 rowId,
                 table,
                 relationshipColumn.fieldName
               )
-            : removeOneToManyRelationships(rowId, table)
+            : this.removeOneToManyRelationships(rowId, table)
         })
       )
     }
@@ -575,21 +597,21 @@ export class ExternalRequest<T extends Operation> {
 
   async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    if (!tableId) {
-    if (!tableName) {
+      throw new Error("Unable to run without a table ID")
-      throw "Unable to run without a table name"
     }
-    if (!this.datasource) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
-      this.datasource = await sdk.datasources.get(datasourceId!)
+    let datasource = this.datasource
-      if (!this.datasource || !this.datasource.entities) {
+    if (!datasource) {
-        throw "No tables found, fetch tables before query."
+      const { datasource: ds } = await this.retrieveMetadata(datasourceId)
-      }
+      datasource = ds
-      this.tables = this.datasource.entities
     }
     const table = this.tables[tableName]
-    let isSql = isSQL(this.datasource)
+    let isSql = isSQL(datasource)
     if (!table) {
-      throw `Unable to process query, table "${tableName}" not defined.`
+      throw new Error(
+        `Unable to process query, table "${tableName}" not defined.`
+      )
     }
     // look for specific components of config which may not be considered acceptable
     let { id, row, filters, sort, paginate, rows } = cleanupConfig(
@@ -612,7 +634,7 @@ export class ExternalRequest<T extends Operation> {
           break
       }
     }
-    filters = buildFilters(id, filters || {}, table)
+    filters = this.prepareFilters(id, filters || {}, table)
     const relationships = buildExternalRelationships(table, this.tables)
 
     const incRelationships =
@@ -660,7 +682,11 @@ export class ExternalRequest<T extends Operation> {
       body: row || rows,
       // pass an id filter into extra, purely for mysql/returning
       extra: {
-        idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+        idFilter: this.prepareFilters(
+          id || generateIdForRow(row, table),
+          {},
+          table
+        ),
       },
       meta: {
         table,

packages/server/src/api/controllers/row/external.ts

@@ -136,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
+  const datasource: Datasource = await sdk.datasources.get(datasourceId)
-  if (!tableName) {
-    ctx.throw(400, "Unable to find table.")
-  }
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
@@ -163,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     }
     const links = row[fieldName]
     const linkedTableId = field.tableId
-    const linkedTableName = breakExternalTableId(linkedTableId).tableName!
+    const linkedTableName = breakExternalTableId(linkedTableId).tableName
     const linkedTable = tables[linkedTableName]
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])

packages/server/src/api/controllers/row/utils/basic.ts

@@ -99,7 +99,7 @@ export function basicProcessing({
         row,
         tableName: table._id!,
         fieldName: internalColumn,
-        isLinked: false,
+        isLinked,
       })
     }
   }

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -2,6 +2,8 @@ import {
   DatasourcePlusQueryResponse,
   DSPlusOperation,
   FieldType,
+  isManyToOne,
+  isOneToMany,
   ManyToManyRelationshipFieldMetadata,
   RelationshipFieldMetadata,
   RelationshipsJson,
@@ -93,12 +95,12 @@ export function buildExternalRelationships(
 ): RelationshipsJson[] {
   const relationships = []
   for (let [fieldName, field] of Object.entries(table.schema)) {
-    if (field.type !== FieldType.LINK) {
+    if (field.type !== FieldType.LINK || !field.tableId) {
       continue
     }
     const { tableName: linkTableName } = breakExternalTableId(field.tableId)
     // no table to link to, this is not a valid relationships
-    if (!linkTableName || !tables[linkTableName]) {
+    if (!tables[linkTableName]) {
       continue
     }
     const linkTable = tables[linkTableName]
@@ -110,7 +112,7 @@ export function buildExternalRelationships(
       // need to specify where to put this back into
       column: fieldName,
     }
-    if (isManyToMany(field)) {
+    if (isManyToMany(field) && field.through) {
       const { tableName: throughTableName } = breakExternalTableId(
         field.through
       )
@@ -120,7 +122,7 @@ export function buildExternalRelationships(
       definition.to = field.throughFrom || linkTable.primary[0]
       definition.fromPrimary = table.primary[0]
       definition.toPrimary = linkTable.primary[0]
-    } else {
+    } else if (isManyToOne(field) || isOneToMany(field)) {
       // if no foreign key specified then use the name of the field in other table
       definition.from = field.foreignKey || table.primary[0]
       definition.to = field.fieldName
@@ -180,16 +182,18 @@ export function buildSqlFieldList(
   }
   let fields = extractRealFields(table)
   for (let field of Object.values(table.schema)) {
-    if (field.type !== FieldType.LINK || !opts?.relationships) {
+    if (
+      field.type !== FieldType.LINK ||
+      !opts?.relationships ||
+      !field.tableId
+    ) {
       continue
     }
     const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-    if (linkTableName) {
+    const linkTable = tables[linkTableName]
-      const linkTable = tables[linkTableName]
+    if (linkTable) {
-      if (linkTable) {
+      const linkedFields = extractRealFields(linkTable, fields)
-        const linkedFields = extractRealFields(linkTable, fields)
+      fields = fields.concat(linkedFields)
-        fields = fields.concat(linkedFields)
-      }
     }
   }
   return fields

packages/server/src/api/controllers/table/external.ts

@@ -19,11 +19,14 @@ import { inputProcessing } from "../../../utilities/rowProcessor"
 
 function getDatasourceId(table: Table) {
   if (!table) {
-    throw "No table supplied"
+    throw new Error("No table supplied")
   }
   if (table.sourceId) {
     return table.sourceId
   }
+  if (!table._id) {
+    throw new Error("No table ID supplied")
+  }
   return breakExternalTableId(table._id).datasourceId
 }
 

packages/server/src/api/routes/tests/table.spec.ts

@@ -276,6 +276,34 @@ describe.each([
         })
     })
 
+    isInternal &&
+      it("shouldn't allow duplicate column names", async () => {
+        const saveTableRequest: SaveTableRequest = {
+          ...basicTable(),
+        }
+        saveTableRequest.schema["Type"] = {
+          type: FieldType.STRING,
+          name: "Type",
+        }
+        await config.api.table.save(saveTableRequest, {
+          status: 400,
+          body: {
+            message:
+              'Column(s) "type" are duplicated - check for other columns with these name (case in-sensitive)',
+          },
+        })
+        saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" }
+        saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" }
+
+        await config.api.table.save(saveTableRequest, {
+          status: 400,
+          body: {
+            message:
+              'Column(s) "type, foo" are duplicated - check for other columns with these name (case in-sensitive)',
+          },
+        })
+      })
+
     it("should add a new column for an internal DB table", async () => {
       const saveTableRequest: SaveTableRequest = {
         ...basicTable(),

packages/server/src/sdk/app/rows/index.ts

@@ -3,12 +3,14 @@ import * as rows from "./rows"
 import * as search from "./search"
 import * as utils from "./utils"
 import * as external from "./external"
+import * as filters from "./search/filters"
 import AliasTables from "./sqlAlias"
 
 export default {
   ...attachments,
   ...rows,
   ...search,
+  filters,
   utils,
   external,
   AliasTables,

packages/server/src/sdk/app/rows/search/external.ts

@@ -145,6 +145,10 @@ export async function exportRows(
     delimiter,
     customHeaders,
   } = options
+
+  if (!tableId) {
+    throw new HTTPError("No table ID for search provided.", 400)
+  }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
 
   let requestQuery: SearchFilters = {}
@@ -167,7 +171,7 @@ export async function exportRows(
     requestQuery = query || {}
   }
 
-  const datasource = await sdk.datasources.get(datasourceId!)
+  const datasource = await sdk.datasources.get(datasourceId)
   const table = await sdk.tables.getTable(tableId)
   if (!datasource || !datasource.entities) {
     throw new HTTPError("Datasource has not been configured for plus API.", 400)
@@ -180,10 +184,6 @@ export async function exportRows(
   let rows: Row[] = []
   let headers
 
-  if (!tableName) {
-    throw new HTTPError("Could not find table name.", 400)
-  }
-
   // Filter data to only specified columns if required
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {

packages/server/src/sdk/app/rows/search/filters.ts

@@ -0,0 +1,62 @@
+import {
+  FieldType,
+  RelationshipFieldMetadata,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+import { isPlainObject } from "lodash"
+
+export function getRelationshipColumns(table: Table): {
+  name: string
+  definition: RelationshipFieldMetadata
+}[] {
+  // performing this with a for loop rather than an array filter improves
+  // type guarding, as no casts are required
+  const linkEntries: [string, RelationshipFieldMetadata][] = []
+  for (let entry of Object.entries(table.schema)) {
+    if (entry[1].type === FieldType.LINK) {
+      const linkColumn: RelationshipFieldMetadata = entry[1]
+      linkEntries.push([entry[0], linkColumn])
+    }
+  }
+  return linkEntries.map(entry => ({
+    name: entry[0],
+    definition: entry[1],
+  }))
+}
+
+export function getTableIDList(
+  tables: Table[]
+): { name: string; id: string }[] {
+  return tables
+    .filter(table => table.originalName && table._id)
+    .map(table => ({ id: table._id!, name: table.originalName! }))
+}
+
+export function updateFilterKeys(
+  filters: SearchFilters,
+  updates: { original: string; updated: string }[]
+): SearchFilters {
+  const makeFilterKeyRegex = (str: string) =>
+    new RegExp(`^${str}\\.|:${str}\\.`)
+  for (let filter of Object.values(filters)) {
+    if (!isPlainObject(filter)) {
+      continue
+    }
+    for (let [key, keyFilter] of Object.entries(filter)) {
+      if (keyFilter === "") {
+        delete filter[key]
+      }
+      const possibleKey = updates.find(({ original }) =>
+        key.match(makeFilterKeyRegex(original))
+      )
+      if (possibleKey && possibleKey.original !== possibleKey.updated) {
+        // only replace the first, not replaceAll
+        filter[key.replace(possibleKey.original, possibleKey.updated)] =
+          filter[key]
+        delete filter[key]
+      }
+    }
+  }
+  return filters
+}

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -1,4 +1,5 @@
 import {
+  Datasource,
   DocumentType,
   FieldType,
   Operation,
@@ -12,7 +13,6 @@ import {
   SortType,
   SqlClient,
   Table,
-  Datasource,
 } from "@budibase/types"
 import {
   buildInternalRelationships,
@@ -30,6 +30,11 @@ import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
 import { processRowCountResponse } from "../utils"
+import {
+  updateFilterKeys,
+  getRelationshipColumns,
+  getTableIDList,
+} from "./filters"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 
@@ -60,34 +65,31 @@ function buildInternalFieldList(
   return fieldList
 }
 
-function tableNameInFieldRegex(tableName: string) {
+function cleanupFilters(
-  return new RegExp(`^${tableName}.|:${tableName}.`, "g")
+  filters: SearchFilters,
-}
+  table: Table,
-
+  allTables: Table[]
-function cleanupFilters(filters: SearchFilters, tables: Table[]) {
+) {
-  for (let filter of Object.values(filters)) {
+  // get a list of all relationship columns in the table for updating
-    if (typeof filter !== "object") {
+  const relationshipColumns = getRelationshipColumns(table)
-      continue
+  // get table names to ID map for relationships
-    }
+  const tableNameToID = getTableIDList(allTables)
-    for (let [key, keyFilter] of Object.entries(filter)) {
+  // all should be applied at once
-      if (keyFilter === "") {
+  filters = updateFilterKeys(
-        delete filter[key]
+    filters,
-      }
+    relationshipColumns
-
+      .map(({ name, definition }) => ({
-      // relationship, switch to table ID
+        original: name,
-      const tableRelated = tables.find(
+        updated: definition.tableId,
-        table =>
+      }))
-          table.originalName &&
+      .concat(
-          key.match(tableNameInFieldRegex(table.originalName))
+        tableNameToID.map(({ name, id }) => ({
+          original: name,
+          updated: id,
+        }))
       )
-      if (tableRelated && tableRelated.originalName) {
+  )
-        // only replace the first, not replaceAll
+
-        filter[key.replace(tableRelated.originalName, tableRelated._id!)] =
-          filter[key]
-        delete filter[key]
-      }
-    }
-  }
   return filters
 }
 
@@ -176,7 +178,7 @@ export async function search(
       operation: Operation.READ,
     },
     filters: {
-      ...cleanupFilters(query, allTables),
+      ...cleanupFilters(query, table, allTables),
       documentType: DocumentType.ROW,
     },
     table,

packages/server/src/sdk/app/tables/getters.ts

@@ -90,10 +90,10 @@ export async function getExternalTable(
 export async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
   let output: Table
-  if (isExternalTableID(tableId)) {
+  if (tableId && isExternalTableID(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await datasources.get(datasourceId!)
+    const datasource = await datasources.get(datasourceId)
-    const table = await getExternalTable(datasourceId!, tableName!)
+    const table = await getExternalTable(datasourceId, tableName)
     output = { ...table, sql: isSQL(datasource) }
   } else {
     output = await db.get<Table>(tableId)

packages/server/src/sdk/app/tables/internal/index.ts

@@ -17,6 +17,7 @@ import { cloneDeep } from "lodash/fp"
 import isEqual from "lodash/isEqual"
 import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
 import { context } from "@budibase/backend-core"
+import { findDuplicateInternalColumns } from "@budibase/shared-core"
 import { getTable } from "../getters"
 import { checkAutoColumns } from "./utils"
 import * as viewsSdk from "../../views"
@@ -44,6 +45,17 @@ export async function save(
   if (hasTypeChanged(table, oldTable)) {
     throw new Error("A column type has changed.")
   }
+
+  // check for case sensitivity - we don't want to allow duplicated columns
+  const duplicateColumn = findDuplicateInternalColumns(table)
+  if (duplicateColumn.length) {
+    throw new Error(
+      `Column(s) "${duplicateColumn.join(
+        ", "
+      )}" are duplicated - check for other columns with these name (case in-sensitive)`
+    )
+  }
+
   // check that subtypes have been maintained
   table = checkAutoColumns(table, oldTable)
 

packages/server/src/sdk/app/views/external.ts

@@ -10,9 +10,9 @@ export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
 
-  const table = ds.entities![tableName!]
+  const table = ds.entities![tableName]
   const views = Object.values(table.views!).filter(isV2)
   const found = views.find(v => v.id === viewId)
   if (!found) {
@@ -25,9 +25,9 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
 
-  const table = ds.entities![tableName!]
+  const table = ds.entities![tableName]
   const views = Object.values(table.views!).filter(isV2)
   const found = views.find(v => v.id === viewId)
   if (!found) {
@@ -49,9 +49,9 @@ export async function create(
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
-  ds.entities![tableName!].views ??= {}
+  ds.entities![tableName].views ??= {}
-  ds.entities![tableName!].views![view.name] = view
+  ds.entities![tableName].views![view.name] = view
   await db.put(ds)
   return view
 }
@@ -60,9 +60,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
-  ds.entities![tableName!].views ??= {}
+  ds.entities![tableName].views ??= {}
-  const views = ds.entities![tableName!].views!
+  const views = ds.entities![tableName].views!
 
   const existingView = Object.values(views).find(
     v => isV2(v) && v.id === view.id
@@ -87,9 +87,9 @@ export async function remove(viewId: string): Promise<ViewV2> {
   }
 
   const { datasourceId, tableName } = breakExternalTableId(view.tableId)
-  const ds = await sdk.datasources.get(datasourceId!)
+  const ds = await sdk.datasources.get(datasourceId)
 
-  delete ds.entities![tableName!].views![view?.name]
+  delete ds.entities![tableName].views![view?.name]
   await db.put(ds)
   return view
 }

packages/shared-core/src/constants/index.ts

@@ -1,5 +1,6 @@
 export * from "./api"
 export * from "./fields"
+export * from "./rows"
 
 export const OperatorOptions = {
   Equals: {
@@ -179,3 +180,5 @@ export enum BpmStatusValue {
   VERIFYING_EMAIL = "verifying_email",
   COMPLETED = "completed",
 }
+
+export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"

packages/shared-core/src/constants/rows.ts

@@ -0,0 +1,14 @@
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+] as const
+
+export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
+
+export function isInternalColumnName(name: string): boolean {
+  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
+}

packages/shared-core/src/table.ts

@@ -1,4 +1,5 @@
-import { FieldType } from "@budibase/types"
+import { FieldType, Table } from "@budibase/types"
+import { CONSTANT_INTERNAL_ROW_COLS } from "./constants"
 
 const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
@@ -51,3 +52,22 @@ export function canBeDisplayColumn(type: FieldType): boolean {
 export function canBeSortColumn(type: FieldType): boolean {
   return !!allowSortColumnByType[type]
 }
+
+export function findDuplicateInternalColumns(table: Table): string[] {
+  // get the column names
+  const columnNames = Object.keys(table.schema)
+    .concat(CONSTANT_INTERNAL_ROW_COLS)
+    .map(colName => colName.toLowerCase())
+  // there are duplicates
+  const set = new Set(columnNames)
+  let duplicates: string[] = []
+  if (set.size !== columnNames.length) {
+    for (let key of set.keys()) {
+      const count = columnNames.filter(name => name === key).length
+      if (count > 1) {
+        duplicates.push(key)
+      }
+    }
+  }
+  return duplicates
+}

yarn.lock

@@ -10296,7 +10296,7 @@ engine.io-parser@~5.0.3:
   resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.6.tgz#7811244af173e157295dec9b2718dfe42a64ef45"
   integrity sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw==
 
-engine.io@~6.4.1:
+engine.io@~6.4.2:
   version "6.4.2"
   resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f"
   integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg==
@@ -20160,17 +20160,25 @@ socket.io-parser@~4.2.1:
     "@socket.io/component-emitter" "~3.1.0"
     debug "~4.3.1"
 
-socket.io@4.6.1:
+socket.io-parser@~4.2.4:
-  version "4.6.1"
+  version "4.2.4"
-  resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.1.tgz#62ec117e5fce0692fa50498da9347cfb52c3bc70"
+  resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83"
-  integrity sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA==
+  integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==
+  dependencies:
+    "@socket.io/component-emitter" "~3.1.0"
+    debug "~4.3.1"
+
+socket.io@4.6.2:
+  version "4.6.2"
+  resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786"
+  integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ==
   dependencies:
     accepts "~1.3.4"
     base64id "~2.0.0"
     debug "~4.3.2"
-    engine.io "~6.4.1"
+    engine.io "~6.4.2"
     socket.io-adapter "~2.5.2"
-    socket.io-parser "~4.2.1"
+    socket.io-parser "~4.2.4"
 
 socks-proxy-agent@^7.0.0:
   version "7.0.0"
@@ -21102,18 +21110,6 @@ tar@6.1.11:
     mkdirp "^1.0.3"
     yallist "^4.0.0"
 
-tar@6.1.15:
-  version "6.1.15"
-  resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69"
-  integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==
-  dependencies:
-    chownr "^2.0.0"
-    fs-minipass "^2.0.0"
-    minipass "^5.0.0"
-    minizlib "^2.1.1"
-    mkdirp "^1.0.3"
-    yallist "^4.0.0"
-
 tar@6.2.1, tar@^6.1.11, tar@^6.1.2:
   version "6.2.1"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a"