Merge remote-tracking branch 'origin/master' into feature/automation-row-ux-update

.eslintrc.json

@@ -92,7 +92,8 @@
         // differs to external, but the API is broadly the same
         "jest/no-conditional-expect": "off",
         // have to turn this off to allow function overloading in typescript
-        "no-dupe-class-members": "off"
+        "no-dupe-class-members": "off",
+        "no-redeclare": "off"
       }
     },
     {

.github/workflows/budibase_ci.yml

@@ -73,9 +73,9 @@ jobs:
       - name: Check types
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
-            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server
           else
-            yarn check:types
+            yarn check:types --ignore @budibase/account-portal-server
           fi
 
   helm-lint:

examples/nextjs-api-sales/yarn.lock

@@ -333,11 +333,11 @@ brace-expansion@^1.1.7:
     concat-map "0.0.1"
 
 braces@^3.0.1, braces@~3.0.2:
-  version "3.0.2"
+  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
-  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+  integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
   dependencies:
-    fill-range "^7.0.1"
+    fill-range "^7.1.1"
 
 bulma@^0.9.3:
   version "0.9.3"
@@ -781,10 +781,10 @@ file-entry-cache@^6.0.1:
   dependencies:
     flat-cache "^3.0.4"
 
-fill-range@^7.0.1:
+fill-range@^7.1.1:
-  version "7.0.1"
+  version "7.1.1"
-  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
-  integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+  integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
   dependencies:
     to-regex-range "^5.0.1"
 
@@ -1709,10 +1709,10 @@ type-fest@^0.20.2:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
   integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
 
-typescript@4.6.2:
+typescript@5.2.2:
-  version "4.6.2"
+  version "5.2.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
-  integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
+  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
 
 unbox-primitive@^1.0.1:
   version "1.0.1"

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.28.6",
+  "version": "2.29.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -40,7 +40,7 @@
     "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
     "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run --concurrency 2 check:types",
+    "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 247f56d455abbd64da17d865275ed978f577549f
+Subproject commit ff16525b73c5751d344f5c161a682609c0a993f2

packages/backend-core/src/constants/db.ts

@@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
 export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
 export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
+export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core"

packages/backend-core/src/db/constants.ts

@@ -1,14 +1,5 @@
-export const CONSTANT_INTERNAL_ROW_COLS = [
+export {
-  "_id",
+  CONSTANT_INTERNAL_ROW_COLS,
-  "_rev",
+  CONSTANT_EXTERNAL_ROW_COLS,
-  "type",
+  isInternalColumnName,
-  "createdAt",
+} from "@budibase/shared-core"
-  "updatedAt",
-  "tableId",
-] as const
-
-export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
-
-export function isInternalColumnName(name: string): boolean {
-  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
-}

packages/backend-core/src/sql/sql.ts

@@ -1,10 +1,10 @@
 import { Knex, knex } from "knex"
 import * as dbCore from "../db"
 import {
-  isIsoDateString,
-  isValidFilter,
   getNativeSql,
   isExternalTable,
+  isIsoDateString,
+  isValidFilter,
 } from "./utils"
 import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
@@ -12,21 +12,21 @@ import {
   BBReferenceFieldMetadata,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   JsonFieldMetadata,
+  JsonTypes,
   Operation,
+  prefixed,
   QueryJson,
-  SqlQuery,
+  QueryOptions,
   RelationshipsJson,
   SearchFilters,
-  SortDirection,
+  SortOrder,
+  SqlClient,
+  SqlQuery,
   SqlQueryBinding,
   Table,
   TableSourceType,
-  INTERNAL_TABLE_SOURCE_ID,
-  SqlClient,
-  QueryOptions,
-  JsonTypes,
-  prefixed,
 } from "@budibase/types"
 import environment from "../environment"
 import { helpers } from "@budibase/shared-core"
@@ -114,7 +114,7 @@ function generateSelectStatement(
 ): (string | Knex.Raw)[] | "*" {
   const { resource, meta } = json
 
-  if (!resource) {
+  if (!resource || !resource.fields || resource.fields.length === 0) {
     return "*"
   }
 
@@ -410,28 +410,50 @@ class InternalBuilder {
     return query
   }
 
-  addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
+  addDistinctCount(
-    let { sort, paginate } = json
+    query: Knex.QueryBuilder,
+    json: QueryJson
+  ): Knex.QueryBuilder {
     const table = json.meta.table
+    const primary = table.primary
+    const aliases = json.tableAliases
+    const aliased =
+      table.name && aliases?.[table.name] ? aliases[table.name] : table.name
+    if (!primary) {
+      throw new Error("SQL counting requires primary key to be supplied")
+    }
+    return query.countDistinct(`${aliased}.${primary[0]} as total`)
+  }
+
+  addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
+    let { sort } = json
+    const table = json.meta.table
+    const primaryKey = table.primary
     const tableName = getTableName(table)
     const aliases = json.tableAliases
     const aliased =
       tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
+    if (!Array.isArray(primaryKey)) {
+      throw new Error("Sorting requires primary key to be specified for table")
+    }
     if (sort && Object.keys(sort || {}).length > 0) {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
-          value.direction === SortDirection.ASCENDING ? "asc" : "desc"
+          value.direction === SortOrder.ASCENDING ? "asc" : "desc"
         let nulls
         if (this.client === SqlClient.POSTGRES) {
           // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
-          nulls = value.direction === SortDirection.ASCENDING ? "first" : "last"
+          nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
         query = query.orderBy(`${aliased}.${key}`, direction, nulls)
       }
-    } else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
+    }
-      // @ts-ignore
+
-      query = query.orderBy(`${aliased}.${table?.primary[0]}`)
+    // add sorting by the primary key if the result isn't already sorted by it,
+    // to make sure result is deterministic
+    if (!sort || sort[primaryKey[0]] === undefined) {
+      query = query.orderBy(`${aliased}.${primaryKey[0]}`)
     }
     return query
   }
@@ -522,7 +544,7 @@ class InternalBuilder {
           })
       }
     }
-    return query.limit(BASE_LIMIT)
+    return query
   }
 
   knexWithAlias(
@@ -533,13 +555,12 @@ class InternalBuilder {
     const tableName = endpoint.entityId
     const tableAlias = aliases?.[tableName]
 
-    const query = knex(
+    return knex(
       this.tableNameWithSchema(tableName, {
         alias: tableAlias,
         schema: endpoint.schema,
       })
     )
-    return query
   }
 
   create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
@@ -571,52 +592,94 @@ class InternalBuilder {
     return query.insert(parsedBody)
   }
 
-  read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder {
+  bulkUpsert(knex: Knex, json: QueryJson): Knex.QueryBuilder {
-    let { endpoint, resource, filters, paginate, relationships, tableAliases } =
+    const { endpoint, body } = json
-      json
+    let query = this.knexWithAlias(knex, endpoint)
+    if (!Array.isArray(body)) {
+      return query
+    }
+    const parsedBody = body.map(row => parseBody(row))
+    if (
+      this.client === SqlClient.POSTGRES ||
+      this.client === SqlClient.SQL_LITE ||
+      this.client === SqlClient.MY_SQL
+    ) {
+      const primary = json.meta.table.primary
+      if (!primary) {
+        throw new Error("Primary key is required for upsert")
+      }
+      const ret = query.insert(parsedBody).onConflict(primary).merge()
+      return ret
+    } else if (this.client === SqlClient.MS_SQL) {
+      // No upsert or onConflict support in MSSQL yet, see:
+      //   https://github.com/knex/knex/pull/6050
+      return query.insert(parsedBody)
+    }
+    return query.upsert(parsedBody)
+  }
+
+  read(
+    knex: Knex,
+    json: QueryJson,
+    opts: {
+      limits?: { base: number; query: number }
+    } = {}
+  ): Knex.QueryBuilder {
+    let { endpoint, filters, paginate, relationships, tableAliases } = json
+    const { limits } = opts
+    const counting = endpoint.operation === Operation.COUNT
 
     const tableName = endpoint.entityId
-    // select all if not specified
+    // start building the query
-    if (!resource) {
+    let query = this.knexWithAlias(knex, endpoint, tableAliases)
-      resource = { fields: [] }
-    }
-    let selectStatement: string | (string | Knex.Raw)[] = "*"
-    // handle select
-    if (resource.fields && resource.fields.length > 0) {
-      // select the resources as the format "table.columnName" - this is what is provided
-      // by the resource builder further up
-      selectStatement = generateSelectStatement(json, knex)
-    }
-    let foundLimit = limit || BASE_LIMIT
     // handle pagination
     let foundOffset: number | null = null
+    let foundLimit = limits?.query || limits?.base
     if (paginate && paginate.page && paginate.limit) {
       // @ts-ignore
       const page = paginate.page <= 1 ? 0 : paginate.page - 1
       const offset = page * paginate.limit
       foundLimit = paginate.limit
       foundOffset = offset
+    } else if (paginate && paginate.offset && paginate.limit) {
+      foundLimit = paginate.limit
+      foundOffset = paginate.offset
     } else if (paginate && paginate.limit) {
       foundLimit = paginate.limit
     }
-    // start building the query
+    // counting should not sort, limit or offset
-    let query = this.knexWithAlias(knex, endpoint, tableAliases)
+    if (!counting) {
-    query = query.limit(foundLimit)
+      // add the found limit if supplied
-    if (foundOffset) {
+      if (foundLimit != null) {
-      query = query.offset(foundOffset)
+        query = query.limit(foundLimit)
+      }
+      // add overall pagination
+      if (foundOffset != null) {
+        query = query.offset(foundOffset)
+      }
+      // add sorting to pre-query
+      // no point in sorting when counting
+      query = this.addSorting(query, json)
     }
+    // add filters to the query (where)
     query = this.addFilters(query, filters, json.meta.table, {
       aliases: tableAliases,
     })
 
-    // add sorting to pre-query
-    query = this.addSorting(query, json)
     const alias = tableAliases?.[tableName] || tableName
-    let preQuery = knex({
+    let preQuery: Knex.QueryBuilder = knex({
-      [alias]: query,
+      // the typescript definition for the knex constructor doesn't support this
-    } as any).select(selectStatement) as any
+      // syntax, but it is the only way to alias a pre-query result as part of
+      // a query - there is an alias dictionary type, but it assumes it can only
+      // be a table name, not a pre-query
+      [alias]: query as any,
+    })
+    // if counting, use distinct count, else select
+    preQuery = !counting
+      ? preQuery.select(generateSelectStatement(json, knex))
+      : this.addDistinctCount(preQuery, json)
     // have to add after as well (this breaks MS-SQL)
-    if (this.client !== SqlClient.MS_SQL) {
+    if (this.client !== SqlClient.MS_SQL && !counting) {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
@@ -627,6 +690,13 @@ class InternalBuilder {
       endpoint.schema,
       tableAliases
     )
+
+    // add a base limit over the whole query
+    // if counting we can't set this limit
+    if (limits?.base) {
+      query = query.limit(limits.base)
+    }
+
     return this.addFilters(query, filters, json.meta.table, {
       relationship: true,
       aliases: tableAliases,
@@ -671,6 +741,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     this.limit = limit
   }
 
+  private convertToNative(query: Knex.QueryBuilder, opts: QueryOptions = {}) {
+    const sqlClient = this.getSqlClient()
+    if (opts?.disableBindings) {
+      return { sql: query.toString() }
+    } else {
+      let native = getNativeSql(query)
+      if (sqlClient === SqlClient.SQL_LITE) {
+        native = convertBooleans(native)
+      }
+      return native
+    }
+  }
+
   /**
    * @param json The JSON query DSL which is to be converted to SQL.
    * @param opts extra options which are to be passed into the query builder, e.g. disableReturning
@@ -694,7 +777,16 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         query = builder.create(client, json, opts)
         break
       case Operation.READ:
-        query = builder.read(client, json, this.limit)
+        query = builder.read(client, json, {
+          limits: {
+            query: this.limit,
+            base: BASE_LIMIT,
+          },
+        })
+        break
+      case Operation.COUNT:
+        // read without any limits to count
+        query = builder.read(client, json)
         break
       case Operation.UPDATE:
         query = builder.update(client, json, opts)
@@ -705,6 +797,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       case Operation.BULK_CREATE:
         query = builder.bulkCreate(client, json)
         break
+      case Operation.BULK_UPSERT:
+        query = builder.bulkUpsert(client, json)
+        break
       case Operation.CREATE_TABLE:
       case Operation.UPDATE_TABLE:
       case Operation.DELETE_TABLE:
@@ -713,15 +808,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         throw `Operation type is not supported by SQL query builder`
     }
 
-    if (opts?.disableBindings) {
+    return this.convertToNative(query, opts)
-      return { sql: query.toString() }
-    } else {
-      let native = getNativeSql(query)
-      if (sqlClient === SqlClient.SQL_LITE) {
-        native = convertBooleans(native)
-      }
-      return native
-    }
   }
 
   async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
@@ -797,6 +884,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         await this.getReturningRow(queryFn, this.checkLookupKeys(id, json))
       )
     }
+    if (operation === Operation.COUNT) {
+      return results
+    }
     if (operation !== Operation.READ) {
       return row
     }

packages/backend-core/src/sql/sqlTable.ts

@@ -109,8 +109,10 @@ function generateSchema(
           const { tableName } = breakExternalTableId(column.tableId)
           // @ts-ignore
           const relatedTable = tables[tableName]
-          if (!relatedTable) {
+          if (!relatedTable || !relatedTable.primary) {
-            throw new Error("Referenced table doesn't exist")
+            throw new Error(
+              "Referenced table doesn't exist or has no primary keys"
+            )
           }
           const relatedPrimary = relatedTable.primary[0]
           const externalType = relatedTable.schema[relatedPrimary].externalType

packages/backend-core/src/sql/utils.ts

@@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
-export function breakExternalTableId(tableId: string | undefined) {
+export function breakExternalTableId(tableId: string) {
-  if (!tableId) {
-    return {}
-  }
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let datasourceId = parts.shift()
   // if they need joined
@@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) {
   if (tableName.includes(ENCODED_SPACE)) {
     tableName = decodeURIComponent(tableName)
   }
+  if (!datasourceId || !tableName) {
+    throw new Error("Unable to get datasource/table name from table ID")
+  }
   return { datasourceId, tableName }
 }
 

packages/backend-core/src/tenancy/db.ts

@@ -9,8 +9,13 @@ export function getTenantDB(tenantId: string) {
 export async function saveTenantInfo(tenantInfo: TenantInfo) {
   const db = getTenantDB(tenantInfo.tenantId)
   // save the tenant info to db
-  return await db.put({
+  return db.put({
     _id: "tenant_info",
     ...tenantInfo,
   })
 }
+
+export async function getTenantInfo(tenantId: string): Promise<TenantInfo> {
+  const db = getTenantDB(tenantId)
+  return db.get("tenant_info")
+}

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -24,7 +24,6 @@ export const account = (partial: Partial<Account> = {}): Account => {
     createdAt: Date.now(),
     verified: true,
     verificationSent: true,
-    tier: "FREE", // DEPRECATED
     authType: AuthType.PASSWORD,
     name: generator.name(),
     size: "10+",

packages/bbui/src/Modal/Modal.svelte

@@ -162,6 +162,7 @@
     max-height: 100%;
   }
   .modal-inner-wrapper {
+    padding: 40px;
     flex: 1 1 auto;
     display: flex;
     flex-direction: row;
@@ -176,7 +177,6 @@
     border: 2px solid var(--spectrum-global-color-gray-200);
     overflow: visible;
     max-height: none;
-    margin: 40px 0;
     transform: none;
     --spectrum-dialog-confirm-border-radius: var(
       --spectrum-global-dimension-size-100

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -120,6 +120,8 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
+  let testDataRowVisibility = {}
+
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -417,7 +419,8 @@
         (automation.trigger?.event === AutomationEventType.ROW_UPDATE ||
           automation.trigger?.event === AutomationEventType.ROW_SAVE)
       ) {
-        if (name !== "id" && name !== "revision") return `trigger.row.${name}`
+        let noRowKeywordBindings = ["id", "revision", "oldRow"]
+        if (!noRowKeywordBindings.includes(name)) return `trigger.row.${name}`
       }
       /* End special cases for generating custom schemas based on triggers */
 
@@ -601,7 +604,7 @@
   function getFieldLabel(key, value) {
     const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
     const label = `${
-      value.title || (key === "row" ? "Table" : key)
+      value.title || (key === "row" ? "Row" : key)
     } ${requiredSuffix}`
     return Helpers.capitalise(label)
   }

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -17,6 +17,8 @@
     SWITCHABLE_TYPES,
     ValidColumnNameRegex,
     helpers,
+    CONSTANT_INTERNAL_ROW_COLS,
+    CONSTANT_EXTERNAL_ROW_COLS,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -52,7 +54,6 @@
   const DATE_TYPE = FieldType.DATETIME
 
   const dispatch = createEventDispatcher()
-  const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
   const { dispatch: gridDispatch, rows } = getContext("grid")
 
   export let field
@@ -487,20 +488,27 @@
       })
     }
     const newError = {}
+    const prohibited = externalTable
+      ? CONSTANT_EXTERNAL_ROW_COLS
+      : CONSTANT_INTERNAL_ROW_COLS
     if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
-    } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
+    } else if (
-      newError.name = `${PROHIBITED_COLUMN_NAMES.join(
+      prohibited.some(
+        name => fieldInfo?.name?.toLowerCase() === name.toLowerCase()
+      )
+    ) {
+      newError.name = `${prohibited.join(
         ", "
-      )} are not allowed as column names`
+      )} are not allowed as column names - case insensitive.`
     } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
 
     if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) {
-      newError.subtype = `Auto Column requires a type`
+      newError.subtype = `Auto Column requires a type.`
     }
 
     if (fieldInfo.fieldName && fieldInfo.tableId) {

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,9 +1,14 @@
 <script>
-  import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
+  import {
+    FieldType,
+    BBReferenceFieldSubType,
+    SourceName,
+  } from "@budibase/types"
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
   import { DB_TYPE_INTERNAL } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
+  import { tables, datasources } from "stores/builder"
 
   let error = null
   let fileName = null
@@ -80,6 +85,9 @@
     schema = fetchSchema(tableId)
   }
 
+  $: table = $tables.list.find(table => table._id === tableId)
+  $: datasource = $datasources.list.find(ds => ds._id === table?.sourceId)
+
   async function fetchSchema(tableId) {
     try {
       const definition = await API.fetchTableDefinition(tableId)
@@ -185,20 +193,25 @@
       </div>
     {/each}
   </div>
-  {#if tableType === DB_TYPE_INTERNAL}
+  <br />
-    <br />
+  <!-- SQL Server doesn't yet support overwriting rows by existing keys -->
+  {#if datasource?.source !== SourceName.SQL_SERVER}
     <Toggle
       bind:value={updateExistingRows}
       on:change={() => (identifierFields = [])}
       thin
       text="Update existing rows"
     />
-    {#if updateExistingRows}
+  {/if}
+  {#if updateExistingRows}
+    {#if tableType === DB_TYPE_INTERNAL}
       <Multiselect
         label="Identifier field(s)"
         options={Object.keys(validation)}
         bind:value={identifierFields}
       />
+    {:else}
+      <p>Rows will be updated based on the table's primary key.</p>
     {/if}
   {/if}
   {#if invalidColumns.length > 0}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseModal.svelte

@@ -0,0 +1,8 @@
+<div class="root">This action doesn't require any settings.</div>
+
+<style>
+  .root {
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte

@@ -53,6 +53,12 @@
         placeholder="Are you sure you want to delete?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 </div>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -83,6 +83,12 @@
         placeholder="Are you sure you want to duplicate this row?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte

@@ -74,6 +74,18 @@
           placeholder="Are you sure you want to execute this query?"
           bind:value={parameters.confirmText}
         />
+
+        <Input
+          label="Confirm Text"
+          placeholder="Confirm"
+          bind:value={parameters.confirmButtonText}
+        />
+
+        <Input
+          label="Cancel Text"
+          placeholder="Cancel"
+          bind:value={parameters.cancelButtonText}
+        />
       {/if}
 
       {#if query?.parameters?.length > 0}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/OpenModal.svelte

@@ -0,0 +1,36 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { selectedScreen } from "stores/builder"
+  import { findAllMatchingComponents } from "helpers/components"
+
+  export let parameters
+
+  $: modalOptions = getModalOptions($selectedScreen)
+
+  const getModalOptions = screen => {
+    const modalComponents = findAllMatchingComponents(screen.props, component =>
+      component._component.endsWith("/modal")
+    )
+    return modalComponents.map(modal => ({
+      label: modal._instanceName,
+      value: modal._id,
+    }))
+  }
+</script>
+
+<div class="root">
+  <Label small>Modal</Label>
+  <Select bind:value={parameters.id} options={modalOptions} />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -80,6 +80,12 @@
         placeholder="Are you sure you want to save this row?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js

@@ -21,5 +21,7 @@ export { default as ShowNotification } from "./ShowNotification.svelte"
 export { default as PromptUser } from "./PromptUser.svelte"
 export { default as OpenSidePanel } from "./OpenSidePanel.svelte"
 export { default as CloseSidePanel } from "./CloseSidePanel.svelte"
+export { default as OpenModal } from "./OpenModal.svelte"
+export { default as CloseModal } from "./CloseModal.svelte"
 export { default as ClearRowSelection } from "./ClearRowSelection.svelte"
 export { default as DownloadFile } from "./DownloadFile.svelte"

packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json

@@ -157,6 +157,18 @@
       "component": "CloseSidePanel",
       "dependsOnFeature": "sidePanel"
     },
+    {
+      "name": "Open Modal",
+      "type": "application",
+      "component": "OpenModal",
+      "dependsOnFeature": "modal"
+    },
+    {
+      "name": "Close Modal",
+      "type": "application",
+      "component": "CloseModal",
+      "dependsOnFeature": "modal"
+    },
     {
       "name": "Clear Row Selection",
       "type": "data",

packages/builder/src/components/design/settings/controls/Explanation/Explanation.svelte

@@ -18,14 +18,11 @@
   import subjects from "./subjects"
   import { appStore } from "stores/builder"
 
-  export let explanation
-  export let columnIcon
-  export let columnType
-  export let columnName
-
   export let tableHref = () => {}
-
   export let schema
+  export let name
+  export let explanation
+  export let componentName
 
   $: explanationWithPresets = getExplanationWithPresets(
     explanation,
@@ -54,14 +51,8 @@
 </script>
 
 <div bind:this={root} class="tooltipContents">
-  <Column
+  <Column {name} {schema} {tableHref} {setExplanationSubject} />
-    {columnName}
+  <Support {componentName} {support} {setExplanationSubject} />
-    {columnIcon}
-    {columnType}
-    {tableHref}
-    {setExplanationSubject}
-  />
-  <Support {support} {setExplanationSubject} />
   {#if messages.includes(messageConstants.stringAsNumber)}
     <StringAsNumber {setExplanationSubject} />
   {/if}
@@ -84,7 +75,7 @@
 
 {#if detailsModalSubject !== subjects.none}
   <DetailsModal
-    {columnName}
+    columnName={name}
     anchor={root}
     {schema}
     subject={detailsModalSubject}

packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte

@@ -1,69 +1,124 @@
 <script>
-  import {
+  import { Line, InfoWord, DocumentationLink, Text } from "../typography"
-    Line,
+  import { FieldType } from "@budibase/types"
-    InfoWord,
+  import { FIELDS } from "constants/backend"
-    DocumentationLink,
-    Text,
-    Period,
-  } from "../typography"
   import subjects from "../subjects"
 
-  export let columnName
+  export let schema
-  export let columnIcon
+  export let name
-  export let columnType
   export let tableHref
   export let setExplanationSubject
 
+  const getTypeName = schema => {
+    const fieldDefinition = Object.values(FIELDS).find(
+      f => f.type === schema?.type
+    )
+
+    if (schema?.type === "jsonarray") {
+      return "JSON Array"
+    }
+    if (schema?.type === "options") {
+      return "Options"
+    }
+
+    return fieldDefinition?.name || schema?.type || "Unknown"
+  }
+
+  const getTypeIcon = schema => {
+    const fieldDefinition = Object.values(FIELDS).find(
+      f => f.type === schema?.type
+    )
+
+    if (schema?.type === "jsonarray") {
+      return "BracketsSquare"
+    }
+
+    return fieldDefinition?.icon || "Circle"
+  }
+
   const getDocLink = columnType => {
-    if (columnType === "Number") {
+    if (columnType === FieldType.NUMBER) {
       return "https://docs.budibase.com/docs/number"
     }
-    if (columnType === "Text") {
+    if (columnType === FieldType.STRING) {
       return "https://docs.budibase.com/docs/text"
     }
-    if (columnType === "Attachment") {
+    if (columnType === FieldType.LONGFORM) {
+      return "https://docs.budibase.com/docs/text"
+    }
+    if (columnType === FieldType.ATTACHMENT_SINGLE) {
       return "https://docs.budibase.com/docs/attachments"
     }
-    if (columnType === "Multi-select") {
+    if (columnType === FieldType.ATTACHMENTS) {
+      // No distinct multi attachment docs, link to attachment instead
+      return "https://docs.budibase.com/docs/attachments"
+    }
+    if (columnType === FieldType.ARRAY) {
       return "https://docs.budibase.com/docs/multi-select"
     }
-    if (columnType === "JSON") {
+    if (columnType === FieldType.JSON) {
       return "https://docs.budibase.com/docs/json"
     }
-    if (columnType === "Date/Time") {
+    if (columnType === "jsonarray") {
+      return "https://docs.budibase.com/docs/json"
+    }
+    if (columnType === FieldType.DATETIME) {
       return "https://docs.budibase.com/docs/datetime"
     }
-    if (columnType === "User") {
+    if (columnType === FieldType.BB_REFERENCE_SINGLE) {
-      return "https://docs.budibase.com/docs/user"
+      return "https://docs.budibase.com/docs/users-1"
     }
-    if (columnType === "QR") {
+    if (columnType === FieldType.BB_REFERENCE) {
+      return "https://docs.budibase.com/docs/users-1"
+    }
+    if (columnType === FieldType.BARCODEQR) {
       return "https://docs.budibase.com/docs/barcodeqr"
     }
-    if (columnType === "Relationship") {
+    if (columnType === FieldType.LINK) {
       return "https://docs.budibase.com/docs/relationships"
     }
-    if (columnType === "Formula") {
+    if (columnType === FieldType.FORMULA) {
       return "https://docs.budibase.com/docs/formula"
     }
-    if (columnType === "Options") {
+    if (columnType === FieldType.OPTIONS) {
       return "https://docs.budibase.com/docs/options"
     }
-    if (columnType === "BigInt") {
+    if (columnType === FieldType.BOOLEAN) {
-      // No BigInt docs
-      return null
-    }
-    if (columnType === "Boolean") {
       return "https://docs.budibase.com/docs/boolean-truefalse"
     }
-    if (columnType === "Signature") {
+    if (columnType === FieldType.SIGNATURE_SINGLE) {
       // No Signature docs
       return null
     }
+    if (columnType === FieldType.BIGINT) {
+      // No BigInt docs
+      return null
+    }
 
     return null
   }
 
-  $: docLink = getDocLink(columnType)
+  // NOTE The correct indefinite article is based on the pronounciation of the word it precedes, not the spelling. So simply checking if the word begins with a vowel is not sufficient.
+
+  // e.g., `an honor`, `a user`
+  const getIndefiniteArticle = schema => {
+    const anTypes = [
+      FieldType.OPTIONS,
+      null, // `null` gets parsed as "unknown"
+      undefined, // `undefined` gets parsed as "unknown"
+    ]
+
+    if (anTypes.includes(schema?.type)) {
+      return "an"
+    }
+
+    return "a"
+  }
+
+  $: columnTypeName = getTypeName(schema)
+  $: columnIcon = getTypeIcon(schema)
+  $: docLink = getDocLink(schema?.type)
+  $: indefiniteArticle = getIndefiniteArticle(schema)
 </script>
 
 <Line noWrap>
@@ -71,14 +126,14 @@
     on:mouseenter={() => setExplanationSubject(subjects.column)}
     on:mouseleave={() => setExplanationSubject(subjects.none)}
     href={tableHref}
-    text={columnName}
+    text={name}
   />
-  <Text value=" is a " />
+  <Text value={` is ${indefiniteArticle} `} />
   <DocumentationLink
     disabled={docLink === null}
     href={docLink}
     icon={columnIcon}
-    text={`${columnType} column`}
+    text={columnTypeName}
   />
-  <Period />
+  <Text value=" column." />
 </Line>

packages/builder/src/components/design/settings/controls/Explanation/lines/Support.svelte

@@ -2,9 +2,16 @@
   import { Line, InfoWord, DocumentationLink, Text } from "../typography"
   import subjects from "../subjects"
   import * as explanation from "../explanation"
+  import { componentStore } from "stores/builder"
 
   export let setExplanationSubject
   export let support
+  export let componentName
+
+  const getComponentDefinition = componentName => {
+    const components = $componentStore.components || {}
+    return components[componentName] || null
+  }
 
   const getIcon = support => {
     if (support === explanation.support.unsupported) {
@@ -39,21 +46,24 @@
   $: icon = getIcon(support)
   $: color = getColor(support)
   $: text = getText(support)
+  $: componentDefinition = getComponentDefinition(componentName)
 </script>
 
-<Line>
+{#if componentDefinition}
-  <InfoWord
+  <Line>
-    on:mouseenter={() => setExplanationSubject(subjects.support)}
+    <InfoWord
-    on:mouseleave={() => setExplanationSubject(subjects.none)}
+      on:mouseenter={() => setExplanationSubject(subjects.support)}
-    {icon}
+      on:mouseleave={() => setExplanationSubject(subjects.none)}
-    {color}
+      {icon}
-    {text}
+      {color}
-  />
+      {text}
-  <Text value=" with this " />
+    />
-  <DocumentationLink
+    <Text value=" with this " />
-    href="https://docs.budibase.com/docs/chart"
+    <DocumentationLink
-    icon="GraphPie"
+      href={componentDefinition.documentationLink}
-    text="Chart component"
+      icon={componentDefinition.icon}
-  />
+      text={componentDefinition.name}
-  <Text value=" input." />
+    />
-</Line>
+    <Text value=" input." />
+  </Line>
+{/if}

packages/builder/src/components/design/settings/controls/FieldSelect.svelte

@@ -6,8 +6,6 @@
   import { Explanation } from "./Explanation"
   import { debounce } from "lodash"
   import { params } from "@roxi/routify"
-  import { Constants } from "@budibase/frontend-core"
-  import { FIELDS } from "constants/backend"
 
   export let componentInstance = {}
   export let value = ""
@@ -60,35 +58,6 @@
   const onOptionMouseleave = e => {
     updateTooltip(e, null)
   }
-  const getOptionIcon = optionKey => {
-    const option = schema[optionKey]
-    if (!option) return ""
-
-    if (option.autocolumn) {
-      return "MagicWand"
-    }
-    const { type, subtype } = option
-
-    const result =
-      typeof Constants.TypeIconMap[type] === "object" && subtype
-        ? Constants.TypeIconMap[type][subtype]
-        : Constants.TypeIconMap[type]
-
-    return result || "Text"
-  }
-
-  const getOptionIconTooltip = optionKey => {
-    const option = schema[optionKey]
-
-    const type = option?.type
-    const field = Object.values(FIELDS).find(f => f.type === type)
-
-    if (field) {
-      return field.name
-    }
-
-    return ""
-  }
 </script>
 
 <Select
@@ -109,10 +78,9 @@
     <Explanation
       tableHref={`/builder/app/${$params.application}/data/table/${datasource?.tableId}`}
       schema={schema[currentOption]}
-      columnIcon={getOptionIcon(currentOption)}
+      name={currentOption}
-      columnName={currentOption}
-      columnType={getOptionIconTooltip(currentOption)}
       {explanation}
+      componentName={componentInstance._component}
     />
   </ContextTooltip>
 {/if}

packages/builder/src/components/design/settings/controls/MultiFieldSelect.svelte

@@ -4,10 +4,8 @@
   import { selectedScreen } from "stores/builder"
   import { createEventDispatcher } from "svelte"
   import { Explanation } from "./Explanation"
-  import { FIELDS } from "constants/backend"
   import { params } from "@roxi/routify"
   import { debounce } from "lodash"
-  import { Constants } from "@budibase/frontend-core"
 
   export let componentInstance = {}
   export let value = ""
@@ -37,40 +35,6 @@
     dispatch("change", boundValue)
   }
 
-  const getOptionIcon = optionKey => {
-    const option = schema[optionKey]
-    if (!option) return ""
-
-    if (option.autocolumn) {
-      return "MagicWand"
-    }
-    const { type, subtype } = option
-
-    const result =
-      typeof Constants.TypeIconMap[type] === "object" && subtype
-        ? Constants.TypeIconMap[type][subtype]
-        : Constants.TypeIconMap[type]
-
-    return result || "Text"
-  }
-
-  const getOptionIconTooltip = optionKey => {
-    const option = schema[optionKey]
-
-    const type = option?.type
-    const field = Object.values(FIELDS).find(f => f.type === type)
-
-    if (field) {
-      return field.name
-    } else if (type === "jsonarray") {
-      // `jsonarray` isn't present in the above FIELDS constant
-
-      return "JSON Array"
-    }
-
-    return ""
-  }
-
   const updateTooltip = debounce((e, option) => {
     if (option == null) {
       contextTooltipVisible = false
@@ -110,10 +74,9 @@
     <Explanation
       tableHref={`/builder/app/${$params.application}/data/table/${datasource?.tableId}`}
       schema={schema[currentOption]}
-      columnIcon={getOptionIcon(currentOption)}
+      name={currentOption}
-      columnName={currentOption}
-      columnType={getOptionIconTooltip(currentOption)}
       {explanation}
+      componentName={componentInstance._component}
     />
   </ContextTooltip>
 {/if}

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -233,9 +233,9 @@
         response.info = response.info || { code: 200 }
         // if existing schema, copy over what it is
         if (schema) {
-          for (let [name, field] of Object.entries(schema)) {
+          for (let [name, field] of Object.entries(response.schema)) {
-            if (response.schema[name]) {
+            if (!schema[name]) {
-              response.schema[name] = field
+              schema[name] = field
             }
           }
         }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -70,7 +70,7 @@
         <input
           class="input"
           value={title}
-          {title}
+          title={componentName}
           placeholder={componentName}
           on:keypress={e => {
             if (e.key.toLowerCase() === "enter") {
@@ -158,7 +158,32 @@
     overflow: hidden;
     text-overflow: ellipsis;
     white-space: nowrap;
+    position: relative;
+    padding: 5px;
+    right: 6px;
+    border: 1px solid transparent;
+    border-radius: 3px;
+    transition: 150ms background-color, 150ms border-color, 150ms color;
   }
+
+  .input:hover,
+  .input:focus {
+    cursor: text;
+    background-color: var(
+      --spectrum-textfield-m-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+    border: 1px solid white;
+    border-color: var(
+      --spectrum-textfield-m-border-color,
+      var(--spectrum-alias-border-color)
+    );
+    color: var(
+      --spectrum-textfield-m-text-color,
+      var(--spectrum-alias-text-color)
+    );
+  }
+
   .panel-title-content {
     display: contents;
   }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/NewComponentPanel.svelte

@@ -59,7 +59,14 @@
     // Build up list of illegal children from ancestors
     let illegalChildren = definition.illegalChildren || []
     path.forEach(ancestor => {
-      if (ancestor._component === `@budibase/standard-components/sidepanel`) {
+      // Sidepanels and modals can be nested anywhere in the component tree, but really they are always rendered at the top level.
+      // Because of this, it doesn't make sense to carry over any parent illegal children to them, so the array is reset here.
+      if (
+        [
+          "@budibase/standard-components/sidepanel",
+          "@budibase/standard-components/modal",
+        ].includes(ancestor._component)
+      ) {
         illegalChildren = []
       }
       const def = componentStore.getDefinition(ancestor._component)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -14,7 +14,7 @@
   {
     "name": "Layout",
     "icon": "ClassicGridView",
-    "children": ["container", "section", "sidepanel"]
+    "children": ["container", "section", "sidepanel", "modal"]
   },
   {
     "name": "Data",

packages/builder/src/pages/builder/maintenance/index.svelte

@@ -33,7 +33,8 @@
             </Body>
           </Layout>
           <Button
-            on:click={() => (window.location = "https://docs.budibase.com")}
+            on:click={() =>
+              (window.location = "https://docs.budibase.com/docs/migrations")}
             >Migration guide</Button
           >
         {/if}

packages/builder/src/stores/builder/screens.js

@@ -125,7 +125,14 @@ export class ScreenStore extends BudiStore {
         return
       }
 
-      if (type === "@budibase/standard-components/sidepanel") {
+      // Sidepanels and modals can be nested anywhere in the component tree, but really they are always rendered at the top level.
+      // Because of this, it doesn't make sense to carry over any parent illegal children to them, so the array is reset here.
+      if (
+        [
+          "@budibase/standard-components/sidepanel",
+          "@budibase/standard-components/modal",
+        ].includes(type)
+      ) {
         illegalChildren = []
       }
 

packages/cli/package.json

@@ -32,7 +32,7 @@
     "pouchdb": "7.3.0",
     "pouchdb-replication-stream": "1.2.9",
     "randomstring": "1.1.5",
-    "tar": "6.1.15",
+    "tar": "6.2.1",
     "yaml": "^2.1.1"
   },
   "devDependencies": {

packages/client/manifest.json

@@ -11,6 +11,7 @@
     "continueIfAction": true,
     "showNotificationAction": true,
     "sidePanel": true,
+    "modal": true,
     "skeletonLoader": true
   },
   "typeSupportPresets": {
@@ -5223,6 +5224,7 @@
     ]
   },
   "chartblock": {
+    "documentationLink": "https://docs.budibase.com/docs/chart",
     "block": true,
     "name": "Chart Block",
     "icon": "GraphPie",
@@ -6974,7 +6976,7 @@
     "name": "Side Panel",
     "icon": "RailRight",
     "hasChildren": true,
-    "illegalChildren": ["section", "sidepanel"],
+    "illegalChildren": ["section", "sidepanel", "modal"],
     "showEmptyState": false,
     "draggable": false,
     "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action.",
@@ -6992,6 +6994,52 @@
       }
     ]
   },
+  "modal": {
+    "name": "Modal",
+    "icon": "MBox",
+    "hasChildren": true,
+    "illegalChildren": ["section", "modal", "sidepanel"],
+    "showEmptyState": false,
+    "draggable": false,
+    "info": "Modals are hidden by default. They will only be revealed when triggered by the 'Open Modal' action.",
+    "settings": [
+      {
+        "type": "boolean",
+        "key": "ignoreClicksOutside",
+        "label": "Ignore clicks outside",
+        "defaultValue": false
+      },
+      {
+        "type": "event",
+        "key": "onClose",
+        "label": "On close"
+      },
+      {
+        "type": "select",
+        "label": "Size",
+        "key": "size",
+        "defaultValue": "small",
+        "options": [
+          {
+            "label": "Small",
+            "value": "small"
+          },
+          {
+            "label": "Medium",
+            "value": "medium"
+          },
+          {
+            "label": "Large",
+            "value": "large"
+          },
+          {
+            "label": "Fullscreen",
+            "value": "fullscreen"
+          }
+        ]
+      }
+    ]
+  },
   "rowexplorer": {
     "block": true,
     "name": "Row Explorer Block",

packages/client/src/components/ClientApp.svelte

@@ -19,6 +19,8 @@
     devToolsStore,
     devToolsEnabled,
     environmentStore,
+    sidePanelStore,
+    modalStore,
   } from "stores"
   import NotificationDisplay from "components/overlay/NotificationDisplay.svelte"
   import ConfirmationDisplay from "components/overlay/ConfirmationDisplay.svelte"
@@ -102,6 +104,21 @@
         embedded: !!$appStore.embedded,
       })
     }
+    const handleHashChange = () => {
+      const { open: sidePanelOpen } = $sidePanelStore
+      if (sidePanelOpen) {
+        sidePanelStore.actions.close()
+      }
+
+      const { open: modalOpen } = $modalStore
+      if (modalOpen) {
+        modalStore.actions.close()
+      }
+    }
+    window.addEventListener("hashchange", handleHashChange)
+    return () => {
+      window.removeEventListener("hashchange", handleHashChange)
+    }
   })
 
   $: {

packages/client/src/components/app/Layout.svelte

@@ -12,6 +12,7 @@
     linkable,
     builderStore,
     sidePanelStore,
+    modalStore,
     appStore,
   } = sdk
   const context = getContext("context")
@@ -77,6 +78,7 @@
     !$builderStore.inBuilder &&
     $sidePanelStore.open &&
     !$sidePanelStore.ignoreClicksOutside
+
   $: screenId = $builderStore.inBuilder
     ? `${$builderStore.screen?._id}-screen`
     : "screen"
@@ -198,6 +200,7 @@
   const handleClickLink = () => {
     mobileOpen = false
     sidePanelStore.actions.close()
+    modalStore.actions.close()
   }
 </script>
 

packages/client/src/components/app/Link.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
 
-  const { linkable, styleable, builderStore, sidePanelStore } =
+  const { linkable, styleable, builderStore, sidePanelStore, modalStore } =
     getContext("sdk")
   const component = getContext("component")
 
@@ -29,6 +29,11 @@
   // overrides the color when it's passed as inline style.
   $: styles = enrichStyles($component.styles, color)
 
+  const handleUrlChange = () => {
+    sidePanelStore.actions.close()
+    modalStore.actions.close()
+  }
+
   const getSanitizedUrl = (url, externalLink, newTab) => {
     if (!url) {
       return externalLink || newTab ? "#/" : "/"
@@ -109,7 +114,7 @@
         class:italic
         class:underline
         class="align--{align || 'left'} size--{size || 'M'}"
-        on:click={sidePanelStore.actions.close}
+        on:click={handleUrlChange}
       >
         {componentText}
       </a>

packages/client/src/components/app/Modal.svelte

@@ -0,0 +1,141 @@
+<script>
+  import { getContext } from "svelte"
+  import { Modal, Icon } from "@budibase/bbui"
+
+  const component = getContext("component")
+  const { styleable, modalStore, builderStore, dndIsDragging } =
+    getContext("sdk")
+
+  export let onClose
+  export let ignoreClicksOutside
+  export let size
+  let modal
+
+  // Open modal automatically in builder
+  $: {
+    if ($builderStore.inBuilder) {
+      if (
+        $component.inSelectedPath &&
+        $modalStore.contentId !== $component.id
+      ) {
+        modalStore.actions.open($component.id)
+      } else if (
+        !$component.inSelectedPath &&
+        $modalStore.contentId === $component.id &&
+        !$dndIsDragging
+      ) {
+        modalStore.actions.close()
+      }
+    }
+  }
+
+  $: open = $modalStore.contentId === $component.id
+
+  const handleModalClose = async () => {
+    if (onClose) {
+      await onClose()
+    }
+    modalStore.actions.close()
+  }
+
+  const handleOpen = (open, modal) => {
+    if (!modal) return
+
+    if (open) {
+      modal.show()
+    } else {
+      modal.hide()
+    }
+  }
+
+  $: handleOpen(open, modal)
+</script>
+
+<!-- Conditional displaying in the builder is necessary otherwise previews don't update properly upon component deletion -->
+{#if !$builderStore.inBuilder || open}
+  <Modal
+    on:cancel={handleModalClose}
+    bind:this={modal}
+    disableCancel={$builderStore.inBuilder}
+    zIndex={2}
+  >
+    <div use:styleable={$component.styles} class={`modal-content ${size}`}>
+      <div class="modal-header">
+        <Icon
+          color="var(--spectrum-global-color-gray-800)"
+          name="Close"
+          hoverable
+          on:click={handleModalClose}
+        />
+      </div>
+      <div class="modal-main">
+        <div class="modal-main-inner">
+          <slot />
+        </div>
+      </div>
+    </div>
+  </Modal>
+{/if}
+
+<style>
+  .modal-content {
+    display: flex;
+    flex-direction: column;
+    max-width: 100%;
+    box-sizing: border-box;
+    padding: 12px 0px 40px;
+  }
+
+  .small {
+    width: 400px;
+    min-height: 200px;
+  }
+
+  .medium {
+    width: 600px;
+    min-height: 400px;
+  }
+
+  .large {
+    width: 800px;
+    min-height: 600px;
+  }
+
+  .fullscreen {
+    width: calc(100vw - 80px);
+    min-height: calc(100vh - 80px);
+  }
+
+  .modal-header {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    flex-shrink: 0;
+    flex-grow: 0;
+    padding: 0 12px 12px;
+    box-sizing: border-box;
+  }
+
+  .modal-main {
+    padding: 0 40px;
+    flex-grow: 1;
+    display: flex;
+    flex-direction: column;
+  }
+
+  .modal-main :global(.component > *) {
+    max-width: 100%;
+  }
+
+  .modal-main-inner {
+    flex-grow: 1;
+    display: flex;
+    flex-direction: column;
+    word-break: break-word;
+  }
+
+  .modal-main-inner:empty {
+    border-radius: 3px;
+    border: 2px dashed var(--spectrum-global-color-gray-400);
+  }
+</style>

packages/client/src/components/app/SidePanel.svelte

@@ -29,10 +29,6 @@
     }
   }
 
-  // $: {
-
-  // }
-
   // Derive visibility
   $: open = $sidePanelStore.contentId === $component.id
 

packages/client/src/components/app/blocks/ChartBlock.svelte

@@ -35,6 +35,7 @@
   export let valueUnits
   export let yAxisLabel
   export let xAxisLabel
+  export let yAxisUnits
   export let curve
 
   // Area
@@ -85,6 +86,7 @@
           valueUnits,
           yAxisLabel,
           xAxisLabel,
+          yAxisUnits,
           stacked,
           horizontal,
           curve,

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -31,41 +31,23 @@
 
   let schema
 
-  $: formattedFields = convertOldFieldFormat(fields)
-  $: fieldsOrDefault = getDefaultFields(formattedFields, schema)
   $: fetchSchema(dataSource)
   $: id = $component.id
-  // We could simply spread $$props into the inner form and append our
+  $: formattedFields = convertOldFieldFormat(fields)
-  // additions, but that would create svelte warnings about unused props and
+  $: fieldsOrDefault = getDefaultFields(formattedFields, schema)
-  // make maintenance in future more confusing as we typically always have a
+  $: buttonsOrDefault =
-  // proper mapping of schema settings to component exports, without having to
+    buttons ||
-  // search multiple files
+    Utils.buildFormBlockButtonConfig({
-  $: innerProps = {
+      _id: id,
-    dataSource,
+      showDeleteButton,
-    actionUrl,
+      showSaveButton,
-    actionType,
+      saveButtonLabel,
-    size,
+      deleteButtonLabel,
-    disabled,
+      notificationOverride,
-    fields: fieldsOrDefault,
+      actionType,
-    title,
+      actionUrl,
-    description,
+      dataSource,
-    schema,
+    })
-    notificationOverride,
-    buttons:
-      buttons ||
-      Utils.buildFormBlockButtonConfig({
-        _id: id,
-        showDeleteButton,
-        showSaveButton,
-        saveButtonLabel,
-        deleteButtonLabel,
-        notificationOverride,
-        actionType,
-        actionUrl,
-        dataSource,
-      }),
-    buttonPosition: buttons ? buttonPosition : "top",
-  }
 
   // Provide additional data context for live binding eval
   export const getAdditionalDataContext = () => {
@@ -123,5 +105,18 @@
 </script>
 
 <FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
-  <InnerFormBlock {...innerProps} />
+  <InnerFormBlock
+    {dataSource}
+    {actionUrl}
+    {actionType}
+    {size}
+    {disabled}
+    fields={fieldsOrDefault}
+    {title}
+    {description}
+    {schema}
+    {notificationOverride}
+    buttons={buttonsOrDefault}
+    buttonPosition={buttons ? buttonPosition : "top"}
+  />
 </FormBlockWrapper>

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -91,15 +91,13 @@
       {#if description}
         <BlockComponent type="text" props={{ text: description }} order={1} />
       {/if}
-      {#key fields}
+      <BlockComponent type="container">
-        <BlockComponent type="container">
+        <div class="form-block fields" class:mobile={$context.device.mobile}>
-          <div class="form-block fields" class:mobile={$context.device.mobile}>
+          {#each fields as field, idx}
-            {#each fields as field, idx}
+            <FormBlockComponent {field} {schema} order={idx} />
-              <FormBlockComponent {field} {schema} order={idx} />
+          {/each}
-            {/each}
+        </div>
-          </div>
+      </BlockComponent>
-        </BlockComponent>
-      {/key}
     </BlockComponent>
     {#if buttonPosition === "bottom"}
       <BlockComponent

packages/client/src/components/app/charts/AreaChart.svelte

@@ -74,7 +74,6 @@
       },
     },
     xaxis: {
-      type: labelType,
       categories,
       labels: {
         formatter: xAxisFormatter,

packages/client/src/components/app/charts/BarChart.svelte

@@ -72,7 +72,6 @@
     },
     // We can just always provide the categories to the xaxis and horizontal mode automatically handles "tranposing" the categories to the yaxis, but certain things like labels need to be manually put on a certain axis based on the selected mode. Titles do not need to be handled this way, they are exposed to the user as "X axis" and Y Axis" so flipping them would be confusing.
     xaxis: {
-      type: labelType,
       categories,
       labels: {
         formatter: xAxisFormatter,

packages/client/src/components/app/charts/LineChart.svelte

@@ -66,7 +66,6 @@
       },
     },
     xaxis: {
-      type: labelType,
       categories,
       labels: {
         formatter: xAxisFormatter,

packages/client/src/components/app/index.js

@@ -37,6 +37,7 @@ export { default as markdownviewer } from "./MarkdownViewer.svelte"
 export { default as embeddedmap } from "./embedded-map/EmbeddedMap.svelte"
 export { default as grid } from "./Grid.svelte"
 export { default as sidepanel } from "./SidePanel.svelte"
+export { default as modal } from "./Modal.svelte"
 export { default as gridblock } from "./GridBlock.svelte"
 export * from "./charts"
 export * from "./forms"

packages/client/src/components/overlay/ConfirmationDisplay.svelte

@@ -8,6 +8,8 @@
     <ModalContent
       title={$confirmationStore.title}
       onConfirm={confirmationStore.actions.confirm}
+      confirmText={$confirmationStore.confirmButtonText}
+      cancelText={$confirmationStore.cancelButtonText}
     >
       {$confirmationStore.text}
     </ModalContent>

packages/client/src/components/preview/IndicatorSet.svelte

@@ -57,7 +57,9 @@
       return
     }
     nextState.indicators[idx].visible =
-      nextState.indicators[idx].insideSidePanel || entries[0].isIntersecting
+      nextState.indicators[idx].insideModal ||
+      nextState.indicators[idx].insideSidePanel ||
+      entries[0].isIntersecting
     if (++callbackCount === observers.length) {
       state = nextState
       updating = false
@@ -139,6 +141,7 @@
         height: elBounds.height + 4,
         visible: false,
         insideSidePanel: !!child.closest(".side-panel"),
+        insideModal: !!child.closest(".modal-content"),
       })
     })
   }

packages/client/src/sdk.js

@@ -11,6 +11,7 @@ import {
   currentRole,
   environmentStore,
   sidePanelStore,
+  modalStore,
   dndIsDragging,
   confirmationStore,
   roleStore,
@@ -53,6 +54,7 @@ export default {
   componentStore,
   environmentStore,
   sidePanelStore,
+  modalStore,
   dndIsDragging,
   currentRole,
   confirmationStore,

packages/client/src/stores/confirmation.js

@@ -4,6 +4,8 @@ const initialState = {
   showConfirmation: false,
   title: null,
   text: null,
+  confirmButtonText: null,
+  cancelButtonText: null,
   onConfirm: null,
   onCancel: null,
 }
@@ -11,11 +13,20 @@ const initialState = {
 const createConfirmationStore = () => {
   const store = writable(initialState)
 
-  const showConfirmation = (title, text, onConfirm, onCancel) => {
+  const showConfirmation = (
+    title,
+    text,
+    onConfirm,
+    onCancel,
+    confirmButtonText,
+    cancelButtonText
+  ) => {
     store.set({
       showConfirmation: true,
       title,
       text,
+      confirmButtonText,
+      cancelButtonText,
       onConfirm,
       onCancel,
     })

packages/client/src/stores/index.js

@@ -27,6 +27,7 @@ export {
   dndIsDragging,
 } from "./dnd"
 export { sidePanelStore } from "./sidePanel"
+export { modalStore } from "./modal"
 export { hoverStore } from "./hover"
 
 // Context stores are layered and duplicated, so it is not a singleton

packages/client/src/stores/modal.js

@@ -0,0 +1,32 @@
+import { writable } from "svelte/store"
+
+export const createModalStore = () => {
+  const initialState = {
+    contentId: null,
+  }
+  const store = writable(initialState)
+
+  const open = id => {
+    store.update(state => {
+      state.contentId = id
+      return state
+    })
+  }
+
+  const close = () => {
+    store.update(state => {
+      state.contentId = null
+      return state
+    })
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: {
+      open,
+      close,
+    },
+  }
+}
+
+export const modalStore = createModalStore()

packages/client/src/utils/buttonActions.js

@@ -12,6 +12,7 @@ import {
   uploadStore,
   rowSelectionStore,
   sidePanelStore,
+  modalStore,
 } from "stores"
 import { API } from "api"
 import { ActionTypes } from "constants"
@@ -436,6 +437,17 @@ const closeSidePanelHandler = () => {
   sidePanelStore.actions.close()
 }
 
+const openModalHandler = action => {
+  const { id } = action.parameters
+  if (id) {
+    modalStore.actions.open(id)
+  }
+}
+
+const closeModalHandler = () => {
+  modalStore.actions.close()
+}
+
 const downloadFileHandler = async action => {
   const { url, fileName } = action.parameters
   try {
@@ -499,6 +511,8 @@ const handlerMap = {
   ["Prompt User"]: promptUserHandler,
   ["Open Side Panel"]: openSidePanelHandler,
   ["Close Side Panel"]: closeSidePanelHandler,
+  ["Open Modal"]: openModalHandler,
+  ["Close Modal"]: closeModalHandler,
   ["Download File"]: downloadFileHandler,
 }
 
@@ -508,6 +522,7 @@ const confirmTextMap = {
   ["Execute Query"]: "Are you sure you want to execute this query?",
   ["Trigger Automation"]: "Are you sure you want to trigger this automation?",
   ["Prompt User"]: "Are you sure you want to continue?",
+  ["Duplicate Row"]: "Are you sure you want to duplicate this row?",
 }
 
 /**
@@ -568,6 +583,11 @@ export const enrichButtonActions = (actions, context) => {
             const defaultTitleText = action["##eventHandlerType"]
             const customTitleText =
               action.parameters?.customTitleText || defaultTitleText
+            const cancelButtonText =
+              action.parameters?.cancelButtonText || "Cancel"
+            const confirmButtonText =
+              action.parameters?.confirmButtonText || "Confirm"
+
             confirmationStore.actions.showConfirmation(
               customTitleText,
               confirmText,
@@ -598,7 +618,9 @@ export const enrichButtonActions = (actions, context) => {
               },
               () => {
                 resolve(false)
-              }
+              },
+              confirmButtonText,
+              cancelButtonText
             )
           })
         }

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -18,7 +18,7 @@
   import FilterUsers from "./FilterUsers.svelte"
   import { getFields } from "../utils/searchFields"
 
-  const { OperatorOptions } = Constants
+  const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants
 
   export let schemaFields
   export let filters = []
@@ -28,6 +28,23 @@
   export let allowBindings = false
   export let filtersLabel = "Filters"
 
+  $: {
+    if (
+      tables.find(
+        table =>
+          table._id === datasource.tableId &&
+          table.sourceId === DEFAULT_BB_DATASOURCE_ID
+      ) &&
+      !schemaFields.some(field => field.name === "_id")
+    ) {
+      schemaFields = [
+        ...schemaFields,
+        { name: "_id", type: "string" },
+        { name: "_rev", type: "string" },
+      ]
+    }
+  }
+
   $: matchAny = filters?.find(filter => filter.operator === "allOr") != null
   $: onEmptyFilter =
     filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
@@ -35,7 +52,6 @@
   $: fieldFilters = filters.filter(
     filter => filter.operator !== "allOr" && !filter.onEmptyFilter
   )
-
   const behaviourOptions = [
     { value: "and", label: "Match all filters" },
     { value: "or", label: "Match any filter" },
@@ -44,7 +60,6 @@
     { value: "all", label: "Return all table rows" },
     { value: "none", label: "Return no rows" },
   ]
-
   const context = getContext("context")
 
   $: fieldOptions = getFields(tables, schemaFields || [], {

packages/frontend-core/src/components/Updating.svelte

@@ -1,18 +1,22 @@
 <script>
   export let isMigrationDone
   export let onMigrationDone
-  export let timeoutSeconds = 10 // 3 minutes
+  export let timeoutSeconds = 60 // 1 minute
+  export let minTimeSeconds = 3
 
   const loadTime = Date.now()
+  const intervalMs = 1000
   let timedOut = false
+  let secondsWaited = 0
 
   async function checkMigrationsFinished() {
     setTimeout(async () => {
       const isMigrated = await isMigrationDone()
 
       const timeoutMs = timeoutSeconds * 1000
-      if (!isMigrated) {
+      if (!isMigrated || secondsWaited <= minTimeSeconds) {
         if (loadTime + timeoutMs > Date.now()) {
+          secondsWaited += 1
           return checkMigrationsFinished()
         }
 
@@ -20,7 +24,7 @@
       }
 
       onMigrationDone()
-    }, 1000)
+    }, intervalMs)
   }
 
   checkMigrationsFinished()
@@ -41,6 +45,11 @@
   <span class="subtext">
     {#if !timedOut}
       Please wait and we will be back in a second!
+      <br />
+      Checkout the
+      <a href="https://docs.budibase.com/docs/app-migrations" target="_blank"
+        >documentation</a
+      > on app migrations.
     {:else}
       An error occurred, please try again later.
       <br />

packages/frontend-core/src/constants.js

@@ -1,7 +1,11 @@
 /**
  * Operator options for lucene queries
  */
-export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
+export {
+  OperatorOptions,
+  SqlNumberTypeRangeMap,
+  DEFAULT_BB_DATASOURCE_ID,
+} from "@budibase/shared-core"
 export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
 import { FieldType, BBReferenceFieldSubType } from "@budibase/types"

packages/frontend-core/src/utils/utils.js

@@ -161,6 +161,9 @@ export const buildFormBlockButtonConfig = props => {
     {
       "##eventHandlerType": "Close Side Panel",
     },
+    {
+      "##eventHandlerType": "Close Modal",
+    },
     // Clear a create form once submitted
     ...(actionType !== "Create"
       ? []

packages/pro

@@ -1 +1 @@
-Subproject commit 85b4fc9ea01472bf69840d046733ad596ef893e2
+Subproject commit 6c8d0174ca58c578a37022965ddb923fdbf8e32a

packages/server/package.json

@@ -109,8 +109,8 @@
     "serialize-error": "^7.0.1",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
-    "socket.io": "4.6.1",
+    "socket.io": "4.6.2",
-    "tar": "6.1.15",
+    "tar": "6.2.1",
     "to-json-schema": "0.2.5",
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",

packages/server/scripts/integrations/mssql/data/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/mssql/server:2017-latest
+FROM mcr.microsoft.com/mssql/server:2022-latest
 
 ENV ACCEPT_EULA=Y
 ENV SA_PASSWORD=Passw0rd

packages/server/scripts/integrations/postgres/init.sql

@@ -54,8 +54,31 @@ INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('Mi
 INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer', 1996);
 INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0, 1993);
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Jonny', 'Muffin', 'Muffin Street', 'Cork', 'support');
-INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Dave', 'Bar', '2 Foo Street', 'Bartown', 'support', 0, 1993);
-INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('James', 'Bar', '3 Foo Street', 'Bartown', 'support', 0, 1993);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Jenny', 'Bar', '4 Foo Street', 'Bartown', 'support', 0, 1993);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Grace', 'Bar', '5 Foo Street', 'Bartown', 'support', 0, 1993);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Sarah', 'Bar', '6 Foo Street', 'Bartown', 'support', 0, 1993);
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Kelly', 'Bar', '7 Foo Street', 'Bartown', 'support', 0, 1993);
+
+-- insert a lot of tasks for testing
+WITH RECURSIVE generate_series AS (
+  SELECT 1 AS n
+  UNION ALL
+  SELECT n + 1 FROM generate_series WHERE n < 6000
+),
+random_data AS (
+  SELECT
+    n,
+    (random() * 9 + 1)::int AS ExecutorID,
+    (random() * 9 + 1)::int AS QaID,
+    'assembling' AS TaskName,
+    (random() < 0.5) AS Completed
+  FROM generate_series
+)
+INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed)
+SELECT ExecutorID, QaID, TaskName, Completed
+FROM random_data;
 INSERT INTO Products (ProductName) VALUES ('Computers');
 INSERT INTO Products (ProductName) VALUES ('Laptops');
 INSERT INTO Products (ProductName) VALUES ('Chairs');

packages/server/src/api/controllers/query/index.ts

@@ -311,8 +311,8 @@ export async function preview(
 
   // if existing schema, update to include any previous schema keys
   if (existingSchema) {
-    for (let key of Object.keys(previewSchema)) {
+    for (let key of Object.keys(existingSchema)) {
-      if (existingSchema[key]) {
+      if (!previewSchema[key]) {
         previewSchema[key] = existingSchema[key]
       }
     }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -7,6 +7,7 @@ import {
   FieldType,
   FilterType,
   IncludeRelationship,
+  isManyToOne,
   OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
@@ -16,29 +17,33 @@ import {
   SortJson,
   SortType,
   Table,
-  isManyToOne,
 } from "@budibase/types"
 import {
   breakExternalTableId,
   breakRowIdField,
   convertRowId,
+  generateRowIdField,
   isRowId,
   isSQL,
-  generateRowIdField,
 } from "../../../integrations/utils"
 import {
   buildExternalRelationships,
   buildSqlFieldList,
   generateIdForRow,
-  sqlOutputProcessing,
+  isKnexEmptyReadResponse,
   isManyToMany,
+  sqlOutputProcessing,
 } from "./utils"
-import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
+import {
+  getDatasourceAndQuery,
+  processRowCountResponse,
+} from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import env from "../../../environment"
+import { makeExternalQuery } from "../../../integrations/base/query"
 
 export interface ManyRelationship {
   tableId?: string
@@ -60,91 +65,12 @@ export interface RunConfig {
   includeSqlRelationships?: IncludeRelationship
 }
 
-function buildFilters(
+export type ExternalRequestReturnType<T extends Operation> =
-  id: string | undefined | string[],
+  T extends Operation.READ
-  filters: SearchFilters,
+    ? Row[]
-  table: Table
+    : T extends Operation.COUNT
-) {
+    ? number
-  const primary = table.primary
+    : { row: Row; table: Table }
-  // if passed in array need to copy for shifting etc
-  let idCopy: undefined | string | any[] = cloneDeep(id)
-  if (filters) {
-    // need to map over the filters and make sure the _id field isn't present
-    let prefix = 1
-    for (let operator of Object.values(filters)) {
-      for (let field of Object.keys(operator || {})) {
-        if (dbCore.removeKeyNumbering(field) === "_id") {
-          if (primary) {
-            const parts = breakRowIdField(operator[field])
-            for (let field of primary) {
-              operator[`${prefix}:${field}`] = parts.shift()
-            }
-            prefix++
-          }
-          // make sure this field doesn't exist on any filter
-          delete operator[field]
-        }
-      }
-    }
-  }
-  // there is no id, just use the user provided filters
-  if (!idCopy || !table) {
-    return filters
-  }
-  // if used as URL parameter it will have been joined
-  if (!Array.isArray(idCopy)) {
-    idCopy = breakRowIdField(idCopy)
-  }
-  const equal: any = {}
-  if (primary && idCopy) {
-    for (let field of primary) {
-      // work through the ID and get the parts
-      equal[field] = idCopy.shift()
-    }
-  }
-  return {
-    equal,
-  }
-}
-
-async function removeManyToManyRelationships(
-  rowId: string,
-  table: Table,
-  colName: string
-) {
-  const tableId = table._id!
-  const filters = buildFilters(rowId, {}, table)
-  // safety check, if there are no filters on deletion bad things happen
-  if (Object.keys(filters).length !== 0) {
-    return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, Operation.DELETE),
-      body: { [colName]: null },
-      filters,
-      meta: {
-        table,
-      },
-    })
-  } else {
-    return []
-  }
-}
-
-async function removeOneToManyRelationships(rowId: string, table: Table) {
-  const tableId = table._id!
-  const filters = buildFilters(rowId, {}, table)
-  // safety check, if there are no filters on deletion bad things happen
-  if (Object.keys(filters).length !== 0) {
-    return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, Operation.UPDATE),
-      filters,
-      meta: {
-        table,
-      },
-    })
-  } else {
-    return []
-  }
-}
 
 /**
  * This function checks the incoming parameters to make sure all the inputs are
@@ -200,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) {
   }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   return {
-    datasourceId: datasourceId!,
+    datasourceId: datasourceId,
-    entityId: tableName!,
+    entityId: tableName,
     operation: operation as Operation,
   }
 }
@@ -223,14 +149,12 @@ function isEditableColumn(column: FieldSchema) {
   return !(isExternalAutoColumn || isFormula)
 }
 
-export type ExternalRequestReturnType<T extends Operation> =
-  T extends Operation.READ ? Row[] : { row: Row; table: Table }
-
 export class ExternalRequest<T extends Operation> {
   private readonly operation: T
   private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
+  private tableList: Table[]
 
   constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
@@ -239,22 +163,134 @@ export class ExternalRequest<T extends Operation> {
     if (datasource && datasource.entities) {
       this.tables = datasource.entities
     }
+    this.tableList = Object.values(this.tables)
+  }
+
+  private prepareFilters(
+    id: string | undefined | string[],
+    filters: SearchFilters,
+    table: Table
+  ): SearchFilters {
+    // replace any relationship columns initially, table names and relationship column names are acceptable
+    const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table)
+    filters = sdk.rows.filters.updateFilterKeys(
+      filters,
+      relationshipColumns.map(({ name, definition }) => {
+        const { tableName } = breakExternalTableId(definition.tableId)
+        return {
+          original: name,
+          updated: tableName,
+        }
+      })
+    )
+    const primary = table.primary
+    // if passed in array need to copy for shifting etc
+    let idCopy: undefined | string | any[] = cloneDeep(id)
+    if (filters) {
+      // need to map over the filters and make sure the _id field isn't present
+      let prefix = 1
+      for (let operator of Object.values(filters)) {
+        for (let field of Object.keys(operator || {})) {
+          if (dbCore.removeKeyNumbering(field) === "_id") {
+            if (primary) {
+              const parts = breakRowIdField(operator[field])
+              for (let field of primary) {
+                operator[`${prefix}:${field}`] = parts.shift()
+              }
+              prefix++
+            }
+            // make sure this field doesn't exist on any filter
+            delete operator[field]
+          }
+        }
+      }
+    }
+    // there is no id, just use the user provided filters
+    if (!idCopy || !table) {
+      return filters
+    }
+    // if used as URL parameter it will have been joined
+    if (!Array.isArray(idCopy)) {
+      idCopy = breakRowIdField(idCopy)
+    }
+    const equal: SearchFilters["equal"] = {}
+    if (primary && idCopy) {
+      for (let field of primary) {
+        // work through the ID and get the parts
+        equal[field] = idCopy.shift()
+      }
+    }
+    return {
+      equal,
+    }
+  }
+
+  private async removeManyToManyRelationships(
+    rowId: string,
+    table: Table,
+    colName: string
+  ) {
+    const tableId = table._id!
+    const filters = this.prepareFilters(rowId, {}, table)
+    // safety check, if there are no filters on deletion bad things happen
+    if (Object.keys(filters).length !== 0) {
+      return getDatasourceAndQuery({
+        endpoint: getEndpoint(tableId, Operation.DELETE),
+        body: { [colName]: null },
+        filters,
+        meta: {
+          table,
+        },
+      })
+    } else {
+      return []
+    }
+  }
+
+  private async removeOneToManyRelationships(rowId: string, table: Table) {
+    const tableId = table._id!
+    const filters = this.prepareFilters(rowId, {}, table)
+    // safety check, if there are no filters on deletion bad things happen
+    if (Object.keys(filters).length !== 0) {
+      return getDatasourceAndQuery({
+        endpoint: getEndpoint(tableId, Operation.UPDATE),
+        filters,
+        meta: {
+          table,
+        },
+      })
+    } else {
+      return []
+    }
   }
 
   getTable(tableId: string | undefined): Table | undefined {
     if (!tableId) {
-      throw "Table ID is unknown, cannot find table"
+      throw new Error("Table ID is unknown, cannot find table")
     }
     const { tableName } = breakExternalTableId(tableId)
-    if (tableName) {
+    return this.tables[tableName]
-      return this.tables[tableName]
+  }
+
+  // seeds the object with table and datasource information
+  async retrieveMetadata(
+    datasourceId: string
+  ): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
+    if (!this.datasource) {
+      this.datasource = await sdk.datasources.get(datasourceId)
+      if (!this.datasource || !this.datasource.entities) {
+        throw "No tables found, fetch tables before query."
+      }
+      this.tables = this.datasource.entities
+      this.tableList = Object.values(this.tables)
     }
+    return { tables: this.tables, datasource: this.datasource }
   }
 
   async getRow(table: Table, rowId: string): Promise<Row> {
     const response = await getDatasourceAndQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
-      filters: buildFilters(rowId, {}, table),
+      filters: this.prepareFilters(rowId, {}, table),
       meta: {
         table,
       },
@@ -280,16 +316,20 @@ export class ExternalRequest<T extends Operation> {
       manyRelationships: ManyRelationship[] = []
     for (let [key, field] of Object.entries(table.schema)) {
       // if set already, or not set just skip it
-      if (row[key] === undefined || newRow[key] || !isEditableColumn(field)) {
+      if (row[key] === undefined || newRow[key]) {
+        continue
+      }
+      if (
+        !(this.operation === Operation.BULK_UPSERT) &&
+        !isEditableColumn(field)
+      ) {
         continue
       }
       // parse floats/numbers
       if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
         newRow[key] = parseFloat(row[key])
       } else if (field.type === FieldType.LINK) {
-        const { tableName: linkTableName } = breakExternalTableId(
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-          field?.tableId
-        )
         // table has to exist for many to many
         if (!linkTableName || !this.tables[linkTableName]) {
           continue
@@ -370,9 +410,6 @@ export class ExternalRequest<T extends Operation> {
       [key: string]: { rows: Row[]; isMany: boolean; tableId: string }
     } = {}
     const { tableName } = breakExternalTableId(tableId)
-    if (!tableName) {
-      return related
-    }
     const table = this.tables[tableName]
     // @ts-ignore
     const primaryKey = table.primary[0]
@@ -428,7 +465,9 @@ export class ExternalRequest<T extends Operation> {
       })
       // this is the response from knex if no rows found
       const rows: Row[] =
-        !Array.isArray(response) || response?.[0].read ? [] : response
+        !Array.isArray(response) || isKnexEmptyReadResponse(response)
+          ? []
+          : response
       const storeTo = isManyToMany(field)
         ? field.throughFrom || linkPrimaryKey
         : fieldName
@@ -503,7 +542,7 @@ export class ExternalRequest<T extends Operation> {
             endpoint: getEndpoint(tableId, operation),
             // if we're doing many relationships then we're writing, only one response
             body,
-            filters: buildFilters(id, {}, linkTable),
+            filters: this.prepareFilters(id, {}, linkTable),
             meta: {
               table: linkTable,
             },
@@ -517,7 +556,7 @@ export class ExternalRequest<T extends Operation> {
     // finally cleanup anything that needs to be removed
     for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
       const table: Table | undefined = this.getTable(tableId)
-      // if its not the foreign key skip it, nothing to do
+      // if it's not the foreign key skip it, nothing to do
       if (
         !table ||
         (!isMany && table.primary && table.primary.indexOf(colName) !== -1)
@@ -527,8 +566,8 @@ export class ExternalRequest<T extends Operation> {
       for (let row of rows) {
         const rowId = generateIdForRow(row, table)
         const promise: Promise<any> = isMany
-          ? removeManyToManyRelationships(rowId, table, colName)
+          ? this.removeManyToManyRelationships(rowId, table, colName)
-          : removeOneToManyRelationships(rowId, table)
+          : this.removeOneToManyRelationships(rowId, table)
         if (promise) {
           promises.push(promise)
         }
@@ -551,12 +590,12 @@ export class ExternalRequest<T extends Operation> {
         rows.map(row => {
           const rowId = generateIdForRow(row, table)
           return isMany
-            ? removeManyToManyRelationships(
+            ? this.removeManyToManyRelationships(
                 rowId,
                 table,
                 relationshipColumn.fieldName
               )
-            : removeOneToManyRelationships(rowId, table)
+            : this.removeOneToManyRelationships(rowId, table)
         })
       )
     }
@@ -564,21 +603,21 @@ export class ExternalRequest<T extends Operation> {
 
   async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    if (!tableId) {
-    if (!tableName) {
+      throw new Error("Unable to run without a table ID")
-      throw "Unable to run without a table name"
     }
-    if (!this.datasource) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
-      this.datasource = await sdk.datasources.get(datasourceId!)
+    let datasource = this.datasource
-      if (!this.datasource || !this.datasource.entities) {
+    if (!datasource) {
-        throw "No tables found, fetch tables before query."
+      const { datasource: ds } = await this.retrieveMetadata(datasourceId)
-      }
+      datasource = ds
-      this.tables = this.datasource.entities
     }
     const table = this.tables[tableName]
-    let isSql = isSQL(this.datasource)
+    let isSql = isSQL(datasource)
     if (!table) {
-      throw `Unable to process query, table "${tableName}" not defined.`
+      throw new Error(
+        `Unable to process query, table "${tableName}" not defined.`
+      )
     }
     // look for specific components of config which may not be considered acceptable
     let { id, row, filters, sort, paginate, rows } = cleanupConfig(
@@ -601,7 +640,7 @@ export class ExternalRequest<T extends Operation> {
           break
       }
     }
-    filters = buildFilters(id, filters || {}, table)
+    filters = this.prepareFilters(id, filters || {}, table)
     const relationships = buildExternalRelationships(table, this.tables)
 
     const incRelationships =
@@ -649,10 +688,15 @@ export class ExternalRequest<T extends Operation> {
       body: row || rows,
       // pass an id filter into extra, purely for mysql/returning
       extra: {
-        idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+        idFilter: this.prepareFilters(
+          id || generateIdForRow(row, table),
+          {},
+          table
+        ),
       },
       meta: {
         table,
+        id: config.id,
       },
     }
 
@@ -662,12 +706,14 @@ export class ExternalRequest<T extends Operation> {
     }
 
     // aliasing can be disabled fully if desired
-    let response
+    const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
-    if (env.SQL_ALIASING_DISABLE) {
+    let response = env.SQL_ALIASING_DISABLE
-      response = await getDatasourceAndQuery(json)
+      ? await getDatasourceAndQuery(json)
-    } else {
+      : await aliasing.queryWithAliasing(json, makeExternalQuery)
-      const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
+
-      response = await aliasing.queryWithAliasing(json)
+    // if it's a counting operation there will be no more processing, just return the number
+    if (this.operation === Operation.COUNT) {
+      return processRowCountResponse(response) as ExternalRequestReturnType<T>
     }
 
     const responseRows = Array.isArray(response) ? response : []

packages/server/src/api/controllers/row/external.ts

@@ -39,9 +39,10 @@ export async function handleRequest<T extends Operation>(
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = utils.getTableId(ctx)
-  const { _id, ...rowData } = ctx.request.body
 
+  const { _id, ...rowData } = ctx.request.body
   const table = await sdk.tables.getTable(tableId)
+
   const { row: dataToUpdate } = await inputProcessing(
     ctx.user?._id,
     cloneDeep(table),
@@ -79,6 +80,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     ...response,
     row: enrichedRow,
     table,
+    oldRow: beforeRow,
   }
 }
 
@@ -134,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
+  const datasource: Datasource = await sdk.datasources.get(datasourceId)
-  if (!tableName) {
-    ctx.throw(400, "Unable to find table.")
-  }
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
@@ -161,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     }
     const links = row[fieldName]
     const linkedTableId = field.tableId
-    const linkedTableName = breakExternalTableId(linkedTableId).tableName!
+    const linkedTableName = breakExternalTableId(linkedTableId).tableName
     const linkedTable = tables[linkedTableName]
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])

packages/server/src/api/controllers/row/index.ts

@@ -55,13 +55,13 @@ export async function patch(
     return save(ctx)
   }
   try {
-    const { row, table } = await pickApi(tableId).patch(ctx)
+    const { row, table, oldRow } = await pickApi(tableId).patch(ctx)
     if (!row) {
       ctx.throw(404, "Row not found")
     }
     ctx.status = 200
     ctx.eventEmitter &&
-      ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
+      ctx.eventEmitter.emitRow(`row:update`, appId, row, table, oldRow)
     ctx.message = `${table.name} updated successfully.`
     ctx.body = row
     gridSocket?.emitRowUpdate(ctx, row)

packages/server/src/api/controllers/row/internal.ts

@@ -85,13 +85,15 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     // the row has been updated, need to put it into the ctx
     ctx.request.body = row as any
     await userController.updateMetadata(ctx as any)
-    return { row: ctx.body as Row, table }
+    return { row: ctx.body as Row, table, oldRow }
   }
 
-  return finaliseRow(table, row, {
+  const result = await finaliseRow(table, row, {
     oldTable: dbTable,
     updateFormula: true,
   })
+
+  return { ...result, oldRow }
 }
 
 export async function find(ctx: UserCtx): Promise<Row> {

packages/server/src/api/controllers/row/utils/basic.ts

@@ -99,7 +99,7 @@ export function basicProcessing({
         row,
         tableName: table._id!,
         fieldName: internalColumn,
-        isLinked: false,
+        isLinked,
       })
     }
   }

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -1,5 +1,9 @@
 import {
+  DatasourcePlusQueryResponse,
+  DSPlusOperation,
   FieldType,
+  isManyToOne,
+  isOneToMany,
   ManyToManyRelationshipFieldMetadata,
   RelationshipFieldMetadata,
   RelationshipsJson,
@@ -91,12 +95,12 @@ export function buildExternalRelationships(
 ): RelationshipsJson[] {
   const relationships = []
   for (let [fieldName, field] of Object.entries(table.schema)) {
-    if (field.type !== FieldType.LINK) {
+    if (field.type !== FieldType.LINK || !field.tableId) {
       continue
     }
     const { tableName: linkTableName } = breakExternalTableId(field.tableId)
     // no table to link to, this is not a valid relationships
-    if (!linkTableName || !tables[linkTableName]) {
+    if (!tables[linkTableName]) {
       continue
     }
     const linkTable = tables[linkTableName]
@@ -108,7 +112,7 @@ export function buildExternalRelationships(
       // need to specify where to put this back into
       column: fieldName,
     }
-    if (isManyToMany(field)) {
+    if (isManyToMany(field) && field.through) {
       const { tableName: throughTableName } = breakExternalTableId(
         field.through
       )
@@ -118,7 +122,7 @@ export function buildExternalRelationships(
       definition.to = field.throughFrom || linkTable.primary[0]
       definition.fromPrimary = table.primary[0]
       definition.toPrimary = linkTable.primary[0]
-    } else {
+    } else if (isManyToOne(field) || isOneToMany(field)) {
       // if no foreign key specified then use the name of the field in other table
       definition.from = field.foreignKey || table.primary[0]
       definition.to = field.fieldName
@@ -178,17 +182,27 @@ export function buildSqlFieldList(
   }
   let fields = extractRealFields(table)
   for (let field of Object.values(table.schema)) {
-    if (field.type !== FieldType.LINK || !opts?.relationships) {
+    if (
+      field.type !== FieldType.LINK ||
+      !opts?.relationships ||
+      !field.tableId
+    ) {
       continue
     }
     const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-    if (linkTableName) {
+    const linkTable = tables[linkTableName]
-      const linkTable = tables[linkTableName]
+    if (linkTable) {
-      if (linkTable) {
+      const linkedFields = extractRealFields(linkTable, fields)
-        const linkedFields = extractRealFields(linkTable, fields)
+      fields = fields.concat(linkedFields)
-        fields = fields.concat(linkedFields)
-      }
     }
   }
   return fields
 }
+
+export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {
+  return (
+    !Array.isArray(resp) ||
+    resp.length === 0 ||
+    (DSPlusOperation.READ in resp[0] && resp[0].read === true)
+  )
+}

packages/server/src/api/controllers/row/utils/utils.ts

@@ -14,7 +14,7 @@ import {
   processDates,
   processFormulas,
 } from "../../../../utilities/rowProcessor"
-import { updateRelationshipColumns } from "./sqlUtils"
+import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils"
 import {
   basicProcessing,
   generateIdForRow,
@@ -137,7 +137,7 @@ export async function sqlOutputProcessing(
   relationships: RelationshipsJson[],
   opts?: { sqs?: boolean }
 ): Promise<Row[]> {
-  if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) {
+  if (isKnexEmptyReadResponse(rows)) {
     return []
   }
   let finalRows: { [key: string]: Row } = {}

packages/server/src/api/controllers/row/views.ts

@@ -69,6 +69,7 @@ export async function searchView(
     limit: body.limit,
     bookmark: body.bookmark,
     paginate: body.paginate,
+    countRows: body.countRows,
   }
 
   const result = await sdk.rows.search(searchOptions)

packages/server/src/api/controllers/table/external.ts

@@ -16,14 +16,18 @@ import {
 import sdk from "../../../sdk"
 import { builderSocket } from "../../../websockets"
 import { inputProcessing } from "../../../utilities/rowProcessor"
+import { isEqual } from "lodash"
 
 function getDatasourceId(table: Table) {
   if (!table) {
-    throw "No table supplied"
+    throw new Error("No table supplied")
   }
   if (table.sourceId) {
     return table.sourceId
   }
+  if (!table._id) {
+    throw new Error("No table ID supplied")
+  }
   return breakExternalTableId(table._id).datasourceId
 }
 
@@ -82,15 +86,30 @@ export async function bulkImport(
   ctx: UserCtx<BulkImportRequest, BulkImportResponse>
 ) {
   let table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows } = ctx.request.body
+  const { rows, identifierFields } = ctx.request.body
   const schema = table.schema
 
+  if (
+    identifierFields &&
+    identifierFields.length > 0 &&
+    !isEqual(identifierFields, table.primary)
+  ) {
+    // This is becuse we make use of the ON CONFLICT functionality in SQL
+    // databases, which only triggers when there's a conflict against a unique
+    // index. The only unique index we can count on atm in Budibase is the
+    // primary key, so this functionality always uses the primary key.
+    ctx.throw(
+      400,
+      "Identifier fields are not supported for bulk import into an external datasource."
+    )
+  }
+
   if (!rows || !isRows(rows) || !isSchema(schema)) {
     ctx.throw(400, "Provided data import information is invalid.")
   }
 
   const parsedRows = []
-  for (const row of parse(rows, schema)) {
+  for (const row of parse(rows, table)) {
     const processed = await inputProcessing(ctx.user?._id, table, row, {
       noAutoRelationships: true,
     })
@@ -98,7 +117,7 @@ export async function bulkImport(
     table = processed.table
   }
 
-  await handleRequest(Operation.BULK_CREATE, table._id!, {
+  await handleRequest(Operation.BULK_UPSERT, table._id!, {
     rows: parsedRows,
   })
   await events.rows.imported(table, parsedRows.length)

packages/server/src/api/controllers/table/utils.ts

@@ -178,7 +178,7 @@ export async function handleDataImport(
   }
 
   const db = context.getAppDB()
-  const data = parse(importRows, schema)
+  const data = parse(importRows, table)
 
   let finalData: any = await importToRows(data, table, user)
 

packages/server/src/api/routes/row.ts

@@ -86,6 +86,7 @@ router
 
 router.post(
   "/api/v2/views/:viewId/search",
+  internalSearchValidator(),
   authorizedResource(PermissionType.VIEW, PermissionLevel.READ, "viewId"),
   rowController.views.searchView
 )

packages/server/src/api/routes/tests/automation.spec.ts

@@ -13,6 +13,7 @@ import { events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { Automation } from "@budibase/types"
 import { mocks } from "@budibase/backend-core/tests"
+import { FilterConditions } from "../../../automations/steps/filter"
 
 const MAX_RETRIES = 4
 let {
@@ -21,6 +22,7 @@ let {
   automationTrigger,
   automationStep,
   collectAutomation,
+  filterAutomation,
 } = setup.structures
 
 describe("/automations", () => {
@@ -155,7 +157,12 @@ describe("/automations", () => {
       automation.appId = config.appId
       automation = await config.createAutomation(automation)
       await setup.delay(500)
-      const res = await testAutomation(config, automation)
+      const res = await testAutomation(config, automation, {
+        row: {
+          name: "Test",
+          description: "TEST",
+        },
+      })
       expect(events.automation.tested).toHaveBeenCalledTimes(1)
       // this looks a bit mad but we don't actually have a way to wait for a response from the automation to
       // know that it has finished all of its actions - this is currently the best way
@@ -436,4 +443,38 @@ describe("/automations", () => {
       expect(res).toEqual(true)
     })
   })
+
+  describe("Update Row Old / New Row comparison", () => {
+    it.each([
+      { oldCity: "asdsadsadsad", newCity: "new" },
+      { oldCity: "Belfast", newCity: "Belfast" },
+    ])(
+      "triggers an update row automation and compares new to old rows with old city '%s' and new city '%s'",
+      async ({ oldCity, newCity }) => {
+        const expectedResult = oldCity === newCity
+
+        let table = await config.createTable()
+
+        let automation = await filterAutomation()
+        automation.definition.trigger.inputs.tableId = table._id
+        automation.definition.steps[0].inputs = {
+          condition: FilterConditions.EQUAL,
+          field: "{{ trigger.row.City }}",
+          value: "{{ trigger.oldRow.City }}",
+        }
+        automation.appId = config.appId!
+        automation = await config.createAutomation(automation)
+        let triggerInputs = {
+          oldRow: {
+            City: oldCity,
+          },
+          row: {
+            City: newCity,
+          },
+        }
+        const res = await testAutomation(config, automation, triggerInputs)
+        expect(res.body.steps[1].outputs.result).toEqual(expectedResult)
+      }
+    )
+  })
 })

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -10,37 +10,11 @@ import * as setup from "../utilities"
 import {
   DatabaseName,
   getDatasource,
-  rawQuery,
+  knexClient,
 } from "../../../../integrations/tests/utils"
 import { Expectations } from "src/tests/utilities/api/base"
 import { events } from "@budibase/backend-core"
-
+import { Knex } from "knex"
-const createTableSQL: Record<string, string> = {
-  [SourceName.POSTGRES]: `
-    CREATE TABLE test_table (
-        id serial PRIMARY KEY,
-        name VARCHAR ( 50 ) NOT NULL,
-        birthday TIMESTAMP,
-        number INT
-    );`,
-  [SourceName.MYSQL]: `
-    CREATE TABLE test_table (
-        id INT AUTO_INCREMENT PRIMARY KEY,
-        name VARCHAR(50) NOT NULL,
-        birthday TIMESTAMP,
-        number INT
-    );`,
-  [SourceName.SQL_SERVER]: `
-    CREATE TABLE test_table (
-        id INT IDENTITY(1,1) PRIMARY KEY,
-        name NVARCHAR(50) NOT NULL,
-        birthday DATETIME,
-        number INT
-    );`,
-}
-
-const insertSQL = `INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')`
-const dropTableSQL = `DROP TABLE test_table;`
 
 describe.each(
   [
@@ -53,6 +27,7 @@ describe.each(
   const config = setup.getConfig()
   let rawDatasource: Datasource
   let datasource: Datasource
+  let client: Knex
 
   async function createQuery(
     query: Partial<Query>,
@@ -82,21 +57,34 @@ describe.each(
     rawDatasource = await dsProvider
     datasource = await config.api.datasource.create(rawDatasource)
 
-    // The Datasource API does not return the password, but we need
+    // The Datasource API doesn ot return the password, but we need it later to
-    // it later to connect to the underlying database, so we fill it
+    // connect to the underlying database, so we fill it back in here.
-    // back in here.
     datasource.config!.password = rawDatasource.config!.password
 
-    await rawQuery(datasource, createTableSQL[datasource.source])
+    client = await knexClient(rawDatasource)
-    await rawQuery(datasource, insertSQL)
+
+    await client.schema.dropTableIfExists("test_table")
+    await client.schema.createTable("test_table", table => {
+      table.increments("id").primary()
+      table.string("name")
+      table.timestamp("birthday")
+      table.integer("number")
+    })
+
+    await client("test_table").insert([
+      { name: "one" },
+      { name: "two" },
+      { name: "three" },
+      { name: "four" },
+      { name: "five" },
+    ])
 
     jest.clearAllMocks()
   })
 
   afterEach(async () => {
     const ds = await config.api.datasource.get(datasource._id!)
-    config.api.datasource.delete(ds)
+    await config.api.datasource.delete(ds)
-    await rawQuery(datasource, dropTableSQL)
   })
 
   afterAll(async () => {
@@ -207,7 +195,7 @@ describe.each(
           },
         })
 
-        await config.publish()
+        await config.api.application.publish(config.getAppId())
         const prodQuery = await config.api.query.getProd(query._id!)
 
         expect(prodQuery._id).toEqual(query._id)
@@ -262,6 +250,67 @@ describe.each(
       expect(events.query.previewed).toHaveBeenCalledTimes(1)
     })
 
+    it("should update schema when column type changes from number to string", async () => {
+      const tableName = "schema_change_test"
+      await client.schema.dropTableIfExists(tableName)
+
+      await client.schema.createTable(tableName, table => {
+        table.increments("id").primary()
+        table.string("name")
+        table.integer("data")
+      })
+
+      await client(tableName).insert({
+        name: "test",
+        data: 123,
+      })
+
+      const firstPreview = await config.api.query.preview({
+        datasourceId: datasource._id!,
+        name: "Test Query",
+        queryVerb: "read",
+        fields: {
+          sql: `SELECT * FROM ${tableName}`,
+        },
+        parameters: [],
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      })
+
+      expect(firstPreview.schema).toEqual(
+        expect.objectContaining({
+          data: { type: "number", name: "data" },
+        })
+      )
+
+      await client.schema.alterTable(tableName, table => {
+        table.string("data").alter()
+      })
+
+      await client(tableName).update({
+        data: "string value",
+      })
+
+      const secondPreview = await config.api.query.preview({
+        datasourceId: datasource._id!,
+        name: "Test Query",
+        queryVerb: "read",
+        fields: {
+          sql: `SELECT * FROM ${tableName}`,
+        },
+        parameters: [],
+        transformer: "return data",
+        schema: firstPreview.schema,
+        readable: true,
+      })
+
+      expect(secondPreview.schema).toEqual(
+        expect.objectContaining({
+          data: { type: "string", name: "data" },
+        })
+      )
+    })
     it("should work with static variables", async () => {
       await config.api.datasource.update({
         ...datasource,
@@ -429,11 +478,11 @@ describe.each(
           },
         ])
 
-        const rows = await rawQuery(
+        const rows = await client("test_table").where({ name: "baz" }).select()
-          datasource,
-          "SELECT * FROM test_table WHERE name = 'baz'"
-        )
         expect(rows).toHaveLength(1)
+        for (const row of rows) {
+          expect(row).toMatchObject({ name: "baz" })
+        }
       })
 
       it("should not allow handlebars as parameters", async () => {
@@ -490,11 +539,14 @@ describe.each(
 
           expect(result.data).toEqual([{ created: true }])
 
-          const rows = await rawQuery(
+          const rows = await client("test_table")
-            datasource,
+            .where({ birthday: datetimeStr })
-            `SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'`
+            .select()
-          )
           expect(rows).toHaveLength(1)
+
+          for (const row of rows) {
+            expect(new Date(row.birthday)).toEqual(date)
+          }
         }
       )
 
@@ -522,10 +574,9 @@ describe.each(
 
           expect(result.data).toEqual([{ created: true }])
 
-          const rows = await rawQuery(
+          const rows = await client("test_table")
-            datasource,
+            .where({ name: notDateStr })
-            `SELECT * FROM test_table WHERE name = '${notDateStr}'`
+            .select()
-          )
           expect(rows).toHaveLength(1)
         }
       )
@@ -660,10 +711,7 @@ describe.each(
           },
         ])
 
-        const rows = await rawQuery(
+        const rows = await client("test_table").where({ id: 1 }).select()
-          datasource,
-          "SELECT * FROM test_table WHERE id = 1"
-        )
         expect(rows).toEqual([
           { id: 1, name: "foo", birthday: null, number: null },
         ])
@@ -731,10 +779,7 @@ describe.each(
           },
         ])
 
-        const rows = await rawQuery(
+        const rows = await client("test_table").where({ id: 1 }).select()
-          datasource,
-          "SELECT * FROM test_table WHERE id = 1"
-        )
         expect(rows).toHaveLength(0)
       })
     })
@@ -750,6 +795,7 @@ describe.each(
             name: entityId,
             schema: {},
             type: "table",
+            primary: ["id"],
             sourceId: datasource._id!,
             sourceType: TableSourceType.EXTERNAL,
           },

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -137,6 +137,67 @@ describe("/queries", () => {
       })
     })
 
+    it("should update schema when structure changes from object to array", async () => {
+      const name = generator.guid()
+
+      await withCollection(async collection => {
+        await collection.insertOne({ name, field: { subfield: "value" } })
+      })
+
+      const firstPreview = await config.api.query.preview({
+        name: "Test Query",
+        datasourceId: datasource._id!,
+        fields: {
+          json: { name: { $eq: name } },
+          extra: {
+            collection,
+            actionType: "findOne",
+          },
+        },
+        schema: {},
+        queryVerb: "read",
+        parameters: [],
+        transformer: "return data",
+        readable: true,
+      })
+
+      expect(firstPreview.schema).toEqual(
+        expect.objectContaining({
+          field: { type: "json", name: "field" },
+        })
+      )
+
+      await withCollection(async collection => {
+        await collection.updateOne(
+          { name },
+          { $set: { field: ["value1", "value2"] } }
+        )
+      })
+
+      const secondPreview = await config.api.query.preview({
+        name: "Test Query",
+        datasourceId: datasource._id!,
+        fields: {
+          json: { name: { $eq: name } },
+          extra: {
+            collection,
+            actionType: "findOne",
+          },
+        },
+        schema: firstPreview.schema,
+        queryVerb: "read",
+        parameters: [],
+        transformer: "return data",
+        readable: true,
+      })
+
+      expect(secondPreview.schema).toEqual(
+        expect.objectContaining({
+          field: { type: "array", name: "field" },
+        })
+      )
+    })
+
     it("should generate a nested schema based on all of the nested items", async () => {
       const name = generator.guid()
       const item = {

packages/server/src/api/routes/tests/queries/rest.spec.ts

@@ -92,6 +92,61 @@ describe("rest", () => {
     expect(cached.rows[0].name).toEqual("one")
   })
 
+  it("should update schema when structure changes from JSON to array", async () => {
+    const datasource = await config.api.datasource.create({
+      name: generator.guid(),
+      type: "test",
+      source: SourceName.REST,
+      config: {},
+    })
+
+    nock("http://www.example.com")
+      .get("/")
+      .reply(200, [{ obj: {}, id: "1" }])
+
+    const firstResponse = await config.api.query.preview({
+      datasourceId: datasource._id!,
+      name: "test query",
+      parameters: [],
+      queryVerb: "read",
+      transformer: "",
+      schema: {},
+      readable: true,
+      fields: {
+        path: "www.example.com",
+      },
+    })
+
+    expect(firstResponse.schema).toEqual({
+      obj: { type: "json", name: "obj" },
+      id: { type: "string", name: "id" },
+    })
+
+    nock.cleanAll()
+
+    nock("http://www.example.com")
+      .get("/")
+      .reply(200, [{ obj: [], id: "1" }])
+
+    const secondResponse = await config.api.query.preview({
+      datasourceId: datasource._id!,
+      name: "test query",
+      parameters: [],
+      queryVerb: "read",
+      transformer: "",
+      schema: firstResponse.schema,
+      readable: true,
+      fields: {
+        path: "www.example.com",
+      },
+    })
+
+    expect(secondResponse.schema).toEqual({
+      obj: { type: "array", name: "obj" },
+      id: { type: "string", name: "id" },
+    })
+  })
+
   it("should parse global and query level header mappings", async () => {
     const datasource = await config.api.datasource.create({
       name: generator.guid(),

packages/server/src/api/routes/tests/row.spec.ts

@@ -1,6 +1,11 @@
-import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import {
+  DatabaseName,
+  getDatasource,
+  knexClient,
+} from "../../../integrations/tests/utils"
 
 import tk from "timekeeper"
+import emitter from "../../../../src/events"
 import { outputProcessing } from "../../../utilities/rowProcessor"
 import * as setup from "./utilities"
 import { context, InternalTable, tenancy } from "@budibase/backend-core"
@@ -24,13 +29,38 @@ import {
   StaticQuotaName,
   Table,
   TableSourceType,
+  UpdatedRowEventEmitter,
+  TableSchema,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import _, { merge } from "lodash"
 import * as uuid from "uuid"
+import { Knex } from "knex"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
+interface WaitOptions {
+  name: string
+  matchFn?: (event: any) => boolean
+}
+async function waitForEvent(
+  opts: WaitOptions,
+  callback: () => Promise<void>
+): Promise<any> {
+  const p = new Promise((resolve: any) => {
+    const listener = (event: any) => {
+      if (opts.matchFn && !opts.matchFn(event)) {
+        return
+      }
+      resolve(event)
+      emitter.off(opts.name, listener)
+    }
+    emitter.on(opts.name, listener)
+  })
+
+  await callback()
+  return await p
+}
 
 describe.each([
   ["internal", undefined],
@@ -40,17 +70,21 @@ describe.each([
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
+  const isMSSQL = providerType === DatabaseName.SQL_SERVER
   const config = setup.getConfig()
 
   let table: Table
   let datasource: Datasource | undefined
+  let client: Knex | undefined
 
   beforeAll(async () => {
     await config.init()
     if (dsProvider) {
+      const rawDatasource = await dsProvider
       datasource = await config.createDatasource({
-        datasource: await dsProvider,
+        datasource: rawDatasource,
       })
+      client = await knexClient(rawDatasource)
     }
   })
 
@@ -64,6 +98,23 @@ describe.each([
     // the table name they're writing to.
     ...overrides: Partial<Omit<SaveTableRequest, "name">>[]
   ): SaveTableRequest {
+    const defaultSchema: TableSchema = {
+      id: {
+        type: FieldType.AUTO,
+        name: "id",
+        autocolumn: true,
+        constraints: {
+          presence: true,
+        },
+      },
+    }
+
+    for (const override of overrides) {
+      if (override.primary) {
+        delete defaultSchema.id
+      }
+    }
+
     const req: SaveTableRequest = {
       name: uuid.v4().substring(0, 10),
       type: "table",
@@ -72,16 +123,7 @@ describe.each([
         : TableSourceType.INTERNAL,
       sourceId: datasource ? datasource._id! : INTERNAL_TABLE_SOURCE_ID,
       primary: ["id"],
-      schema: {
+      schema: defaultSchema,
-        id: {
-          type: FieldType.AUTO,
-          name: "id",
-          autocolumn: true,
-          constraints: {
-            presence: true,
-          },
-        },
-      },
     }
     return merge(req, ...overrides)
   }
@@ -273,13 +315,13 @@ describe.each([
         // as quickly as possible.
         await Promise.all(
           sequence.map(async () => {
-            const attempts = 20
+            const attempts = 30
             for (let attempt = 0; attempt < attempts; attempt++) {
               try {
                 await config.api.row.save(table._id!, {})
                 return
               } catch (e) {
-                await new Promise(r => setTimeout(r, Math.random() * 15))
+                await new Promise(r => setTimeout(r, Math.random() * 50))
               }
             }
             throw new Error(`Failed to create row after ${attempts} attempts`)
@@ -564,6 +606,35 @@ describe.each([
       expect(res.name).toEqual("Updated Name")
       await assertRowUsage(rowUsage)
     })
+
+    !isInternal &&
+      it("can update a row on an external table with a primary key", async () => {
+        const tableName = uuid.v4().substring(0, 10)
+        await client!.schema.createTable(tableName, table => {
+          table.increments("id").primary()
+          table.string("name")
+        })
+
+        const res = await config.api.datasource.fetchSchema({
+          datasourceId: datasource!._id!,
+        })
+        const table = res.datasource.entities![tableName]
+
+        const row = await config.api.row.save(table._id!, {
+          id: 1,
+          name: "Row 1",
+        })
+
+        const updatedRow = await config.api.row.save(table._id!, {
+          _id: row._id!,
+          name: "Row 1 Updated",
+        })
+
+        expect(updatedRow.name).toEqual("Row 1 Updated")
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows).toHaveLength(1)
+      })
   })
 
   describe("patch", () => {
@@ -608,6 +679,32 @@ describe.each([
       await assertRowUsage(rowUsage)
     })
 
+    it("should update only the fields that are supplied and emit the correct oldRow", async () => {
+      let beforeRow = await config.api.row.save(table._id!, {
+        name: "test",
+        description: "test",
+      })
+      const opts = {
+        name: "row:update",
+        matchFn: (event: UpdatedRowEventEmitter) =>
+          event.row._id === beforeRow._id,
+      }
+      const event = await waitForEvent(opts, async () => {
+        await config.api.row.patch(table._id!, {
+          _id: beforeRow._id!,
+          _rev: beforeRow._rev!,
+          tableId: table._id!,
+          name: "Updated Name",
+        })
+      })
+
+      expect(event.oldRow).toBeDefined()
+      expect(event.oldRow.name).toEqual("test")
+      expect(event.row.name).toEqual("Updated Name")
+      expect(event.oldRow.description).toEqual(beforeRow.description)
+      expect(event.row.description).toEqual(beforeRow.description)
+    })
+
     it("should throw an error when given improper types", async () => {
       const existing = await config.api.row.save(table._id!, {})
       const rowUsage = await getRowUsage()
@@ -699,7 +796,8 @@ describe.each([
     })
 
     !isInternal &&
-      // TODO: SQL is having issues creating composite keys
+      // MSSQL needs a setting called IDENTITY_INSERT to be set to ON to allow writing
+      // to identity columns. This is not something Budibase does currently.
       providerType !== DatabaseName.SQL_SERVER &&
       it("should support updating fields that are part of a composite key", async () => {
         const tableRequest = saveTableRequest({
@@ -852,32 +950,21 @@ describe.each([
       await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
     })
 
-    it("Should ignore malformed/invalid delete requests", async () => {
+    it.each([{ not: "valid" }, { rows: 123 }, "invalid"])(
-      const rowUsage = await getRowUsage()
+      "Should ignore malformed/invalid delete request: %s",
+      async (request: any) => {
+        const rowUsage = await getRowUsage()
 
-      await config.api.row.delete(table._id!, { not: "valid" } as any, {
+        await config.api.row.delete(table._id!, request, {
-        status: 400,
+          status: 400,
-        body: {
+          body: {
-          message: "Invalid delete rows request",
+            message: "Invalid delete rows request",
-        },
+          },
-      })
+        })
 
-      await config.api.row.delete(table._id!, { rows: 123 } as any, {
+        await assertRowUsage(rowUsage)
-        status: 400,
+      }
-        body: {
+    )
-          message: "Invalid delete rows request",
-        },
-      })
-
-      await config.api.row.delete(table._id!, "invalid" as any, {
-        status: 400,
-        body: {
-          message: "Invalid delete rows request",
-        },
-      })
-
-      await assertRowUsage(rowUsage)
-    })
   })
 
   describe("bulkImport", () => {
@@ -911,6 +998,236 @@ describe.each([
         row = await config.api.row.save(table._id!, {})
         expect(row.autoId).toEqual(3)
       })
+
+    it("should be able to bulkImport rows", async () => {
+      const table = await config.api.table.save(
+        saveTableRequest({
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+            },
+            description: {
+              type: FieldType.STRING,
+              name: "description",
+            },
+          },
+        })
+      )
+
+      const rowUsage = await getRowUsage()
+
+      await config.api.row.bulkImport(table._id!, {
+        rows: [
+          {
+            name: "Row 1",
+            description: "Row 1 description",
+          },
+          {
+            name: "Row 2",
+            description: "Row 2 description",
+          },
+        ],
+      })
+
+      const rows = await config.api.row.fetch(table._id!)
+      expect(rows.length).toEqual(2)
+
+      rows.sort((a, b) => a.name.localeCompare(b.name))
+      expect(rows[0].name).toEqual("Row 1")
+      expect(rows[0].description).toEqual("Row 1 description")
+      expect(rows[1].name).toEqual("Row 2")
+      expect(rows[1].description).toEqual("Row 2 description")
+
+      await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage)
+    })
+
+    // Upserting isn't yet supported in MSSQL, see:
+    //   https://github.com/knex/knex/pull/6050
+    !isMSSQL &&
+      it("should be able to update existing rows with bulkImport", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            primary: ["userId"],
+            schema: {
+              userId: {
+                type: FieldType.NUMBER,
+                name: "userId",
+                constraints: {
+                  presence: true,
+                },
+              },
+              name: {
+                type: FieldType.STRING,
+                name: "name",
+              },
+              description: {
+                type: FieldType.STRING,
+                name: "description",
+              },
+            },
+          })
+        )
+
+        const row1 = await config.api.row.save(table._id!, {
+          userId: 1,
+          name: "Row 1",
+          description: "Row 1 description",
+        })
+
+        const row2 = await config.api.row.save(table._id!, {
+          userId: 2,
+          name: "Row 2",
+          description: "Row 2 description",
+        })
+
+        await config.api.row.bulkImport(table._id!, {
+          identifierFields: ["userId"],
+          rows: [
+            {
+              userId: row1.userId,
+              name: "Row 1 updated",
+              description: "Row 1 description updated",
+            },
+            {
+              userId: row2.userId,
+              name: "Row 2 updated",
+              description: "Row 2 description updated",
+            },
+            {
+              userId: 3,
+              name: "Row 3",
+              description: "Row 3 description",
+            },
+          ],
+        })
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows.length).toEqual(3)
+
+        rows.sort((a, b) => a.name.localeCompare(b.name))
+        expect(rows[0].name).toEqual("Row 1 updated")
+        expect(rows[0].description).toEqual("Row 1 description updated")
+        expect(rows[1].name).toEqual("Row 2 updated")
+        expect(rows[1].description).toEqual("Row 2 description updated")
+        expect(rows[2].name).toEqual("Row 3")
+        expect(rows[2].description).toEqual("Row 3 description")
+      })
+
+    // Upserting isn't yet supported in MSSQL, see:
+    //   https://github.com/knex/knex/pull/6050
+    !isMSSQL &&
+      !isInternal &&
+      it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
+        const tableName = uuid.v4()
+        await client?.schema.createTable(tableName, table => {
+          table.integer("companyId")
+          table.integer("userId")
+          table.string("name")
+          table.string("description")
+          table.primary(["companyId", "userId"])
+        })
+
+        const resp = await config.api.datasource.fetchSchema({
+          datasourceId: datasource!._id!,
+        })
+        const table = resp.datasource.entities![tableName]
+
+        const row1 = await config.api.row.save(table._id!, {
+          companyId: 1,
+          userId: 1,
+          name: "Row 1",
+          description: "Row 1 description",
+        })
+
+        const row2 = await config.api.row.save(table._id!, {
+          companyId: 1,
+          userId: 2,
+          name: "Row 2",
+          description: "Row 2 description",
+        })
+
+        await config.api.row.bulkImport(table._id!, {
+          identifierFields: ["companyId", "userId"],
+          rows: [
+            {
+              companyId: 1,
+              userId: row1.userId,
+              name: "Row 1 updated",
+              description: "Row 1 description updated",
+            },
+            {
+              companyId: 1,
+              userId: row2.userId,
+              name: "Row 2 updated",
+              description: "Row 2 description updated",
+            },
+            {
+              companyId: 1,
+              userId: 3,
+              name: "Row 3",
+              description: "Row 3 description",
+            },
+          ],
+        })
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows.length).toEqual(3)
+
+        rows.sort((a, b) => a.name.localeCompare(b.name))
+        expect(rows[0].name).toEqual("Row 1 updated")
+        expect(rows[0].description).toEqual("Row 1 description updated")
+        expect(rows[1].name).toEqual("Row 2 updated")
+        expect(rows[1].description).toEqual("Row 2 description updated")
+        expect(rows[2].name).toEqual("Row 3")
+        expect(rows[2].description).toEqual("Row 3 description")
+      })
+
+    // Upserting isn't yet supported in MSSQL, see:
+    //   https://github.com/knex/knex/pull/6050
+    !isMSSQL &&
+      !isInternal &&
+      it("should be able to update existing rows an autoID primary key", async () => {
+        const tableName = uuid.v4()
+        await client!.schema.createTable(tableName, table => {
+          table.increments("userId").primary()
+          table.string("name")
+        })
+
+        const resp = await config.api.datasource.fetchSchema({
+          datasourceId: datasource!._id!,
+        })
+        const table = resp.datasource.entities![tableName]
+
+        const row1 = await config.api.row.save(table._id!, {
+          name: "Clare",
+        })
+
+        const row2 = await config.api.row.save(table._id!, {
+          name: "Jeff",
+        })
+
+        await config.api.row.bulkImport(table._id!, {
+          identifierFields: ["userId"],
+          rows: [
+            {
+              userId: row1.userId,
+              name: "Clare updated",
+            },
+            {
+              userId: row2.userId,
+              name: "Jeff updated",
+            },
+          ],
+        })
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows.length).toEqual(2)
+
+        rows.sort((a, b) => a.name.localeCompare(b.name))
+        expect(rows[0].name).toEqual("Clare updated")
+        expect(rows[1].name).toEqual("Jeff updated")
+      })
   })
 
   describe("enrich", () => {

packages/server/src/api/routes/tests/table.spec.ts

@@ -276,6 +276,34 @@ describe.each([
         })
     })
 
+    isInternal &&
+      it("shouldn't allow duplicate column names", async () => {
+        const saveTableRequest: SaveTableRequest = {
+          ...basicTable(),
+        }
+        saveTableRequest.schema["Type"] = {
+          type: FieldType.STRING,
+          name: "Type",
+        }
+        await config.api.table.save(saveTableRequest, {
+          status: 400,
+          body: {
+            message:
+              'Column(s) "type" are duplicated - check for other columns with these name (case in-sensitive)',
+          },
+        })
+        saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" }
+        saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" }
+
+        await config.api.table.save(saveTableRequest, {
+          status: 400,
+          body: {
+            message:
+              'Column(s) "type, foo" are duplicated - check for other columns with these name (case in-sensitive)',
+          },
+        })
+      })
+
     it("should add a new column for an internal DB table", async () => {
       const saveTableRequest: SaveTableRequest = {
         ...basicTable(),

packages/server/src/api/routes/tests/utilities/TestFunctions.ts

@@ -158,15 +158,16 @@ export const getDB = () => {
   return context.getAppDB()
 }
 
-export const testAutomation = async (config: any, automation: any) => {
+export const testAutomation = async (
+  config: any,
+  automation: any,
+  triggerInputs: any
+) => {
   return runRequest(automation.appId, async () => {
     return await config.request
       .post(`/api/automations/${automation._id}/test`)
       .send({
-        row: {
+        ...triggerInputs,
-          name: "Test",
-          description: "TEST",
-        },
       })
       .set(config.defaultHeaders())
       .expect("Content-Type", /json/)

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -7,6 +7,7 @@ import {
   INTERNAL_TABLE_SOURCE_ID,
   PermissionLevel,
   QuotaUsageType,
+  Row,
   SaveTableRequest,
   SearchFilterOperator,
   SortOrder,
@@ -17,6 +18,7 @@ import {
   UpdateViewRequest,
   ViewUIFieldMetadata,
   ViewV2,
+  SearchResponse,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -25,17 +27,21 @@ import { quotas } from "@budibase/pro"
 import { db, roles } from "@budibase/backend-core"
 
 describe.each([
-  ["internal", undefined],
+  ["lucene", undefined],
+  ["sqs", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-])("/v2/views (%s)", (_, dsProvider) => {
+])("/v2/views (%s)", (name, dsProvider) => {
   const config = setup.getConfig()
-  const isInternal = !dsProvider
+  const isSqs = name === "sqs"
+  const isLucene = name === "lucene"
+  const isInternal = isSqs || isLucene
 
   let table: Table
   let datasource: Datasource
+  let envCleanup: (() => void) | undefined
 
   function saveTableRequest(
     ...overrides: Partial<Omit<SaveTableRequest, "name">>[]
@@ -82,6 +88,9 @@ describe.each([
   }
 
   beforeAll(async () => {
+    if (isSqs) {
+      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+    }
     await config.init()
 
     if (dsProvider) {
@@ -94,6 +103,9 @@ describe.each([
 
   afterAll(async () => {
     setup.afterAll()
+    if (envCleanup) {
+      envCleanup()
+    }
   })
 
   beforeEach(() => {
@@ -1252,12 +1264,13 @@ describe.each([
           paginate: true,
           limit: 4,
           query: {},
+          countRows: true,
         })
         expect(page1).toEqual({
           rows: expect.arrayContaining(rows.slice(0, 4)),
-          totalRows: isInternal ? 10 : undefined,
           hasNextPage: true,
           bookmark: expect.anything(),
+          totalRows: 10,
         })
 
         const page2 = await config.api.viewV2.search(view.id, {
@@ -1265,12 +1278,13 @@ describe.each([
           limit: 4,
           bookmark: page1.bookmark,
           query: {},
+          countRows: true,
         })
         expect(page2).toEqual({
           rows: expect.arrayContaining(rows.slice(4, 8)),
-          totalRows: isInternal ? 10 : undefined,
           hasNextPage: true,
           bookmark: expect.anything(),
+          totalRows: 10,
         })
 
         const page3 = await config.api.viewV2.search(view.id, {
@@ -1278,13 +1292,17 @@ describe.each([
           limit: 4,
           bookmark: page2.bookmark,
           query: {},
+          countRows: true,
         })
-        expect(page3).toEqual({
+        const expectation: SearchResponse<Row> = {
           rows: expect.arrayContaining(rows.slice(8)),
-          totalRows: isInternal ? 10 : undefined,
           hasNextPage: false,
-          bookmark: expect.anything(),
+          totalRows: 10,
-        })
+        }
+        if (isLucene) {
+          expectation.bookmark = expect.anything()
+        }
+        expect(page3).toEqual(expectation)
       })
 
       const sortTestOptions: [

packages/server/src/api/routes/utils/validators.ts

@@ -109,6 +109,7 @@ export function internalSearchValidator() {
       sortOrder: OPTIONAL_STRING,
       sortType: OPTIONAL_STRING,
       paginate: Joi.boolean(),
+      countRows: Joi.boolean(),
       bookmark: Joi.alternatives()
         .try(OPTIONAL_STRING, OPTIONAL_NUMBER)
         .optional(),

packages/server/src/appMigrations/appMigrationMetadata.ts

@@ -23,16 +23,15 @@ const getCacheKey = (appId: string) => `appmigrations_${env.VERSION}_${appId}`
 export async function getAppMigrationVersion(appId: string): Promise<string> {
   const cacheKey = getCacheKey(appId)
 
-  let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
+  let version: string | undefined = await cache.get(cacheKey)
 
   // returned cached version if we found one
-  if (metadata?.version) {
+  if (version) {
-    return metadata.version
+    return version
   }
 
-  let version
   try {
-    metadata = await getFromDB(appId)
+    const metadata = await getFromDB(appId)
     version = metadata.version || ""
   } catch (err: any) {
     if (err.status !== 404) {

packages/server/src/automations/loopUtils.ts

@@ -0,0 +1,36 @@
+import * as automationUtils from "./automationUtils"
+
+type ObjValue = {
+  [key: string]: string | ObjValue
+}
+
+export function replaceFakeBindings(
+  originalStepInput: Record<string, any>,
+  loopStepNumber: number
+) {
+  for (const [key, value] of Object.entries(originalStepInput)) {
+    originalStepInput[key] = replaceBindingsRecursive(value, loopStepNumber)
+  }
+  return originalStepInput
+}
+
+function replaceBindingsRecursive(
+  value: string | ObjValue,
+  loopStepNumber: number
+) {
+  if (typeof value === "object") {
+    for (const [innerKey, innerValue] of Object.entries(value)) {
+      if (typeof innerValue === "string") {
+        value[innerKey] = automationUtils.substituteLoopStep(
+          innerValue,
+          `steps.${loopStepNumber}`
+        )
+      } else if (typeof innerValue === "object") {
+        value[innerKey] = replaceBindingsRecursive(innerValue, loopStepNumber)
+      }
+    }
+  } else if (typeof value === "string") {
+    value = automationUtils.substituteLoopStep(value, `steps.${loopStepNumber}`)
+  }
+  return value
+}

packages/server/src/automations/steps/filter.ts

@@ -73,7 +73,12 @@ export async function run({ inputs }: AutomationStepInput) {
   try {
     let { field, condition, value } = inputs
     // coerce types so that we can use them
-    if (!isNaN(value) && !isNaN(field)) {
+    if (
+      !isNaN(value) &&
+      !isNaN(field) &&
+      typeof field !== "boolean" &&
+      typeof value !== "boolean"
+    ) {
       value = parseFloat(value)
       field = parseFloat(field)
     } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {

packages/server/src/automations/steps/queryRows.ts

@@ -14,14 +14,10 @@ import {
   EmptyFilterOption,
   SearchFilters,
   Table,
+  SortOrder,
 } from "@budibase/types"
 import { db as dbCore } from "@budibase/backend-core"
 
-enum SortOrder {
-  ASCENDING = "ascending",
-  DESCENDING = "descending",
-}
-
 const SortOrderPretty = {
   [SortOrder.ASCENDING]: "Ascending",
   [SortOrder.DESCENDING]: "Descending",

packages/server/src/automations/triggerInfo/rowUpdated.ts

@@ -28,10 +28,17 @@ export const definition: AutomationTriggerSchema = {
     },
     outputs: {
       properties: {
-        row: {
+        oldRow: {
           type: AutomationIOType.OBJECT,
           customType: AutomationCustomIOType.ROW,
           description: "The row that was updated",
+          title: "Old Row",
+        },
+        row: {
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
+          description: "The row before it was updated",
+          title: "Row",
         },
         id: {
           type: AutomationIOType.STRING,

packages/server/src/automations/triggers.ts

@@ -14,6 +14,7 @@ import {
   AutomationData,
   AutomationJob,
   AutomationEventType,
+  UpdatedRowEventEmitter,
 } from "@budibase/types"
 import { executeInThread } from "../threads/automation"
 
@@ -71,13 +72,16 @@ async function queueRelevantRowAutomations(
   })
 }
 
-emitter.on(AutomationEventType.ROW_SAVE, async function (event) {
+emitter.on(
-  /* istanbul ignore next */
+  AutomationEventType.ROW_SAVE,
-  if (!event || !event.row || !event.row.tableId) {
+  async function (event: UpdatedRowEventEmitter) {
-    return
+    /* istanbul ignore next */
+    if (!event || !event.row || !event.row.tableId) {
+      return
+    }
+    await queueRelevantRowAutomations(event, AutomationEventType.ROW_SAVE)
   }
-  await queueRelevantRowAutomations(event, AutomationEventType.ROW_SAVE)
+)
-})
 
 emitter.on(AutomationEventType.ROW_UPDATE, async function (event) {
   /* istanbul ignore next */

packages/server/src/constants/index.ts

@@ -70,11 +70,6 @@ export enum DatasourceAuthTypes {
   GOOGLE = "google",
 }
 
-export enum SortDirection {
-  ASCENDING = "ASCENDING",
-  DESCENDING = "DESCENDING",
-}
-
 export const USERS_TABLE_SCHEMA: Table = {
   _id: "ta_users",
   type: "table",

packages/server/src/events/BudibaseEmitter.ts

@@ -13,8 +13,14 @@ import { Table, Row } from "@budibase/types"
  * This is specifically quite important for template strings used in automations.
  */
 class BudibaseEmitter extends EventEmitter {
-  emitRow(eventName: string, appId: string, row: Row, table?: Table) {
+  emitRow(
-    rowEmission({ emitter: this, eventName, appId, row, table })
+    eventName: string,
+    appId: string,
+    row: Row,
+    table?: Table,
+    oldRow?: Row
+  ) {
+    rowEmission({ emitter: this, eventName, appId, row, table, oldRow })
   }
 
   emitTable(eventName: string, appId: string, table?: Table) {

packages/server/src/events/utils.ts

@@ -7,6 +7,7 @@ type BBEventOpts = {
   appId: string
   table?: Table
   row?: Row
+  oldRow?: Row
   metadata?: any
 }
 
@@ -18,6 +19,7 @@ type BBEvent = {
   appId: string
   tableId?: string
   row?: Row
+  oldRow?: Row
   table?: BBEventTable
   id?: string
   revision?: string
@@ -31,9 +33,11 @@ export function rowEmission({
   row,
   table,
   metadata,
+  oldRow,
 }: BBEventOpts) {
   let event: BBEvent = {
     row,
+    oldRow,
     appId,
     tableId: row?.tableId,
   }

packages/server/src/integration-test/mysql.spec.ts

@@ -1,19 +1,12 @@
-import fetch from "node-fetch"
-import {
-  generateMakeRequest,
-  MakeRequestResponse,
-} from "../api/routes/public/tests/utils"
 import * as setup from "../api/routes/tests/utilities"
 import { Datasource, FieldType } from "@budibase/types"
 import {
   DatabaseName,
   getDatasource,
-  rawQuery,
+  knexClient,
 } from "../integrations/tests/utils"
 import { generator } from "@budibase/backend-core/tests"
-import { tableForDatasource } from "../../src/tests/utilities/structures"
+import { Knex } from "knex"
-// @ts-ignore
-fetch.mockSearch()
 
 function uniqueTableName(length?: number): string {
   return generator
@@ -24,129 +17,74 @@ function uniqueTableName(length?: number): string {
 
 const config = setup.getConfig()!
 
-jest.mock("../websockets", () => ({
-  clientAppSocket: jest.fn(),
-  gridAppSocket: jest.fn(),
-  initialise: jest.fn(),
-  builderSocket: {
-    emitTableUpdate: jest.fn(),
-    emitTableDeletion: jest.fn(),
-    emitDatasourceUpdate: jest.fn(),
-    emitDatasourceDeletion: jest.fn(),
-    emitScreenUpdate: jest.fn(),
-    emitAppMetadataUpdate: jest.fn(),
-    emitAppPublish: jest.fn(),
-  },
-}))
-
 describe("mysql integrations", () => {
-  let makeRequest: MakeRequestResponse,
+  let datasource: Datasource
-    rawDatasource: Datasource,
+  let client: Knex
-    datasource: Datasource
 
   beforeAll(async () => {
     await config.init()
-    const apiKey = await config.generateApiKey()
+    const rawDatasource = await getDatasource(DatabaseName.MYSQL)
-
-    makeRequest = generateMakeRequest(apiKey, true)
-
-    rawDatasource = await getDatasource(DatabaseName.MYSQL)
     datasource = await config.api.datasource.create(rawDatasource)
+    client = await knexClient(rawDatasource)
   })
 
   afterAll(config.end)
 
-  it("validate table schema", async () => {
-    // Creating a table so that `entities` is populated.
-    await config.api.table.save(tableForDatasource(datasource))
-
-    const res = await makeRequest("get", `/api/datasources/${datasource._id}`)
-
-    expect(res.status).toBe(200)
-    expect(res.body).toEqual({
-      config: {
-        database: expect.any(String),
-        host: datasource.config!.host,
-        password: "--secret-value--",
-        port: datasource.config!.port,
-        user: "root",
-      },
-      plus: true,
-      source: "MYSQL",
-      type: "datasource_plus",
-      isSQL: true,
-      _id: expect.any(String),
-      _rev: expect.any(String),
-      createdAt: expect.any(String),
-      updatedAt: expect.any(String),
-      entities: expect.any(Object),
-    })
-  })
-
   describe("Integration compatibility with mysql search_path", () => {
-    let datasource: Datasource, rawDatasource: Datasource
+    let datasource: Datasource
+    let rawDatasource: Datasource
+    let client: Knex
     const database = generator.guid()
     const database2 = generator.guid()
 
     beforeAll(async () => {
       rawDatasource = await getDatasource(DatabaseName.MYSQL)
+      client = await knexClient(rawDatasource)
 
-      await rawQuery(rawDatasource, `CREATE DATABASE \`${database}\`;`)
+      await client.raw(`CREATE DATABASE \`${database}\`;`)
-      await rawQuery(rawDatasource, `CREATE DATABASE \`${database2}\`;`)
+      await client.raw(`CREATE DATABASE \`${database2}\`;`)
 
-      const pathConfig: any = {
+      rawDatasource.config!.database = database
-        ...rawDatasource,
+      datasource = await config.api.datasource.create(rawDatasource)
-        config: {
-          ...rawDatasource.config!,
-          database,
-        },
-      }
-      datasource = await config.api.datasource.create(pathConfig)
     })
 
     afterAll(async () => {
-      await rawQuery(rawDatasource, `DROP DATABASE \`${database}\`;`)
+      await client.raw(`DROP DATABASE \`${database}\`;`)
-      await rawQuery(rawDatasource, `DROP DATABASE \`${database2}\`;`)
+      await client.raw(`DROP DATABASE \`${database2}\`;`)
     })
 
     it("discovers tables from any schema in search path", async () => {
-      await rawQuery(
+      await client.schema.createTable(`${database}.table1`, table => {
-        rawDatasource,
+        table.increments("id1").primary()
-        `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);`
-      )
-      const response = await makeRequest("post", "/api/datasources/info", {
-        datasource: datasource,
       })
-      expect(response.status).toBe(200)
+      const res = await config.api.datasource.info(datasource)
-      expect(response.body.tableNames).toBeDefined()
+      expect(res.tableNames).toBeDefined()
-      expect(response.body.tableNames).toEqual(
+      expect(res.tableNames).toEqual(expect.arrayContaining(["table1"]))
-        expect.arrayContaining(["table1"])
-      )
     })
 
     it("does not mix columns from different tables", async () => {
       const repeated_table_name = "table_same_name"
-      await rawQuery(
+      await client.schema.createTable(
-        rawDatasource,
+        `${database}.${repeated_table_name}`,
-        `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+        table => {
-      )
+          table.increments("id").primary()
-      await rawQuery(
+          table.string("val1")
-        rawDatasource,
-        `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
-      )
-      const response = await makeRequest(
-        "post",
-        `/api/datasources/${datasource._id}/schema`,
-        {
-          tablesFilter: [repeated_table_name],
         }
       )
-      expect(response.status).toBe(200)
+      await client.schema.createTable(
-      expect(
+        `${database2}.${repeated_table_name}`,
-        response.body.datasource.entities[repeated_table_name].schema
+        table => {
-      ).toBeDefined()
+          table.increments("id2").primary()
-      const schema =
+          table.string("val2")
-        response.body.datasource.entities[repeated_table_name].schema
+        }
+      )
+
+      const res = await config.api.datasource.fetchSchema({
+        datasourceId: datasource._id!,
+        tablesFilter: [repeated_table_name],
+      })
+      expect(res.datasource.entities![repeated_table_name].schema).toBeDefined()
+      const schema = res.datasource.entities![repeated_table_name].schema
       expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
     })
   })
@@ -159,28 +97,27 @@ describe("mysql integrations", () => {
     })
 
     afterEach(async () => {
-      await rawQuery(rawDatasource, `DROP TABLE IF EXISTS \`${tableName}\``)
+      await client.schema.dropTableIfExists(tableName)
     })
 
     it("recognises enum columns as options", async () => {
       const enumColumnName = "status"
 
-      const createTableQuery = `
+      await client.schema.createTable(tableName, table => {
-        CREATE TABLE \`${tableName}\` (
+        table.increments("order_id").primary()
-          \`order_id\` INT AUTO_INCREMENT PRIMARY KEY,
+        table.string("customer_name", 100).notNullable()
-          \`customer_name\` VARCHAR(100) NOT NULL,
+        table.enum(
-          \`${enumColumnName}\` ENUM('pending', 'processing', 'shipped', 'delivered', 'cancelled')
+          enumColumnName,
-        );
+          ["pending", "processing", "shipped", "delivered", "cancelled"],
-      `
+          { useNative: true, enumName: `${tableName}_${enumColumnName}` }
+        )
+      })
 
-      await rawQuery(rawDatasource, createTableQuery)
+      const res = await config.api.datasource.fetchSchema({
+        datasourceId: datasource._id!,
+      })
 
-      const response = await makeRequest(
+      const table = res.datasource.entities![tableName]
-        "post",
-        `/api/datasources/${datasource._id}/schema`
-      )
-
-      const table = response.body.datasource.entities[tableName]
 
       expect(table).toBeDefined()
       expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)

packages/server/src/integration-test/postgres.spec.ts

@@ -1,9 +1,3 @@
-import fetch from "node-fetch"
-import {
-  generateMakeRequest,
-  MakeRequestResponse,
-} from "../api/routes/public/tests/utils"
-
 import * as setup from "../api/routes/tests/utilities"
 import { Datasource, FieldType } from "@budibase/types"
 import _ from "lodash"
@@ -11,29 +5,21 @@ import { generator } from "@budibase/backend-core/tests"
 import {
   DatabaseName,
   getDatasource,
-  rawQuery,
+  knexClient,
 } from "../integrations/tests/utils"
-
+import { Knex } from "knex"
-// @ts-ignore
-fetch.mockSearch()
 
 const config = setup.getConfig()!
 
-jest.mock("../websockets")
-
 describe("postgres integrations", () => {
-  let makeRequest: MakeRequestResponse,
+  let datasource: Datasource
-    rawDatasource: Datasource,
+  let client: Knex
-    datasource: Datasource
 
   beforeAll(async () => {
     await config.init()
-    const apiKey = await config.generateApiKey()
+    const rawDatasource = await getDatasource(DatabaseName.POSTGRES)
-
-    makeRequest = generateMakeRequest(apiKey, true)
-
-    rawDatasource = await getDatasource(DatabaseName.POSTGRES)
     datasource = await config.api.datasource.create(rawDatasource)
+    client = await knexClient(rawDatasource)
   })
 
   afterAll(config.end)
@@ -46,11 +32,13 @@ describe("postgres integrations", () => {
     })
 
     afterEach(async () => {
-      await rawQuery(rawDatasource, `DROP TABLE IF EXISTS "${tableName}"`)
+      await client.schema.dropTableIfExists(tableName)
     })
 
     it("recognises when a table has no primary key", async () => {
-      await rawQuery(rawDatasource, `CREATE TABLE "${tableName}" (id SERIAL)`)
+      await client.schema.createTable(tableName, table => {
+        table.increments("id", { primaryKey: false })
+      })
 
       const response = await config.api.datasource.fetchSchema({
         datasourceId: datasource._id!,
@@ -62,10 +50,9 @@ describe("postgres integrations", () => {
     })
 
     it("recognises when a table is using a reserved column name", async () => {
-      await rawQuery(
+      await client.schema.createTable(tableName, table => {
-        rawDatasource,
+        table.increments("_id").primary()
-        `CREATE TABLE "${tableName}" (_id SERIAL PRIMARY KEY) `
+      })
-      )
 
       const response = await config.api.datasource.fetchSchema({
         datasourceId: datasource._id!,
@@ -81,20 +68,15 @@ describe("postgres integrations", () => {
         .guid()
         .replaceAll("-", "")
         .substring(0, 6)}`
-      const enumColumnName = "status"
 
-      await rawQuery(
+      await client.schema.createTable(tableName, table => {
-        rawDatasource,
+        table.increments("order_id").primary()
-        `
+        table.string("customer_name").notNullable()
-        CREATE TYPE order_status AS ENUM ('pending', 'processing', 'shipped', 'delivered', 'cancelled');
+        table.enum("status", ["pending", "processing", "shipped"], {
-        
+          useNative: true,
-        CREATE TABLE ${tableName} (
+          enumName: `${tableName}_status`,
-          order_id SERIAL PRIMARY KEY,
+        })
-          customer_name VARCHAR(100) NOT NULL,
+      })
-          ${enumColumnName} order_status
-        );
-      `
-      )
 
       const response = await config.api.datasource.fetchSchema({
         datasourceId: datasource._id!,
@@ -103,69 +85,70 @@ describe("postgres integrations", () => {
       const table = response.datasource.entities?.[tableName]
 
       expect(table).toBeDefined()
-      expect(table?.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
+      expect(table?.schema["status"].type).toEqual(FieldType.OPTIONS)
     })
   })
 
   describe("Integration compatibility with postgres search_path", () => {
-    let rawDatasource: Datasource,
+    let datasource: Datasource
-      datasource: Datasource,
+    let client: Knex
-      schema1: string,
+    let schema1: string
-      schema2: string
+    let schema2: string
 
     beforeEach(async () => {
       schema1 = generator.guid().replaceAll("-", "")
       schema2 = generator.guid().replaceAll("-", "")
 
-      rawDatasource = await getDatasource(DatabaseName.POSTGRES)
+      const rawDatasource = await getDatasource(DatabaseName.POSTGRES)
-      const dbConfig = rawDatasource.config!
+      client = await knexClient(rawDatasource)
 
-      await rawQuery(rawDatasource, `CREATE SCHEMA "${schema1}";`)
+      await client.schema.createSchema(schema1)
-      await rawQuery(rawDatasource, `CREATE SCHEMA "${schema2}";`)
+      await client.schema.createSchema(schema2)
 
-      const pathConfig: any = {
+      rawDatasource.config!.schema = `${schema1}, ${schema2}`
-        ...rawDatasource,
+
-        config: {
+      client = await knexClient(rawDatasource)
-          ...dbConfig,
+      datasource = await config.api.datasource.create(rawDatasource)
-          schema: `${schema1}, ${schema2}`,
-        },
-      }
-      datasource = await config.api.datasource.create(pathConfig)
     })
 
     afterEach(async () => {
-      await rawQuery(rawDatasource, `DROP SCHEMA "${schema1}" CASCADE;`)
+      await client.schema.dropSchema(schema1, true)
-      await rawQuery(rawDatasource, `DROP SCHEMA "${schema2}" CASCADE;`)
+      await client.schema.dropSchema(schema2, true)
     })
 
     it("discovers tables from any schema in search path", async () => {
-      await rawQuery(
+      await client.schema.createTable(`${schema1}.table1`, table => {
-        rawDatasource,
+        table.increments("id1").primary()
-        `CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);`
-      )
-      await rawQuery(
-        rawDatasource,
-        `CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);`
-      )
-      const response = await makeRequest("post", "/api/datasources/info", {
-        datasource: datasource,
       })
-      expect(response.status).toBe(200)
+
-      expect(response.body.tableNames).toBeDefined()
+      await client.schema.createTable(`${schema2}.table2`, table => {
-      expect(response.body.tableNames).toEqual(
+        table.increments("id2").primary()
+      })
+
+      const response = await config.api.datasource.info(datasource)
+      expect(response.tableNames).toBeDefined()
+      expect(response.tableNames).toEqual(
         expect.arrayContaining(["table1", "table2"])
       )
     })
 
     it("does not mix columns from different tables", async () => {
       const repeated_table_name = "table_same_name"
-      await rawQuery(
+
-        rawDatasource,
+      await client.schema.createTable(
-        `CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+        `${schema1}.${repeated_table_name}`,
+        table => {
+          table.increments("id").primary()
+          table.string("val1")
+        }
       )
-      await rawQuery(
+
-        rawDatasource,
+      await client.schema.createTable(
-        `CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
+        `${schema2}.${repeated_table_name}`,
+        table => {
+          table.increments("id2").primary()
+          table.string("val2")
+        }
       )
 
       const response = await config.api.datasource.fetchSchema({
@@ -182,15 +165,11 @@ describe("postgres integrations", () => {
 
   describe("check custom column types", () => {
     beforeAll(async () => {
-      await rawQuery(
+      await client.schema.createTable("binaryTable", table => {
-        rawDatasource,
+        table.binary("id").primary()
-        `CREATE TABLE binaryTable (
+        table.string("column1")
-          id BYTEA PRIMARY KEY,
+        table.integer("column2")
-          column1 TEXT,
+      })
-          column2 INT
-        );
-      `
-      )
     })
 
     it("should handle binary columns", async () => {
@@ -198,7 +177,7 @@ describe("postgres integrations", () => {
         datasourceId: datasource._id!,
       })
       expect(response.datasource.entities).toBeDefined()
-      const table = response.datasource.entities?.["binarytable"]
+      const table = response.datasource.entities?.["binaryTable"]
       expect(table).toBeDefined()
       expect(table?.schema.id.externalType).toBe("bytea")
       const row = await config.api.row.save(table?._id!, {
@@ -214,14 +193,10 @@ describe("postgres integrations", () => {
 
   describe("check fetching null/not null table", () => {
     beforeAll(async () => {
-      await rawQuery(
+      await client.schema.createTable("nullableTable", table => {
-        rawDatasource,
+        table.increments("order_id").primary()
-        `CREATE TABLE nullableTable (
+        table.integer("order_number").notNullable()
-          order_id SERIAL PRIMARY KEY,
+      })
-          order_number INT NOT NULL
-        );
-      `
-      )
     })
 
     it("should be able to change the table to allow nullable and refetch this", async () => {
@@ -230,25 +205,24 @@ describe("postgres integrations", () => {
       })
       const entities = response.datasource.entities
       expect(entities).toBeDefined()
-      const nullableTable = entities?.["nullabletable"]
+      const nullableTable = entities?.["nullableTable"]
       expect(nullableTable).toBeDefined()
       expect(
         nullableTable?.schema["order_number"].constraints?.presence
       ).toEqual(true)
+
       // need to perform these calls raw to the DB so that the external state of the DB differs to what Budibase
       // is aware of - therefore we can try to fetch and make sure BB updates correctly
-      await rawQuery(
+      await client.schema.alterTable("nullableTable", table => {
-        rawDatasource,
+        table.setNullable("order_number")
-        `ALTER TABLE nullableTable
+      })
-             ALTER COLUMN order_number DROP NOT NULL;
+
-        `
-      )
       const responseAfter = await config.api.datasource.fetchSchema({
         datasourceId: datasource._id!,
       })
       const entitiesAfter = responseAfter.datasource.entities
       expect(entitiesAfter).toBeDefined()
-      const nullableTableAfter = entitiesAfter?.["nullabletable"]
+      const nullableTableAfter = entitiesAfter?.["nullableTable"]
       expect(nullableTableAfter).toBeDefined()
       expect(
         nullableTableAfter?.schema["order_number"].constraints?.presence

packages/server/src/integrations/base/query.ts

@@ -22,6 +22,9 @@ export async function makeExternalQuery(
   ) {
     throw new Error("Entity ID and table metadata do not align")
   }
+  if (!datasource) {
+    throw new Error("No datasource provided for external query")
+  }
   datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function

packages/server/src/integrations/googlesheets.ts

@@ -566,7 +566,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
           query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id
         }
       }
-      let filtered = dataFilters.runQuery(rows, query.filters)
+      let filtered = dataFilters.runQuery(rows, query.filters || {})
       if (hasFilters && query.paginate) {
         filtered = filtered.slice(offset, offset + limit)
       }