Merge remote-tracking branch 'refs/remotes/origin/feat/update-automation-tests' into feat/update-automation-tests

.github/workflows/budibase_ci.yml

@@ -281,6 +281,7 @@ jobs:
 
   check-lockfile:
     runs-on: ubuntu-latest
+    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

.github/workflows/readme-openapi.yml

@@ -0,0 +1,28 @@
+name: ReadMe GitHub Action 🦉
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  rdme-openapi:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+          cache: yarn
+      - run: yarn --frozen-lockfile
+
+      - name: update specs
+        run: cd packages/server && yarn specs
+
+      - name: Run `openapi` command
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=6728a74f5918b50036c61841

globalSetup.ts

@@ -6,6 +6,26 @@ import {
 import { ContainerInfo } from "dockerode"
 import path from "path"
 import lockfile from "proper-lockfile"
+import { execSync } from "child_process"
+
+interface DockerContext {
+  Name: string
+  Description: string
+  DockerEndpoint: string
+  ContextType: string
+  Error: string
+}
+
+function getCurrentDockerContext(): DockerContext {
+  const out = execSync("docker context ls --format json")
+  for (const line of out.toString().split("\n")) {
+    const parsed = JSON.parse(line)
+    if (parsed.Current) {
+      return parsed as DockerContext
+    }
+  }
+  throw new Error("No current Docker context")
+}
 
 async function getBudibaseContainers() {
   const client = await getContainerRuntimeClient()
@@ -27,6 +47,14 @@ async function killContainers(containers: ContainerInfo[]) {
 }
 
 export default async function setup() {
+  // For whatever reason, testcontainers doesn't always use the correct current
+  // docker context. This bit of code forces the issue by finding the current
+  // context and setting it as the DOCKER_HOST environment
+  if (!process.env.DOCKER_HOST) {
+    const dockerContext = getCurrentDockerContext()
+    process.env.DOCKER_HOST = dockerContext.DockerEndpoint
+  }
+
   const lockPath = path.resolve(__dirname, "globalSetup.ts")
   // If you run multiple tests at the same time, it's possible for the CouchDB
   // shared container to get started multiple times despite having an

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.12",
+  "version": "3.2.18",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

package.json

@@ -9,6 +9,7 @@
     "@types/node": "20.10.0",
     "@types/proper-lockfile": "^4.1.4",
     "@typescript-eslint/parser": "6.9.0",
+    "cross-spawn": "7.0.6",
     "depcheck": "^1.4.7",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.14.0",
@@ -29,8 +30,7 @@
     "svelte-eslint-parser": "^0.33.1",
     "typescript": "5.5.2",
     "typescript-eslint": "^7.3.1",
-    "yargs": "^17.7.2",
+    "yargs": "^17.7.2"
-    "cross-spawn": "7.0.6"
   },
   "scripts": {
     "get-past-client-version": "node scripts/getPastClientVersion.js",
@@ -76,7 +76,6 @@
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 -t budibase/couchdb:v3.3.3-sqs-v2.1.1 --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
-    "release:helm": "node scripts/releaseHelmChart",
     "env:multi:enable": "lerna run --stream env:multi:enable",
     "env:multi:disable": "lerna run --stream env:multi:disable",
     "env:selfhost:enable": "lerna run --stream env:selfhost:enable",

packages/backend-core/src/events/identification.ts

@@ -121,7 +121,7 @@ const identifyInstallationGroup = async (
 
 const identifyTenantGroup = async (
   tenantId: string,
-  account: Account | undefined,
+  hosting: Hosting,
   timestamp?: string | number
 ): Promise<void> => {
   const id = await getEventTenantId(tenantId)
@@ -129,26 +129,12 @@ const identifyTenantGroup = async (
   const installationId = await getInstallationId()
   const environment = getDeploymentEnvironment()
 
-  let hosting: Hosting
-  let profession: string | undefined
-  let companySize: string | undefined
-
-  if (account) {
-    profession = account.profession
-    companySize = account.size
-    hosting = account.hosting
-  } else {
-    hosting = getHostingFromEnv()
-  }
-
   const group: TenantGroup = {
     id,
     type,
     hosting,
     environment,
     installationId,
-    profession,
-    companySize,
   }
 
   await identifyGroup(group, timestamp)

packages/backend-core/src/features/features.ts

@@ -266,12 +266,14 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 // new flag, add it here and use the `fetch` and `get` functions to access it.
 // All of the machinery in this file is to make sure that flags have their
 // default values set correctly and their types flow through the system.
-export const flags = new FlagSet({
+const flagsConfig: Record<FeatureFlag, Flag<any>> = {
   [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
   [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
   [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
-})
+  [FeatureFlag.USE_ZOD_VALIDATOR]: Flag.boolean(env.isDev()),
+}
+export const flags = new FlagSet(flagsConfig)
 
 type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
 export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

packages/backend-core/src/sql/sql.ts

@@ -18,6 +18,7 @@ import {
   BasicOperator,
   BBReferenceFieldMetadata,
   CalculationType,
+  EnrichedQueryJson,
   FieldSchema,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
@@ -27,7 +28,6 @@ import {
   LogicalOperator,
   Operation,
   prefixed,
-  QueryJson,
   QueryOptions,
   RangeOperator,
   RelationshipsJson,
@@ -134,18 +134,18 @@ const allowEmptyRelationships: Record<SearchFilterKey, boolean> = {
 
 class InternalBuilder {
   private readonly client: SqlClient
-  private readonly query: QueryJson
+  private readonly query: EnrichedQueryJson
   private readonly splitter: dataFilters.ColumnSplitter
   private readonly knex: Knex
 
-  constructor(client: SqlClient, knex: Knex, query: QueryJson) {
+  constructor(client: SqlClient, knex: Knex, query: EnrichedQueryJson) {
     this.client = client
     this.query = query
     this.knex = knex
 
     this.splitter = new dataFilters.ColumnSplitter([this.table], {
       aliases: this.query.tableAliases,
-      columnPrefix: this.query.meta.columnPrefix,
+      columnPrefix: this.query.meta?.columnPrefix,
     })
   }
 
@@ -167,7 +167,7 @@ class InternalBuilder {
   }
 
   get table(): Table {
-    return this.query.meta.table
+    return this.query.table
   }
 
   get knexClient(): Knex.Client {
@@ -273,8 +273,7 @@ class InternalBuilder {
   }
 
   private isFullSelectStatementRequired(): boolean {
-    const { meta } = this.query
+    for (let column of Object.values(this.table.schema)) {
-    for (let column of Object.values(meta.table.schema)) {
       if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
         return true
       } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
@@ -285,14 +284,14 @@ class InternalBuilder {
   }
 
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
-    const { meta, endpoint, resource } = this.query
+    const { table, resource } = this.query
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
     }
 
-    const alias = this.getTableName(endpoint.entityId)
+    const alias = this.getTableName(table)
-    const schema = meta.table.schema
+    const schema = this.table.schema
     if (!this.isFullSelectStatementRequired()) {
       return [this.knex.raw("??", [`${alias}.*`])]
     }
@@ -497,9 +496,8 @@ class InternalBuilder {
     filterKey: string,
     whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder
   ): Knex.QueryBuilder {
-    const { relationships, endpoint, tableAliases: aliases } = this.query
+    const { relationships, schema, tableAliases: aliases, table } = this.query
-    const tableName = endpoint.entityId
+    const fromAlias = aliases?.[table.name] || table.name
-    const fromAlias = aliases?.[tableName] || tableName
     const matches = (value: string) =>
       filterKey.match(new RegExp(`^${value}\\.`))
     if (!relationships) {
@@ -539,7 +537,7 @@ class InternalBuilder {
             aliases?.[manyToMany.through] || relationship.through
           let throughTable = this.tableNameWithSchema(manyToMany.through, {
             alias: throughAlias,
-            schema: endpoint.schema,
+            schema,
           })
           subQuery = subQuery
             // add a join through the junction table
@@ -1012,28 +1010,10 @@ class InternalBuilder {
     return isSqs(this.table)
   }
 
-  getTableName(tableOrName?: Table | string): string {
+  getTableName(table?: Table): string {
-    let table: Table
+    if (!table) {
-    if (typeof tableOrName === "string") {
-      const name = tableOrName
-      if (this.query.table?.name === name) {
-        table = this.query.table
-      } else if (this.query.meta.table?.name === name) {
-        table = this.query.meta.table
-      } else if (!this.query.meta.tables?.[name]) {
-        // This can legitimately happen in custom queries, where the user is
-        // querying against a table that may not have been imported into
-        // Budibase.
-        return name
-      } else {
-        table = this.query.meta.tables[name]
-      }
-    } else if (tableOrName) {
-      table = tableOrName
-    } else {
       table = this.table
     }
-
     let name = table.name
     if (isSqs(table) && table._id) {
       // SQS uses the table ID rather than the table name
@@ -1191,8 +1171,9 @@ class InternalBuilder {
     return withSchema
   }
 
-  private buildJsonField(field: string): string {
+  private buildJsonField(table: Table, field: string): [string, Knex.Raw] {
     const parts = field.split(".")
+    let baseName = parts[parts.length - 1]
     let unaliased: string
 
     let tableField: string
@@ -1205,10 +1186,19 @@ class InternalBuilder {
       tableField = unaliased
     }
 
-    const separator = this.client === SqlClient.ORACLE ? " VALUE " : ","
+    if (this.query.meta?.columnPrefix) {
-    return this.knex
+      baseName = baseName.replace(this.query.meta.columnPrefix, "")
-      .raw(`?${separator}??`, [unaliased, this.rawQuotedIdentifier(tableField)])
+    }
-      .toString()
+
+    let identifier = this.rawQuotedIdentifier(tableField)
+    // Internal tables have special _id, _rev, createdAt, and updatedAt fields
+    // that do not appear in the schema, meaning schema could actually be
+    // undefined.
+    const schema: FieldSchema | undefined = table.schema[baseName]
+    if (schema && schema.type === FieldType.BIGINT) {
+      identifier = this.castIntToString(identifier)
+    }
+    return [unaliased, identifier]
   }
 
   maxFunctionParameters() {
@@ -1234,7 +1224,7 @@ class InternalBuilder {
   ): Knex.QueryBuilder {
     const sqlClient = this.client
     const knex = this.knex
-    const { resource, tableAliases: aliases, endpoint, meta } = this.query
+    const { resource, tableAliases: aliases, schema, tables } = this.query
     const fields = resource?.fields || []
     for (let relationship of relationships) {
       const {
@@ -1249,13 +1239,16 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
-      const relatedTable = meta.tables?.[toTable]
+      const relatedTable = tables[toTable]
+      if (!relatedTable) {
+        throw new Error(`related table "${toTable}" not found in datasource`)
+      }
       const toAlias = aliases?.[toTable] || toTable,
         fromAlias = aliases?.[fromTable] || fromTable,
         throughAlias = (throughTable && aliases?.[throughTable]) || throughTable
       let toTableWithSchema = this.tableNameWithSchema(toTable, {
         alias: toAlias,
-        schema: endpoint.schema,
+        schema,
       })
       const requiredFields = [
         ...(relatedTable?.primary || []),
@@ -1271,8 +1264,14 @@ class InternalBuilder {
         0,
         Math.floor(this.maxFunctionParameters() / 2)
       )
-      const fieldList: string = relationshipFields
+      const fieldList = relationshipFields.map(field =>
-        .map(field => this.buildJsonField(field))
+        this.buildJsonField(relatedTable, field)
+      )
+      const fieldListFormatted = fieldList
+        .map(f => {
+          const separator = this.client === SqlClient.ORACLE ? " VALUE " : ","
+          return this.knex.raw(`?${separator}??`, [f[0], f[1]]).toString()
+        })
         .join(",")
       // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
       // it reduces the result set rather than limiting how much data it filters over
@@ -1293,7 +1292,7 @@ class InternalBuilder {
       if (isManyToMany) {
         let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
           alias: throughAlias,
-          schema: endpoint.schema,
+          schema,
         })
         subQuery = subQuery.join(throughTableWithSchema, function () {
           this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
@@ -1320,35 +1319,42 @@ class InternalBuilder {
           // need to check the junction table document is to the right column, this is just for SQS
           subQuery = this.addJoinFieldCheck(subQuery, relationship)
           wrapperQuery = standardWrap(
-            this.knex.raw(`json_group_array(json_object(${fieldList}))`)
+            this.knex.raw(
+              `json_group_array(json_object(${fieldListFormatted}))`
+            )
           )
           break
         case SqlClient.POSTGRES:
           wrapperQuery = standardWrap(
-            this.knex.raw(`json_agg(json_build_object(${fieldList}))`)
+            this.knex.raw(`json_agg(json_build_object(${fieldListFormatted}))`)
           )
           break
         case SqlClient.MARIADB:
           // can't use the standard wrap due to correlated sub-query limitations in MariaDB
           wrapperQuery = subQuery.select(
             knex.raw(
-              `json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})`
+              `json_arrayagg(json_object(${fieldListFormatted}) LIMIT ${getRelationshipLimit()})`
             )
           )
           break
         case SqlClient.MY_SQL:
         case SqlClient.ORACLE:
           wrapperQuery = standardWrap(
-            this.knex.raw(`json_arrayagg(json_object(${fieldList}))`)
+            this.knex.raw(`json_arrayagg(json_object(${fieldListFormatted}))`)
           )
           break
         case SqlClient.MS_SQL: {
           const comparatorQuery = knex
-            .select(`${fromAlias}.*`)
+            .select(`*`)
             // @ts-ignore - from alias syntax not TS supported
             .from({
               [fromAlias]: subQuery
-                .select(`${toAlias}.*`)
+                .select(
+                  fieldList.map(f => {
+                    // @ts-expect-error raw is fine here, knex types are wrong
+                    return knex.ref(f[1]).as(f[0])
+                  })
+                )
                 .limit(getRelationshipLimit()),
             })
 
@@ -1377,8 +1383,7 @@ class InternalBuilder {
       toPrimary?: string
     }[]
   ): Knex.QueryBuilder {
-    const { tableAliases: aliases, endpoint } = this.query
+    const { tableAliases: aliases, schema } = this.query
-    const schema = endpoint.schema
     const toTable = tables.to,
       fromTable = tables.from,
       throughTable = tables.through
@@ -1429,16 +1434,16 @@ class InternalBuilder {
   }
 
   qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder {
-    let alias = this.query.tableAliases?.[this.query.endpoint.entityId]
+    let alias = this.query.tableAliases?.[this.query.table.name]
     if (opts?.alias === false) {
       alias = undefined
     } else if (typeof opts?.alias === "string") {
       alias = opts.alias
     }
     return this.knex(
-      this.tableNameWithSchema(this.query.endpoint.entityId, {
+      this.tableNameWithSchema(this.query.table.name, {
         alias,
-        schema: this.query.endpoint.schema,
+        schema: this.query.schema,
       })
     )
   }
@@ -1455,9 +1460,7 @@ class InternalBuilder {
     if (this.client === SqlClient.ORACLE) {
       // Oracle doesn't seem to automatically insert nulls
       // if we don't specify them, so we need to do that here
-      for (const [column, schema] of Object.entries(
+      for (const [column, schema] of Object.entries(this.query.table.schema)) {
-        this.query.meta.table.schema
-      )) {
         if (
           schema.constraints?.presence === true ||
           schema.type === FieldType.FORMULA ||
@@ -1534,11 +1537,9 @@ class InternalBuilder {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    let { endpoint, filters, paginate, relationships } = this.query
+    let { operation, filters, paginate, relationships, table } = this.query
     const { limits } = opts
-    const counting = endpoint.operation === Operation.COUNT
 
-    const tableName = endpoint.entityId
     // start building the query
     let query = this.qualifiedKnex()
     // handle pagination
@@ -1557,7 +1558,7 @@ class InternalBuilder {
       foundLimit = paginate.limit
     }
     // counting should not sort, limit or offset
-    if (!counting) {
+    if (operation !== Operation.COUNT) {
       // add the found limit if supplied
       if (foundLimit != null) {
         query = query.limit(foundLimit)
@@ -1569,7 +1570,7 @@ class InternalBuilder {
     }
 
     const aggregations = this.query.resource?.aggregations || []
-    if (counting) {
+    if (operation === Operation.COUNT) {
       query = this.addDistinctCount(query)
     } else if (aggregations.length > 0) {
       query = this.addAggregations(query, aggregations)
@@ -1578,7 +1579,7 @@ class InternalBuilder {
     }
 
     // have to add after as well (this breaks MS-SQL)
-    if (!counting) {
+    if (operation !== Operation.COUNT) {
       query = this.addSorting(query)
     }
 
@@ -1586,9 +1587,7 @@ class InternalBuilder {
 
     // handle relationships with a CTE for all others
     if (relationships?.length && aggregations.length === 0) {
-      const mainTable =
+      const mainTable = this.query.tableAliases?.[table.name] || table.name
-        this.query.tableAliases?.[this.query.endpoint.entityId] ||
-        this.query.endpoint.entityId
       const cte = this.addSorting(
         this.knex
           .with("paginated", query)
@@ -1598,7 +1597,7 @@ class InternalBuilder {
           })
       )
       // add JSON aggregations attached to the CTE
-      return this.addJsonRelationships(cte, tableName, relationships)
+      return this.addJsonRelationships(cte, table.name, relationships)
     }
 
     return query
@@ -1661,7 +1660,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes.
    * @return the query ready to be passed to the driver.
    */
-  _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
+  _query(
+    json: EnrichedQueryJson,
+    opts: QueryOptions = {}
+  ): SqlQuery | SqlQuery[] {
     const sqlClient = this.getSqlClient()
     const config: Knex.Config = {
       client: this.getBaseSqlClient(),
@@ -1711,34 +1713,30 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return this.convertToNative(query, opts)
   }
 
-  async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
+  async getReturningRow(queryFn: QueryFunction, json: EnrichedQueryJson) {
     if (!json.extra || !json.extra.idFilter) {
       return {}
     }
     const input = this._query({
-      endpoint: {
+      operation: Operation.READ,
-        ...json.endpoint,
+      datasource: json.datasource,
-        operation: Operation.READ,
+      schema: json.schema,
-      },
+      table: json.table,
-      resource: {
+      tables: json.tables,
-        fields: [],
+      resource: { fields: [] },
-      },
       filters: json.extra?.idFilter,
-      paginate: {
+      paginate: { limit: 1 },
-        limit: 1,
-      },
-      meta: json.meta,
     })
     return queryFn(input, Operation.READ)
   }
 
   // when creating if an ID has been inserted need to make sure
   // the id filter is enriched with it before trying to retrieve the row
-  checkLookupKeys(id: any, json: QueryJson) {
+  checkLookupKeys(id: any, json: EnrichedQueryJson) {
-    if (!id || !json.meta.table || !json.meta.table.primary) {
+    if (!id || !json.table.primary) {
       return json
     }
-    const primaryKey = json.meta.table.primary?.[0]
+    const primaryKey = json.table.primary[0]
     json.extra = {
       idFilter: {
         equal: {
@@ -1751,7 +1749,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 
   // this function recreates the returning functionality of postgres
   async queryWithReturning(
-    json: QueryJson,
+    json: EnrichedQueryJson,
     queryFn: QueryFunction,
     processFn: Function = (result: any) => result
   ) {

packages/backend-core/src/sql/sqlTable.ts

@@ -3,13 +3,13 @@ import {
   FieldType,
   NumberFieldMetadata,
   Operation,
-  QueryJson,
   RelationshipType,
   RenameColumn,
   SqlQuery,
   Table,
   TableSourceType,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import { breakExternalTableId, getNativeSql } from "./utils"
 import { helpers, utils } from "@budibase/shared-core"
@@ -25,7 +25,7 @@ function generateSchema(
   schema: CreateTableBuilder,
   table: Table,
   tables: Record<string, Table>,
-  oldTable: null | Table = null,
+  oldTable?: Table,
   renamed?: RenameColumn
 ) {
   let primaryKeys = table && table.primary ? table.primary : []
@@ -55,7 +55,7 @@ function generateSchema(
   )
   for (let [key, column] of Object.entries(table.schema)) {
     // skip things that are already correct
-    const oldColumn = oldTable ? oldTable.schema[key] : null
+    const oldColumn = oldTable?.schema[key]
     if (
       (oldColumn && oldColumn.type) ||
       columnTypeSet.includes(key) ||
@@ -199,8 +199,8 @@ function buildUpdateTable(
   knex: SchemaBuilder,
   table: Table,
   tables: Record<string, Table>,
-  oldTable: Table,
+  oldTable?: Table,
-  renamed: RenameColumn
+  renamed?: RenameColumn
 ): SchemaBuilder {
   return knex.alterTable(table.name, schema => {
     generateSchema(schema, table, tables, oldTable, renamed)
@@ -238,19 +238,18 @@ class SqlTableQueryBuilder {
    * @param json the input JSON structure from which an SQL query will be built.
    * @return the operation that was found in the JSON.
    */
-  _operation(json: QueryJson): Operation {
+  _operation(json: EnrichedQueryJson): Operation {
-    return json.endpoint.operation
+    return json.operation
   }
 
-  _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] {
+  _tableQuery(json: EnrichedQueryJson): SqlQuery | SqlQuery[] {
     let client = knex({ client: this.sqlClient }).schema
-    let schemaName = json?.endpoint?.schema
+    if (json?.schema) {
-    if (schemaName) {
+      client = client.withSchema(json.schema)
-      client = client.withSchema(schemaName)
     }
 
     let query: Knex.SchemaBuilder
-    if (!json.table || !json.meta || !json.meta.tables) {
+    if (!json.table || !json.tables) {
       throw new Error("Cannot execute without table being specified")
     }
     if (json.table.sourceType === TableSourceType.INTERNAL) {
@@ -259,17 +258,17 @@ class SqlTableQueryBuilder {
 
     switch (this._operation(json)) {
       case Operation.CREATE_TABLE:
-        query = buildCreateTable(client, json.table, json.meta.tables)
+        query = buildCreateTable(client, json.table, json.tables)
         break
       case Operation.UPDATE_TABLE:
-        if (!json.meta || !json.meta.table) {
+        if (!json.table) {
           throw new Error("Must specify old table for update")
         }
         // renameColumn does not work for MySQL, so return a raw query
-        if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
+        if (this.sqlClient === SqlClient.MY_SQL && json.meta?.renamed) {
           const updatedColumn = json.meta.renamed.updated
-          const tableName = schemaName
+          const tableName = json?.schema
-            ? `\`${schemaName}\`.\`${json.table.name}\``
+            ? `\`${json.schema}\`.\`${json.table.name}\``
             : `\`${json.table.name}\``
           return {
             sql: `alter table ${tableName} rename column \`${json.meta.renamed.old}\` to \`${updatedColumn}\`;`,
@@ -280,18 +279,18 @@ class SqlTableQueryBuilder {
         query = buildUpdateTable(
           client,
           json.table,
-          json.meta.tables,
+          json.tables,
-          json.meta.table,
+          json.meta?.oldTable,
-          json.meta.renamed!
+          json.meta?.renamed
         )
 
         // renameColumn for SQL Server returns a parameterised `sp_rename` query,
         // which is not supported by SQL Server and gives a syntax error.
-        if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) {
+        if (this.sqlClient === SqlClient.MS_SQL && json.meta?.renamed) {
           const oldColumn = json.meta.renamed.old
           const updatedColumn = json.meta.renamed.updated
-          const tableName = schemaName
+          const tableName = json?.schema
-            ? `${schemaName}.${json.table.name}`
+            ? `${json.schema}.${json.table.name}`
             : `${json.table.name}`
           const sql = getNativeSql(query)
           if (Array.isArray(sql)) {

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -22,7 +22,6 @@ export function price(): PurchasedPrice {
     currency: "usd",
     duration: PriceDuration.MONTHLY,
     priceId: "price_123",
-    dayPasses: undefined,
     isPerUser: true,
   }
 }
@@ -50,11 +49,6 @@ export function quotas(): Quotas {
           value: 1,
           triggers: [],
         },
-        dayPasses: {
-          name: "Queries",
-          value: 1,
-          triggers: [],
-        },
         budibaseAICredits: {
           name: "Budibase AI Credits",
           value: 1,

packages/backend-core/tests/core/utilities/structures/quotas.ts

@@ -15,7 +15,6 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
     monthly: {
       "01-2023": {
         automations: 0,
-        dayPasses: 0,
         queries: 0,
         budibaseAICredits: 0,
         triggers: {},
@@ -45,14 +44,12 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
       },
       "02-2023": {
         automations: 0,
-        dayPasses: 0,
         queries: 0,
         budibaseAICredits: 0,
         triggers: {},
       },
       current: {
         automations: 0,
-        dayPasses: 0,
         queries: 0,
         budibaseAICredits: 0,
         triggers: {},

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -25,7 +25,7 @@ function getTestcontainers(): ContainerInfo[] {
   // We use --format json to make sure the output is nice and machine-readable,
   // and we use --no-trunc so that the command returns full container IDs so we
   // can filter on them correctly.
-  return execSync("docker ps --format json --no-trunc")
+  return execSync("docker ps --all --format json --no-trunc")
     .toString()
     .split("\n")
     .filter(x => x.length > 0)
@@ -37,6 +37,10 @@ function getTestcontainers(): ContainerInfo[] {
     )
 }
 
+function removeContainer(container: ContainerInfo) {
+  execSync(`docker rm ${container.ID}`)
+}
+
 export function getContainerByImage(image: string) {
   const containers = getTestcontainers().filter(x => x.Image.startsWith(image))
   if (containers.length > 1) {
@@ -49,6 +53,10 @@ export function getContainerByImage(image: string) {
   return containers[0]
 }
 
+function getContainerByName(name: string) {
+  return getTestcontainers().find(x => x.Names === name)
+}
+
 export function getContainerById(id: string) {
   return getTestcontainers().find(x => x.ID === id)
 }
@@ -70,7 +78,34 @@ export function getExposedV4Port(container: ContainerInfo, port: number) {
   return getExposedV4Ports(container).find(x => x.container === port)?.host
 }
 
+interface DockerContext {
+  Name: string
+  Description: string
+  DockerEndpoint: string
+  ContextType: string
+  Error: string
+}
+
+function getCurrentDockerContext(): DockerContext {
+  const out = execSync("docker context ls --format json")
+  for (const line of out.toString().split("\n")) {
+    const parsed = JSON.parse(line)
+    if (parsed.Current) {
+      return parsed as DockerContext
+    }
+  }
+  throw new Error("No current Docker context")
+}
+
 export function setupEnv(...envs: any[]) {
+  // For whatever reason, testcontainers doesn't always use the correct current
+  // docker context. This bit of code forces the issue by finding the current
+  // context and setting it as the DOCKER_HOST environment
+  if (!process.env.DOCKER_HOST) {
+    const dockerContext = getCurrentDockerContext()
+    process.env.DOCKER_HOST = dockerContext.DockerEndpoint
+  }
+
   // We start couchdb in globalSetup.ts, in the root of the monorepo, so it
   // should be relatively safe to look for it by its image name.
   const couch = getContainerByImage("budibase/couchdb")
@@ -116,6 +151,16 @@ export async function startContainer(container: GenericContainer) {
     key = imageName.split("@")[0]
   }
   key = key.replace(/\//g, "-").replace(/:/g, "-")
+  const name = `${key}_testcontainer`
+
+  // If a container has died it hangs around and future attempts to start a
+  // container with the same name will fail. What we do here is if we find a
+  // matching container and it has exited, we remove it before carrying on. This
+  // removes the need to do this removal manually.
+  const existingContainer = getContainerByName(name)
+  if (existingContainer?.State === "exited") {
+    removeContainer(existingContainer)
+  }
 
   container = container
     .withReuse()

packages/bbui/package.json

@@ -4,27 +4,21 @@
   "version": "0.0.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
-  "module": "dist/bbui.es.js",
+  "module": "dist/bbui.mjs",
   "exports": {
     ".": {
-      "import": "./dist/bbui.es.js"
+      "import": "./dist/bbui.mjs"
     },
     "./package.json": "./package.json",
     "./spectrum-icons-rollup.js": "./src/spectrum-icons-rollup.js",
     "./spectrum-icons-vite.js": "./src/spectrum-icons-vite.js"
   },
   "scripts": {
-    "build": "rollup -c"
+    "build": "vite build"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^16.0.0",
+    "@sveltejs/vite-plugin-svelte": "1.4.0",
-    "@rollup/plugin-json": "^4.1.0",
+    "vite-plugin-css-injected-by-js": "3.5.2"
-    "@rollup/plugin-node-resolve": "^11.2.1",
-    "postcss": "^8.2.9",
-    "rollup": "^2.45.2",
-    "rollup-plugin-postcss": "^4.0.0",
-    "rollup-plugin-svelte": "^7.1.0",
-    "rollup-plugin-terser": "^7.0.2"
   },
   "keywords": [
     "svelte"
@@ -96,8 +90,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/string-templates",
+              "@budibase/string-templates"
-              "@budibase/shared-core"
             ],
             "target": "build"
           }

packages/bbui/rollup.config.js

@@ -1,32 +0,0 @@
-import svelte from "rollup-plugin-svelte"
-import resolve from "@rollup/plugin-node-resolve"
-import commonjs from "@rollup/plugin-commonjs"
-import json from "@rollup/plugin-json"
-import { terser } from "rollup-plugin-terser"
-import postcss from "rollup-plugin-postcss"
-
-export default {
-  input: "src/index.js",
-  output: {
-    sourcemap: true,
-    format: "esm",
-    file: "dist/bbui.es.js",
-  },
-  onwarn(warning, warn) {
-    // suppress eval warnings
-    if (warning.code === "EVAL") {
-      return
-    }
-    warn(warning)
-  },
-  plugins: [
-    resolve(),
-    commonjs(),
-    svelte({
-      emitCss: true,
-    }),
-    postcss(),
-    terser(),
-    json(),
-  ],
-}

packages/bbui/vite.config.js

@@ -0,0 +1,29 @@
+import { defineConfig } from "vite"
+import { svelte } from "@sveltejs/vite-plugin-svelte"
+import path from "path"
+import cssInjectedByJsPlugin from "vite-plugin-css-injected-by-js"
+
+export default defineConfig(({ mode }) => {
+  const isProduction = mode === "production"
+  return {
+    build: {
+      sourcemap: !isProduction,
+      lib: {
+        entry: "src/index.js",
+        formats: ["es"],
+      },
+    },
+    plugins: [
+      svelte({
+        emitCss: true,
+      }),
+      cssInjectedByJsPlugin(),
+    ],
+    resolve: {
+      alias: {
+        "@budibase/shared-core": path.resolve(__dirname, "../shared-core/src"),
+        "@budibase/types": path.resolve(__dirname, "../types/src"),
+      },
+    },
+  }
+})

packages/builder/.gitignore

@@ -5,4 +5,5 @@ package-lock.json
 release/
 dist/
 routify
-.routify/
+.routify/
+.rollup.cache

packages/builder/package.json

@@ -4,14 +4,14 @@
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
-    "svelte-check": "svelte-check --no-tsconfig",
+    "build": "routify -b && NODE_OPTIONS=\"--max_old_space_size=4096\" vite build --emptyOutDir",
-    "build": "yarn svelte-check && routify -b && vite build --emptyOutDir",
     "start": "routify -c rollup",
     "dev": "routify -c dev:vite",
     "dev:vite": "vite --host 0.0.0.0",
     "rollup": "rollup -c -w",
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "check:types": "yarn svelte-check"
   },
   "jest": {
     "globals": {
@@ -89,6 +89,7 @@
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-env": "^7.13.12",
     "@rollup/plugin-replace": "^5.0.3",
+    "@rollup/plugin-typescript": "8.3.0",
     "@roxi/routify": "2.18.12",
     "@sveltejs/vite-plugin-svelte": "1.4.0",
     "@testing-library/jest-dom": "6.4.2",

packages/builder/src/api.ts

@@ -8,7 +8,7 @@ import { get } from "svelte/store"
 import { auth, navigation } from "./stores/portal"
 
 export const API = createAPIClient({
-  attachHeaders: headers => {
+  attachHeaders: (headers: Record<string, string>) => {
     // Attach app ID header from store
     let appId = get(appStore).appId
     if (appId) {
@@ -16,13 +16,13 @@ export const API = createAPIClient({
     }
 
     // Add csrf token if authenticated
-    const user = get(auth).user
+    const user: any = get(auth).user
     if (user?.csrfToken) {
       headers["x-csrf-token"] = user.csrfToken
     }
   },
 
-  onError: error => {
+  onError: (error: any) => {
     const { url, message, status, method, handled } = error || {}
 
     // Log any errors that we haven't manually handled
@@ -45,14 +45,14 @@ export const API = createAPIClient({
       }
     }
   },
-  onMigrationDetected: appId => {
+  onMigrationDetected: (appId: string) => {
     const updatingUrl = `/builder/app/updating/${appId}`
 
     if (window.location.pathname === updatingUrl) {
       return
     }
 
-    get(navigation).goto(
+    get(navigation)?.goto(
       `${updatingUrl}?returnUrl=${encodeURIComponent(window.location.pathname)}`
     )
   },

packages/builder/src/components/portal/licensing/DayPassWarningModal.svelte

@@ -1,81 +0,0 @@
-<script>
-  import { Modal, ModalContent, Body, TooltipWrapper } from "@budibase/bbui"
-  import { licensing, auth, admin } from "stores/portal"
-
-  export let onDismiss = () => {}
-  export let onShow = () => {}
-
-  let dayPassModal
-
-  $: accountUrl = $admin.accountPortalUrl
-  $: upgradeUrl = `${accountUrl}/portal/upgrade`
-
-  $: daysRemaining = $licensing.quotaResetDaysRemaining
-  $: quotaResetDate = $licensing.quotaResetDate
-  $: dayPassesUsed =
-    $licensing.usageMetrics?.dayPasses > 100
-      ? 100
-      : $licensing.usageMetrics?.dayPasses
-  $: dayPassesTitle =
-    dayPassesUsed >= 100
-      ? "You have run out of Day Passes"
-      : "You are almost out of Day Passes"
-  $: dayPassesBody =
-    dayPassesUsed >= 100
-      ? "Upgrade your account to bring your apps back online."
-      : "Upgrade your account to prevent your apps from going offline."
-
-  export function show() {
-    dayPassModal.show()
-  }
-
-  export function hide() {
-    dayPassModal.hide()
-  }
-</script>
-
-<Modal bind:this={dayPassModal} on:show={onShow} on:hide={onDismiss}>
-  {#if $auth.user.accountPortalAccess}
-    <ModalContent
-      title={dayPassesTitle}
-      size="M"
-      confirmText="Upgrade"
-      onConfirm={() => {
-        window.location.href = upgradeUrl
-      }}
-    >
-      <Body>
-        You have used <span class="daypass_percent">{dayPassesUsed}%</span> of
-        your plans Day Passes with {daysRemaining} day{daysRemaining == 1
-          ? ""
-          : "s"} remaining.
-        <span class="tooltip">
-          <TooltipWrapper tooltip={quotaResetDate} size="S" />
-        </span>
-      </Body>
-      <Body>{dayPassesBody}</Body>
-    </ModalContent>
-  {:else}
-    <ModalContent title={dayPassesTitle} size="M" showCancelButton={false}>
-      <Body>
-        You have used <span class="daypass_percent">{dayPassesUsed}%</span> of
-        your plans Day Passes with {daysRemaining} day{daysRemaining == 1
-          ? ""
-          : "s"} remaining.
-        <span class="tooltip">
-          <TooltipWrapper tooltip={quotaResetDate} size="S" />
-        </span>
-      </Body>
-      <Body>Please contact your account holder to upgrade.</Body>
-    </ModalContent>
-  {/if}
-</Modal>
-
-<style>
-  .tooltip {
-    display: inline-block;
-  }
-  .tooltip :global(.icon-container) {
-    margin: 0px;
-  }
-</style>

packages/builder/src/components/portal/licensing/LicensingOverlays.svelte

@@ -1,7 +1,6 @@
 <script>
   import { licensing, auth, temporalStore } from "stores/portal"
   import { onMount } from "svelte"
-  import DayPassWarningModal from "./DayPassWarningModal.svelte"
   import PaymentFailedModal from "./PaymentFailedModal.svelte"
   import AccountDowngradedModal from "./AccountDowngradedModal.svelte"
   import { ExpiringKeys } from "./constants"
@@ -12,7 +11,6 @@
 
   let queuedBanners = []
   let queuedModals = []
-  let dayPassModal
   let paymentFailedModal
   let accountDowngradeModal
   let userLoaded = false
@@ -26,18 +24,6 @@
     }
 
     const dismissableModals = [
-      {
-        key: ExpiringKeys.LICENSING_DAYPASS_WARNING_MODAL,
-        criteria: () => {
-          return $licensing?.usageMetrics?.dayPasses >= 90
-        },
-        action: () => {
-          dayPassModal.show()
-        },
-        cache: () => {
-          defaultCacheFn(ExpiringKeys.LICENSING_DAYPASS_WARNING_MODAL)
-        },
-      },
       {
         key: ExpiringKeys.LICENSING_PAYMENT_FAILED,
         criteria: () => {
@@ -102,7 +88,6 @@
   })
 </script>
 
-<DayPassWarningModal bind:this={dayPassModal} onDismiss={showNextModal} />
 <PaymentFailedModal bind:this={paymentFailedModal} onDismiss={showNextModal} />
 <AccountDowngradedModal
   bind:this={accountDowngradeModal}

packages/builder/src/components/portal/licensing/constants.js

@@ -1,6 +1,4 @@
 export const ExpiringKeys = {
-  LICENSING_DAYPASS_WARNING_MODAL: "licensing_daypass_warning_90_modal",
-  LICENSING_DAYPASS_WARNING_BANNER: "licensing_daypass_warning_90_banner",
   LICENSING_PAYMENT_FAILED: "licensing_payment_failed",
   LICENSING_ACCOUNT_DOWNGRADED_MODAL: "licensing_account_downgraded_modal",
   LICENSING_APP_LIMIT_MODAL: "licensing_app_limit_modal",

packages/builder/src/components/portal/licensing/licensingBanners.js

@@ -84,45 +84,6 @@ const buildUsageInfoBanner = (
       }
 }
 
-const buildDayPassBanner = () => {
-  const appAuth = get(auth)
-  const appLicensing = get(licensing)
-  if (get(licensing)?.usageMetrics["dayPasses"] >= 100) {
-    return {
-      key: "max_dayPasses",
-      type: BANNER_TYPES.NEGATIVE,
-      criteria: () => {
-        return true
-      },
-      message: `Your apps are currently offline. You have exceeded your plans limit for Day Passes. ${
-        appAuth.user.accountPortalAccess
-          ? ""
-          : "Please contact your account holder to upgrade."
-      }`,
-      ...upgradeAction(),
-      showCloseButton: false,
-    }
-  }
-
-  return buildUsageInfoBanner(
-    "dayPasses",
-    "Day Passes",
-    ExpiringKeys.LICENSING_DAYPASS_WARNING_BANNER,
-    90,
-    `You have used ${
-      appLicensing?.usageMetrics["dayPasses"]
-    }% of your monthly usage of Day Passes with ${
-      appLicensing?.quotaResetDaysRemaining
-    } day${
-      get(licensing).quotaResetDaysRemaining == 1 ? "" : "s"
-    } remaining. All apps will be taken offline if this limit is reached. ${
-      appAuth.user.accountPortalAccess
-        ? ""
-        : "Please contact your account holder to upgrade."
-    }`
-  )
-}
-
 const buildPaymentFailedBanner = () => {
   return {
     key: "payment_Failed",
@@ -166,7 +127,6 @@ const buildUsersAboveLimitBanner = EXPIRY_KEY => {
 export const getBanners = () => {
   return [
     buildPaymentFailedBanner(),
-    buildDayPassBanner(ExpiringKeys.LICENSING_DAYPASS_WARNING_BANNER),
     buildUsageInfoBanner(
       "rows",
       "Rows",

packages/builder/src/constants/index.js

@@ -68,7 +68,6 @@ export const OnboardingType = {
 
 export const PlanModel = {
   PER_USER: "perUser",
-  DAY_PASS: "dayPass",
 }
 
 export const ChangelogURL = "https://docs.budibase.com/changelog"

packages/builder/src/index.d.ts

@@ -0,0 +1,8 @@
+declare module "api" {
+  const API: {
+    getPlugins: () => Promise<any>
+    createPlugin: (plugin: object) => Promise<any>
+    uploadPlugin: (plugin: FormData) => Promise<any>
+    deletePlugin: (id: string) => Promise<void>
+  }
+}

packages/builder/src/pages/builder/apps/index.svelte

@@ -136,7 +136,7 @@
             </Body>
           </Layout>
           <Divider />
-          {#if $licensing.usageMetrics?.dayPasses >= 100 || $licensing.errUserLimit}
+          {#if $licensing.errUserLimit}
             <div>
               <Layout gap="S" justifyItems="center">
                 <img class="spaceman" alt="spaceman" src={Spaceman} />

packages/builder/src/pages/builder/portal/account/usage.svelte

@@ -28,7 +28,7 @@
   const upgradeUrl = `${$admin.accountPortalUrl}/portal/upgrade`
   const manageUrl = `${$admin.accountPortalUrl}/portal/billing`
 
-  const WARN_USAGE = ["Queries", "Automations", "Rows", "Day Passes", "Users"]
+  const WARN_USAGE = ["Queries", "Automations", "Rows", "Users"]
   const oneDayInSeconds = 86400
 
   const EXCLUDE_QUOTAS = {
@@ -36,9 +36,6 @@
     Users: license => {
       return license.plan.model !== PlanModel.PER_USER
     },
-    "Day Passes": license => {
-      return license.plan.model !== PlanModel.DAY_PASS
-    },
   }
 
   function excludeQuota(name) {

packages/builder/src/stores/portal/navigation.ts

@@ -1,13 +1,20 @@
 import { writable } from "svelte/store"
 
+type GotoFuncType = (path: string) => void
+
+interface Store {
+  initialisated: boolean
+  goto: GotoFuncType
+}
+
 export function createNavigationStore() {
-  const store = writable({
+  const store = writable<Store>({
     initialisated: false,
-    goto: undefined,
+    goto: undefined as any,
   })
   const { set, subscribe } = store
 
-  const init = gotoFunc => {
+  const init = (gotoFunc: GotoFuncType) => {
     if (typeof gotoFunc !== "function") {
       throw new Error(
         `gotoFunc must be a function, found a "${typeof gotoFunc}" instead`

packages/builder/src/stores/portal/plugins.ts

@@ -1,16 +1,21 @@
 import { writable } from "svelte/store"
+import { PluginSource } from "constants/index"
+
 import { API } from "api"
-import { PluginSource } from "constants"
+
+interface Plugin {
+  _id: string
+}
 
 export function createPluginsStore() {
-  const { subscribe, set, update } = writable([])
+  const { subscribe, set, update } = writable<Plugin[]>([])
 
   async function load() {
     const plugins = await API.getPlugins()
     set(plugins)
   }
 
-  async function deletePlugin(pluginId) {
+  async function deletePlugin(pluginId: string) {
     await API.deletePlugin(pluginId)
     update(state => {
       state = state.filter(existing => existing._id !== pluginId)
@@ -18,8 +23,8 @@ export function createPluginsStore() {
     })
   }
 
-  async function createPlugin(source, url, auth = null) {
+  async function createPlugin(source: string, url: string, auth = null) {
-    let pluginData = {
+    let pluginData: any = {
       source,
       url,
     }
@@ -46,7 +51,7 @@ export function createPluginsStore() {
     })
   }
 
-  async function uploadPlugin(file) {
+  async function uploadPlugin(file: File) {
     if (!file) {
       return
     }

packages/builder/tsconfig.build.json

@@ -9,15 +9,9 @@
     "noImplicitAny": true,
     "esModuleInterop": true,
     "resolveJsonModule": true,
-    "incremental": true
+    "incremental": true,
+    "skipLibCheck": true
   },
-  "include": [
+  "include": ["./src/**/*"],
-    "./src/**/*"
+  "exclude": ["node_modules", "**/*.json", "**/*.spec.ts", "**/*.spec.js"]
-  ],
-  "exclude": [
-    "node_modules",
-    "**/*.json",
-    "**/*.spec.ts",
-    "**/*.spec.js"
-  ]
 }

packages/builder/vite.config.mjs

@@ -3,6 +3,7 @@ import replace from "@rollup/plugin-replace"
 import { defineConfig, loadEnv } from "vite"
 import { viteStaticCopy } from "vite-plugin-static-copy"
 import path from "path"
+import typescript from "@rollup/plugin-typescript"
 
 const ignoredWarnings = [
   "unused-export-let",
@@ -35,7 +36,7 @@ export default defineConfig(({ mode }) => {
     // Copy fonts to an additional path so that svelte's automatic
     // prefixing of the base URL path can still resolve assets
     copyFonts("builder/fonts"),
-]
+  ]
 
   return {
     test: {
@@ -61,6 +62,7 @@ export default defineConfig(({ mode }) => {
       sourcemap: !isProduction,
     },
     plugins: [
+      typescript({ outDir: "../server/builder/dist" }),
       svelte({
         hot: !isProduction,
         emitCss: true,

packages/client/package.json

@@ -15,8 +15,8 @@
     "./manifest.json": "./manifest.json"
   },
   "scripts": {
-    "build": "rollup -c",
+    "build": "vite build",
-    "dev": "rollup -cw"
+    "dev": "vite build --watch"
   },
   "dependencies": {
     "@budibase/bbui": "0.0.0",
@@ -36,19 +36,9 @@
     "svelte-spa-router": "^4.0.1"
   },
   "devDependencies": {
-    "@rollup/plugin-alias": "^5.1.0",
+    "@sveltejs/vite-plugin-svelte": "1.4.0",
-    "@rollup/plugin-commonjs": "^25.0.7",
+    "vite": "^4.5.0",
-    "@rollup/plugin-image": "^3.0.3",
+    "vite-plugin-css-injected-by-js": "3.5.2"
-    "@rollup/plugin-node-resolve": "^15.2.3",
-    "postcss": "^8.4.35",
-    "rollup": "^4.9.6",
-    "rollup-plugin-json": "^4.0.0",
-    "rollup-plugin-polyfill-node": "^0.13.0",
-    "rollup-plugin-postcss": "^4.0.2",
-    "rollup-plugin-svelte": "^7.1.6",
-    "rollup-plugin-svg": "^2.0.0",
-    "rollup-plugin-terser": "^7.0.2",
-    "rollup-plugin-visualizer": "^5.12.0"
   },
   "resolutions": {
     "loader-utils": "1.4.1"

packages/client/rollup.config.js

@@ -1,117 +0,0 @@
-import commonjs from "@rollup/plugin-commonjs"
-import resolve from "@rollup/plugin-node-resolve"
-import alias from "@rollup/plugin-alias"
-import svelte from "rollup-plugin-svelte"
-import { terser } from "rollup-plugin-terser"
-import postcss from "rollup-plugin-postcss"
-import svg from "rollup-plugin-svg"
-import image from "@rollup/plugin-image"
-import json from "rollup-plugin-json"
-import nodePolyfills from "rollup-plugin-polyfill-node"
-import path from "path"
-import { visualizer } from "rollup-plugin-visualizer"
-
-const production = !process.env.ROLLUP_WATCH
-const ignoredWarnings = [
-  "unused-export-let",
-  "css-unused-selector",
-  "module-script-reactive-declaration",
-  "a11y-no-onchange",
-  "a11y-click-events-have-key-events",
-]
-
-const devPaths = production
-  ? []
-  : [
-      {
-        find: "@budibase/shared-core",
-        replacement: path.resolve("../shared-core/dist/index"),
-      },
-      {
-        find: "@budibase/types",
-        replacement: path.resolve("../types/dist/index"),
-      },
-    ]
-
-export default {
-  input: "src/index.js",
-  output: [
-    {
-      sourcemap: false,
-      format: "iife",
-      file: `./dist/budibase-client.js`,
-    },
-  ],
-  onwarn(warning, warn) {
-    if (
-      warning.code === "THIS_IS_UNDEFINED" ||
-      warning.code === "CIRCULAR_DEPENDENCY" ||
-      warning.code === "EVAL"
-    ) {
-      return
-    }
-    warn(warning)
-  },
-  plugins: [
-    alias({
-      entries: [
-        {
-          find: "manifest.json",
-          replacement: path.resolve("./manifest.json"),
-        },
-        {
-          find: "api",
-          replacement: path.resolve("./src/api"),
-        },
-        {
-          find: "components",
-          replacement: path.resolve("./src/components"),
-        },
-        {
-          find: "stores",
-          replacement: path.resolve("./src/stores"),
-        },
-        {
-          find: "utils",
-          replacement: path.resolve("./src/utils"),
-        },
-        {
-          find: "constants",
-          replacement: path.resolve("./src/constants"),
-        },
-        {
-          find: "sdk",
-          replacement: path.resolve("./src/sdk"),
-        },
-        ...devPaths,
-      ],
-    }),
-    svelte({
-      emitCss: true,
-      onwarn: (warning, handler) => {
-        // Ignore some warnings
-        if (!ignoredWarnings.includes(warning.code)) {
-          handler(warning)
-        }
-      },
-    }),
-    postcss(),
-    commonjs(),
-    nodePolyfills(),
-    resolve({
-      preferBuiltins: true,
-      browser: true,
-      dedupe: ["svelte", "svelte/internal"],
-    }),
-    svg(),
-    image({
-      exclude: "**/*.svg",
-    }),
-    json(),
-    production && terser(),
-    !production && visualizer(),
-  ],
-  watch: {
-    clearScreen: false,
-  },
-}

packages/client/src/index.js

@@ -10,7 +10,7 @@ import {
   eventStore,
   hoverStore,
 } from "./stores"
-import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-rollup.js"
+import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
 import { get } from "svelte/store"
 import { initWebsocket } from "./websocket.js"
 

packages/client/tsconfig.json

@@ -3,6 +3,10 @@
     "allowJs": true,
     "strict": true,
     "outDir": "dist",
+    "allowSyntheticDefaultImports": true,
+    "target": "ESNext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
     "paths": {
       "@budibase/*": [
         "../*/src/index.ts",
@@ -12,6 +16,5 @@
       ],
       "*": ["./src/*"]
     }
-  },
+  }
-  "include": ["src/**/*"]
 }

packages/client/vite.config.mjs

@@ -0,0 +1,89 @@
+import { svelte } from "@sveltejs/vite-plugin-svelte"
+import { defineConfig } from "vite"
+import path from "path"
+import cssInjectedByJsPlugin from "vite-plugin-css-injected-by-js"
+
+const ignoredWarnings = [
+  "unused-export-let",
+  "css-unused-selector",
+  "module-script-reactive-declaration",
+  "a11y-no-onchange",
+  "a11y-click-events-have-key-events",
+]
+
+export default defineConfig(({ mode }) => {
+  const isProduction = mode === "production"
+
+  return {
+    server: {
+      open: false,
+    },
+    build: {
+      lib: {
+        entry: "src/index.js",
+        formats: ["iife"],
+        outDir: "dist",
+        name: "budibase_client",
+        fileName: () => "budibase-client.js",
+        minify: isProduction,
+      },
+    },
+    plugins: [
+      svelte({
+        emitCss: true,
+        onwarn: (warning, handler) => {
+          // Ignore some warnings
+          if (!ignoredWarnings.includes(warning.code)) {
+            handler(warning)
+          }
+        },
+      }),
+      cssInjectedByJsPlugin(),
+    ],
+    resolve: {
+      dedupe: ["svelte", "svelte/internal"],
+      alias: [
+        {
+          find: "manifest.json",
+          replacement: path.resolve("./manifest.json"),
+        },
+        {
+          find: "api",
+          replacement: path.resolve("./src/api"),
+        },
+        {
+          find: "components",
+          replacement: path.resolve("./src/components"),
+        },
+        {
+          find: "stores",
+          replacement: path.resolve("./src/stores"),
+        },
+        {
+          find: "utils",
+          replacement: path.resolve("./src/utils"),
+        },
+        {
+          find: "constants",
+          replacement: path.resolve("./src/constants"),
+        },
+        {
+          find: "sdk",
+          replacement: path.resolve("./src/sdk"),
+        },
+        {
+          find: "@budibase/types",
+          replacement: path.resolve("../types/src"),
+        },
+        {
+          find: "@budibase/shared-core",
+          replacement: path.resolve("../shared-core/src"),
+        },
+        {
+          find: "@budibase/bbui",
+          replacement: path.resolve("../bbui/src"),
+        },
+      ],
+    },
+  }
+})

packages/pro

@@ -1 +1 @@
-Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd
+Subproject commit d9245f3d6d0b41ec2e6b3406b791f9e7448882cb

packages/server/package.json

@@ -129,7 +129,8 @@
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
-    "xml2js": "0.6.2"
+    "xml2js": "0.6.2",
+    "zod-validation-error": "^3.4.0"
   },
   "devDependencies": {
     "@babel/core": "^7.22.5",
@@ -175,7 +176,8 @@
     "tsconfig-paths": "4.0.0",
     "typescript": "5.5.2",
     "update-dotenv": "1.1.1",
-    "yargs": "13.2.4"
+    "yargs": "^13.2.4",
+    "zod": "^3.23.8"
   },
   "nx": {
     "targets": {

packages/server/specs/openapi.json

@@ -2166,7 +2166,6 @@
           "query": {
             "description": "Search parameters for view",
             "type": "object",
-            "required": [],
             "properties": {
               "logicalOperator": {
                 "description": "When using groups this defines whether all of the filters must match, or only one of them.",
@@ -2449,7 +2448,6 @@
               "query": {
                 "description": "Search parameters for view",
                 "type": "object",
-                "required": [],
                 "properties": {
                   "logicalOperator": {
                     "description": "When using groups this defines whether all of the filters must match, or only one of them.",
@@ -2743,7 +2741,6 @@
                 "query": {
                   "description": "Search parameters for view",
                   "type": "object",
-                  "required": [],
                   "properties": {
                     "logicalOperator": {
                       "description": "When using groups this defines whether all of the filters must match, or only one of them.",

packages/server/specs/openapi.yaml

@@ -1802,7 +1802,6 @@ components:
         query:
           description: Search parameters for view
           type: object
-          required: []
           properties:
             logicalOperator:
               description: When using groups this defines whether all of the filters must
@@ -2012,7 +2011,6 @@ components:
             query:
               description: Search parameters for view
               type: object
-              required: []
               properties:
                 logicalOperator:
                   description: When using groups this defines whether all of the filters must
@@ -2229,7 +2227,6 @@ components:
               query:
                 description: Search parameters for view
                 type: object
-                required: []
                 properties:
                   logicalOperator:
                     description: When using groups this defines whether all of the filters must

packages/server/specs/resources/view.ts

@@ -142,7 +142,6 @@ layeredFilterGroup.items.properties.groups = filterGroup
 const viewQuerySchema = {
   description: "Search parameters for view",
   type: "object",
-  required: [],
   properties: {
     logicalOperator,
     onEmptyFilter: {

packages/server/src/api/controllers/datasource.ts

@@ -296,16 +296,6 @@ export async function find(ctx: UserCtx) {
   ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
-// dynamic query functionality
-export async function query(ctx: UserCtx) {
-  const queryJson = ctx.request.body
-  try {
-    ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
-  } catch (err: any) {
-    ctx.throw(400, err)
-  }
-}
-
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -42,7 +42,7 @@ import {
   sqlOutputProcessing,
 } from "./utils"
 import {
-  getDatasourceAndQuery,
+  enrichQueryJson,
   processRowCountResponse,
 } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
@@ -135,16 +135,9 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   return config
 }
 
-function getEndpoint(tableId: string | undefined, operation: string) {
+function getEndpoint(tableId: string, operation: Operation) {
-  if (!tableId) {
-    throw new Error("Cannot get endpoint information - no table ID specified")
-  }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  return {
+  return { datasourceId, entityId: tableName, operation }
-    datasourceId: datasourceId,
-    entityId: tableName,
-    operation: operation as Operation,
-  }
 }
 
 function isOneSide(
@@ -268,12 +261,9 @@ export class ExternalRequest<T extends Operation> {
     const filters = this.prepareFilters(rowId, {}, table)
     // safety check, if there are no filters on deletion bad things happen
     if (Object.keys(filters).length !== 0) {
-      return getDatasourceAndQuery({
+      return makeExternalQuery({
         endpoint: getEndpoint(tableId, Operation.DELETE),
         filters,
-        meta: {
-          table,
-        },
       })
     } else {
       return []
@@ -289,13 +279,10 @@ export class ExternalRequest<T extends Operation> {
     const filters = this.prepareFilters(rowId, {}, table)
     // safety check, if there are no filters on deletion bad things happen
     if (Object.keys(filters).length !== 0) {
-      return getDatasourceAndQuery({
+      return makeExternalQuery({
         endpoint: getEndpoint(tableId, Operation.UPDATE),
         body: { [colName]: null },
         filters,
-        meta: {
-          table,
-        },
       })
     } else {
       return []
@@ -311,12 +298,9 @@ export class ExternalRequest<T extends Operation> {
   }
 
   async getRow(table: Table, rowId: string): Promise<Row> {
-    const response = await getDatasourceAndQuery({
+    const response = await makeExternalQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
       filters: this.prepareFilters(rowId, {}, table),
-      meta: {
-        table,
-      },
     })
     if (Array.isArray(response) && response.length > 0) {
       return response[0]
@@ -490,16 +474,13 @@ export class ExternalRequest<T extends Operation> {
       if (!relatedTable) {
         throw new Error("unable to find related table")
       }
-      const response = await getDatasourceAndQuery({
+      const response = await makeExternalQuery({
-        endpoint: endpoint,
+        endpoint,
         filters: {
           equal: {
             [fieldName]: row[lookupField],
           },
         },
-        meta: {
-          table: relatedTable,
-        },
       })
       // this is the response from knex if no rows found
       const rows: Row[] =
@@ -537,6 +518,11 @@ export class ExternalRequest<T extends Operation> {
     for (let relationship of relationships) {
       const { key, tableId, isUpdate, id, relationshipType, ...rest } =
         relationship
+
+      if (!tableId) {
+        throw new Error("Table ID is unknown, cannot find table")
+      }
+
       const body: { [key: string]: any } = processObjectSync(rest, row, {})
       const linkTable = this.getTable(tableId)
       const relationshipPrimary = linkTable?.primary || []
@@ -583,14 +569,11 @@ export class ExternalRequest<T extends Operation> {
       const operation = isUpdate ? Operation.UPDATE : Operation.CREATE
       if (!existingRelationship) {
         promises.push(
-          getDatasourceAndQuery({
+          makeExternalQuery({
             endpoint: getEndpoint(tableId, operation),
             // if we're doing many relationships then we're writing, only one response
             body,
             filters: this.prepareFilters(id, {}, linkTable),
-            meta: {
-              table: linkTable,
-            },
           })
         )
       } else {
@@ -723,8 +706,8 @@ export class ExternalRequest<T extends Operation> {
 
     let json: QueryJson = {
       endpoint: {
-        datasourceId: this.datasource._id!,
+        datasourceId: this.datasource,
-        entityId: table.name,
+        entityId: table,
         operation,
       },
       resource: {
@@ -749,10 +732,6 @@ export class ExternalRequest<T extends Operation> {
           table
         ),
       },
-      meta: {
-        table,
-        tables: this.tables,
-      },
     }
 
     // remove any relationships that could block deletion
@@ -773,8 +752,11 @@ export class ExternalRequest<T extends Operation> {
       response = [unprocessedRow]
     } else {
       response = env.SQL_ALIASING_DISABLE
-        ? await getDatasourceAndQuery(json)
+        ? await makeExternalQuery(json)
-        : await aliasing.queryWithAliasing(json, makeExternalQuery)
+        : await aliasing.queryWithAliasing(
+            await enrichQueryJson(json),
+            makeExternalQuery
+          )
     }
 
     // if it's a counting operation there will be no more processing, just return the number

packages/server/src/api/controllers/row/index.ts

@@ -19,6 +19,7 @@ import {
   isRelationshipField,
   PatchRowRequest,
   PatchRowResponse,
+  RequiredKeys,
   Row,
   RowAttachment,
   RowSearchParams,
@@ -239,7 +240,8 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
 
   await context.ensureSnippetContext(true)
 
-  let { query } = ctx.request.body
+  const searchRequest = ctx.request.body
+  let { query } = searchRequest
   if (query) {
     const allTables = await sdk.tables.getAllTables()
     query = replaceTableNamesInFilters(tableId, query, allTables)
@@ -249,11 +251,22 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
     user: sdk.users.getUserContextBindings(ctx.user),
   })
 
-  const searchParams: RowSearchParams = {
+  const searchParams: RequiredKeys<RowSearchParams> = {
-    ...ctx.request.body,
     query: enrichedQuery,
     tableId,
     viewId,
+    bookmark: searchRequest.bookmark ?? undefined,
+    paginate: searchRequest.paginate,
+    limit: searchRequest.limit,
+    sort: searchRequest.sort ?? undefined,
+    sortOrder: searchRequest.sortOrder,
+    sortType: searchRequest.sortType ?? undefined,
+    countRows: searchRequest.countRows,
+    version: searchRequest.version,
+    disableEscaping: searchRequest.disableEscaping,
+    fields: undefined,
+    indexer: undefined,
+    rows: undefined,
   }
 
   ctx.status = 200

packages/server/src/api/controllers/row/staticFormula.ts

@@ -15,10 +15,21 @@ import {
 } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
-import { cloneDeep } from "lodash/fp"
+import { cloneDeep, merge } from "lodash/fp"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 
+function mergeRows(row1: Row, row2: Row) {
+  const merged = merge(row1, row2)
+  // make sure any specifically undefined fields are removed
+  for (const key of Object.keys(row2)) {
+    if (row2[key] === undefined) {
+      delete merged[key]
+    }
+  }
+  return merged
+}
+
 /**
  * This function runs through a list of enriched rows, looks at the rows which
  * are related and then checks if they need the state of their formulas
@@ -162,9 +173,14 @@ export async function finaliseRow(
     })
   }
 
-  const response = await db.put(row)
+  await db.put(row)
-  // for response, calculate the formulas for the enriched row
+  const retrieved = await db.tryGet<Row>(row._id)
-  enrichedRow._rev = response.rev
+  if (!retrieved) {
+    throw new Error(`Unable to retrieve row ${row._id} after saving.`)
+  }
+
+  delete enrichedRow._rev
+  enrichedRow = mergeRows(retrieved, enrichedRow)
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })

packages/server/src/api/controllers/row/utils/utils.ts

@@ -175,7 +175,7 @@ export async function enrichArrayContext(
 }
 
 export async function enrichSearchContext(
-  fields: Record<string, any>,
+  fields: Record<string, any> | undefined,
   inputs = {},
   helpers = true
 ): Promise<Record<string, any>> {

packages/server/src/api/controllers/row/views.ts

@@ -29,19 +29,20 @@ export async function searchView(
 
   await context.ensureSnippetContext(true)
 
-  const searchOptions: RequiredKeys<SearchViewRowRequest> &
+  const searchOptions: RequiredKeys<RowSearchParams> = {
-    RequiredKeys<
-      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
-    > = {
     tableId: view.tableId,
     viewId: view.id,
-    query: body.query,
+    query: body.query || {},
     fields: viewFields,
     ...getSortOptions(body, view),
     limit: body.limit,
-    bookmark: body.bookmark,
+    bookmark: body.bookmark ?? undefined,
     paginate: body.paginate,
     countRows: body.countRows,
+    version: undefined,
+    disableEscaping: undefined,
+    indexer: undefined,
+    rows: undefined,
   }
 
   const result = await sdk.rows.search(searchOptions, {
@@ -56,7 +57,7 @@ function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
     return {
       sort: request.sort,
       sortOrder: request.sortOrder,
-      sortType: request.sortType,
+      sortType: request.sortType ?? undefined,
     }
   }
   if (view.sort) {

packages/server/src/api/controllers/table/ExternalRequest.ts

@@ -11,27 +11,24 @@ export async function makeTableRequest(
   datasource: Datasource,
   operation: Operation,
   table: Table,
-  tables: Record<string, Table>,
   oldTable?: Table,
   renamed?: RenameColumn
 ) {
   const json: QueryJson = {
     endpoint: {
-      datasourceId: datasource._id!,
+      datasourceId: datasource,
-      entityId: table._id!,
+      entityId: table,
       operation,
     },
-    meta: {
+  }
-      table,
+  if (!json.meta) {
-      tables,
+    json.meta = {}
-    },
-    table,
   }
   if (oldTable) {
-    json.meta!.table = oldTable
+    json.meta.oldTable = oldTable
   }
   if (renamed) {
-    json.meta!.renamed = renamed
+    json.meta.renamed = renamed
   }
-  return makeExternalQuery(datasource, json)
+  return makeExternalQuery(json)
 }

packages/server/src/api/routes/datasource.ts

@@ -2,10 +2,7 @@ import Router from "@koa/router"
 import * as datasourceController from "../controllers/datasource"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
-import {
+import { datasourceValidator } from "./utils/validators"
-  datasourceValidator,
-  datasourceQueryValidator,
-} from "./utils/validators"
 
 const router: Router = new Router()
 
@@ -41,15 +38,6 @@ router
     ),
     datasourceController.update
   )
-  .post(
-    "/api/datasources/query",
-    authorized(
-      permissions.PermissionType.TABLE,
-      permissions.PermissionLevel.READ
-    ),
-    datasourceQueryValidator(),
-    datasourceController.query
-  )
   .post(
     "/api/datasources/:datasourceId/schema",
     authorized(permissions.BUILDER),

packages/server/src/api/routes/row.ts

@@ -5,6 +5,8 @@ import { paramResource, paramSubResource } from "../../middleware/resourceId"
 import { permissions } from "@budibase/backend-core"
 import { internalSearchValidator } from "./utils/validators"
 import trimViewRowInfo from "../../middleware/trimViewRowInfo"
+import { validateBody } from "../../middleware/zod-validator"
+import { searchRowRequestValidator } from "@budibase/types"
 
 const { PermissionType, PermissionLevel } = permissions
 
@@ -32,6 +34,7 @@ router
   .post(
     "/api/:sourceId/search",
     internalSearchValidator(),
+    validateBody(searchRowRequestValidator),
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
@@ -87,6 +90,7 @@ router
 router.post(
   "/api/v2/views/:viewId/search",
   internalSearchValidator(),
+  validateBody(searchRowRequestValidator),
   authorizedResource(PermissionType.VIEW, PermissionLevel.READ, "viewId"),
   rowController.views.searchView
 )

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -1,10 +1,4 @@
-import {
+import { Datasource, Query, QueryPreview } from "@budibase/types"
-  Datasource,
-  Operation,
-  Query,
-  QueryPreview,
-  TableSourceType,
-} from "@budibase/types"
 import {
   DatabaseName,
   datasourceDescribe,
@@ -817,49 +811,6 @@ if (descriptions.length) {
       })
 
       describe("query through datasource", () => {
-        it("should be able to query the datasource", async () => {
-          const datasource = await config.api.datasource.create(rawDatasource)
-
-          const entityId = tableName
-          await config.api.datasource.update({
-            ...datasource,
-            entities: {
-              [entityId]: {
-                name: entityId,
-                schema: {},
-                type: "table",
-                primary: ["id"],
-                sourceId: datasource._id!,
-                sourceType: TableSourceType.EXTERNAL,
-              },
-            },
-          })
-
-          const res = await config.api.datasource.query({
-            endpoint: {
-              datasourceId: datasource._id!,
-              operation: Operation.READ,
-              entityId,
-            },
-            resource: {
-              fields: ["id", "name"],
-            },
-            filters: {
-              string: {
-                name: "two",
-              },
-            },
-          })
-          expect(res).toHaveLength(1)
-          expect(res[0]).toEqual({
-            id: 2,
-            name: "two",
-            // the use of table.* introduces the possibility of nulls being returned
-            birthday: null,
-            number: null,
-          })
-        })
-
         // this parameter really only impacts SQL queries
         describe("confirm nullDefaultSupport", () => {
           let queryParams: Partial<Query>

packages/server/src/api/routes/tests/row.spec.ts

@@ -2607,6 +2607,8 @@ if (descriptions.length) {
             name: "foo",
             description: "bar",
             tableId,
+            createdAt: isInternal ? new Date().toISOString() : undefined,
+            updatedAt: isInternal ? new Date().toISOString() : undefined,
           })
         })
 
@@ -2628,6 +2630,8 @@ if (descriptions.length) {
             id: isInternal ? undefined : expect.any(Number),
             type: isInternal ? "row" : undefined,
             [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+            createdAt: isInternal ? new Date().toISOString() : undefined,
+            updatedAt: isInternal ? new Date().toISOString() : undefined,
           })
         })
 
@@ -2650,6 +2654,8 @@ if (descriptions.length) {
             _rev: expect.any(String),
             id: isInternal ? undefined : expect.any(Number),
             type: isInternal ? "row" : undefined,
+            createdAt: isInternal ? new Date().toISOString() : undefined,
+            updatedAt: isInternal ? new Date().toISOString() : undefined,
           })
         })
 
@@ -2729,6 +2735,8 @@ if (descriptions.length) {
             id: isInternal ? undefined : expect.any(Number),
             type: isInternal ? "row" : undefined,
             [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+            createdAt: isInternal ? new Date().toISOString() : undefined,
+            updatedAt: isInternal ? new Date().toISOString() : undefined,
           })
         })
 
@@ -2745,15 +2753,8 @@ if (descriptions.length) {
             user: null,
             users: null,
           })
-          expect(updatedRow).toEqual({
+          expect(updatedRow.user).toBeUndefined()
-            name: "foo",
+          expect(updatedRow.users).toBeUndefined()
-            description: "bar",
-            tableId,
-            _id: row._id,
-            _rev: expect.any(String),
-            id: isInternal ? undefined : expect.any(Number),
-            type: isInternal ? "row" : undefined,
-          })
         })
 
         it("fetch all will populate the relationships", async () => {
@@ -3268,7 +3269,7 @@ if (descriptions.length) {
               formula: {
                 name: "formula",
                 type: FieldType.FORMULA,
-                formula: formula,
+                formula,
                 responseType: opts?.responseType,
                 formulaType: opts?.formulaType || FormulaType.DYNAMIC,
               },
@@ -3495,6 +3496,72 @@ if (descriptions.length) {
           )
         })
       })
+
+      if (!isInternal && !isOracle) {
+        describe("bigint ids", () => {
+          let table1: Table, table2: Table
+          let table1Name: string, table2Name: string
+
+          beforeAll(async () => {
+            table1Name = `table1-${generator.guid().substring(0, 5)}`
+            await client!.schema.createTable(table1Name, table => {
+              table.bigInteger("table1Id").primary()
+            })
+
+            table2Name = `table2-${generator.guid().substring(0, 5)}`
+            await client!.schema.createTable(table2Name, table => {
+              table.bigInteger("table2Id").primary()
+              table
+                .bigInteger("table1Ref")
+                .references("table1Id")
+                .inTable(table1Name)
+            })
+
+            const resp = await config.api.datasource.fetchSchema({
+              datasourceId: datasource!._id!,
+            })
+
+            const tables = Object.values(resp.datasource.entities || {})
+            table1 = tables.find(t => t.name === table1Name)!
+            table2 = tables.find(t => t.name === table2Name)!
+
+            await config.api.datasource.addExistingRelationship({
+              one: {
+                tableId: table2._id!,
+                relationshipName: "one",
+                foreignKey: "table1Ref",
+              },
+              many: {
+                tableId: table1._id!,
+                relationshipName: "many",
+                primaryKey: "table1Id",
+              },
+            })
+          })
+
+          it("should be able to fetch rows with related bigint ids", async () => {
+            const row = await config.api.row.save(table1._id!, {
+              table1Id: "1",
+            })
+            await config.api.row.save(table2._id!, {
+              table2Id: "2",
+              table1Ref: row.table1Id,
+            })
+
+            let resp = await config.api.row.search(table1._id!)
+            expect(resp.rows).toHaveLength(1)
+            expect(resp.rows[0]._id).toBe("%5B'1'%5D")
+            expect(resp.rows[0].many).toHaveLength(1)
+            expect(resp.rows[0].many[0]._id).toBe("%5B'2'%5D")
+
+            resp = await config.api.row.search(table2._id!)
+            expect(resp.rows).toHaveLength(1)
+            expect(resp.rows[0]._id).toBe("%5B'2'%5D")
+            expect(resp.rows[0].one).toHaveLength(1)
+            expect(resp.rows[0].one[0]._id).toBe("%5B'1'%5D")
+          })
+        })
+      }
     }
   )
 }

packages/server/src/api/routes/tests/search.spec.ts

@@ -24,6 +24,7 @@ import {
   JsonFieldSubType,
   LogicalOperator,
   RelationshipType,
+  RequiredKeys,
   Row,
   RowSearchParams,
   SearchFilters,
@@ -208,9 +209,25 @@ if (descriptions.length) {
 
               private async performSearch(): Promise<SearchResponse<Row>> {
                 if (isInMemory) {
-                  return dataFilters.search(_.cloneDeep(rows), {
+                  const inMemoryQuery: RequiredKeys<
-                    ...this.query,
+                    Omit<RowSearchParams, "tableId">
-                  })
+                  > = {
+                    sort: this.query.sort ?? undefined,
+                    query: { ...this.query.query },
+                    paginate: this.query.paginate,
+                    bookmark: this.query.bookmark ?? undefined,
+                    limit: this.query.limit,
+                    sortOrder: this.query.sortOrder,
+                    sortType: this.query.sortType ?? undefined,
+                    version: this.query.version,
+                    disableEscaping: this.query.disableEscaping,
+                    countRows: this.query.countRows,
+                    viewId: undefined,
+                    fields: undefined,
+                    indexer: undefined,
+                    rows: undefined,
+                  }
+                  return dataFilters.search(_.cloneDeep(rows), inMemoryQuery)
                 } else {
                   return config.api.row.search(tableOrViewId, this.query)
                 }

packages/server/src/api/routes/utils/validators.ts

@@ -1,5 +1,4 @@
 import { auth, permissions } from "@budibase/backend-core"
-import { DataSourceOperation } from "../../../constants"
 import {
   AutomationActionStepId,
   AutomationStep,
@@ -231,30 +230,6 @@ export function externalSearchValidator() {
   )
 }
 
-export function datasourceQueryValidator() {
-  return auth.joiValidator.body(
-    Joi.object({
-      endpoint: Joi.object({
-        datasourceId: Joi.string().required(),
-        operation: Joi.string()
-          .required()
-          .valid(...Object.values(DataSourceOperation)),
-        entityId: Joi.string().required(),
-      }).required(),
-      resource: Joi.object({
-        fields: Joi.array().items(Joi.string()).optional(),
-      }).optional(),
-      body: Joi.object().optional(),
-      sort: Joi.object().optional(),
-      filters: filterObject().optional(),
-      paginate: Joi.object({
-        page: Joi.string().alphanum().optional(),
-        limit: Joi.number().optional(),
-      }).optional(),
-    })
-  )
-}
-
 export function webhookValidator() {
   return auth.joiValidator.body(
     Joi.object({

packages/server/src/automations/tests/scenarios/looping.spec.ts

@@ -152,6 +152,44 @@ describe("Loop automations", () => {
     )
   })
 
+  it("ensure the loop stops if the max iterations are reached", async () => {
+    const builder = createAutomationBuilder({
+      name: "Test Loop max iterations",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .loop({
+        option: LoopStepType.ARRAY,
+        binding: ["test", "test2", "test3"],
+        iterations: 2,
+      })
+      .serverLog({ text: "{{loop.currentItem}}" })
+      .serverLog({ text: "{{steps.1.iterations}}" })
+      .run()
+
+    expect(results.steps[0].outputs.iterations).toBe(2)
+  })
+
+  it("should run an automation with loop and max iterations to ensure context correctness further down the tree", async () => {
+    const builder = createAutomationBuilder({
+      name: "Test context down tree with Loop and max iterations",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .loop({
+        option: LoopStepType.ARRAY,
+        binding: ["test", "test2", "test3"],
+        iterations: 2,
+      })
+      .serverLog({ text: "{{loop.currentItem}}" })
+      .serverLog({ text: "{{steps.1.iterations}}" })
+      .run()
+
+    expect(results.steps[1].outputs.message).toContain("- 2")
+  })
+
   it("should run an automation where a loop is successfully run twice", async () => {
     const builder = createAutomationBuilder({
       name: "Test Trigger with Loop and Create Row",

packages/server/src/constants/index.ts

@@ -45,17 +45,6 @@ export enum AuthTypes {
   EXTERNAL = "external",
 }
 
-export enum DataSourceOperation {
-  CREATE = "CREATE",
-  READ = "READ",
-  UPDATE = "UPDATE",
-  DELETE = "DELETE",
-  BULK_CREATE = "BULK_CREATE",
-  CREATE_TABLE = "CREATE_TABLE",
-  UPDATE_TABLE = "UPDATE_TABLE",
-  DELETE_TABLE = "DELETE_TABLE",
-}
-
 export enum DatasourceAuthTypes {
   GOOGLE = "google",
 }
@@ -148,7 +137,6 @@ export enum InvalidColumns {
 
 export enum AutomationErrors {
   INCORRECT_TYPE = "INCORRECT_TYPE",
-  MAX_ITERATIONS = "MAX_ITERATIONS_REACHED",
   FAILURE_CONDITION = "FAILURE_CONDITION_MET",
 }
 

packages/server/src/integrations/base/query.ts

@@ -1,37 +1,39 @@
 import {
-  QueryJson,
-  Datasource,
   DatasourcePlusQueryResponse,
-  RowOperations,
+  EnrichedQueryJson,
+  QueryJson,
 } from "@budibase/types"
 import { getIntegration } from "../index"
 import sdk from "../../sdk"
+import { enrichQueryJson } from "../../sdk/app/rows/utils"
+
+function isEnriched(
+  json: QueryJson | EnrichedQueryJson
+): json is EnrichedQueryJson {
+  return "datasource" in json
+}
 
 export async function makeExternalQuery(
-  datasource: Datasource,
+  json: QueryJson | EnrichedQueryJson
-  json: QueryJson
 ): Promise<DatasourcePlusQueryResponse> {
-  const entityId = json.endpoint.entityId,
+  if (!isEnriched(json)) {
-    tableName = json.meta.table.name,
+    json = await enrichQueryJson(json)
-    tableId = json.meta.table._id
+    if (json.datasource) {
-  // case found during testing - make sure this doesn't happen again
+      json.datasource = await sdk.datasources.enrich(json.datasource)
-  if (
+    }
-    RowOperations.includes(json.endpoint.operation) &&
-    entityId !== tableId &&
-    entityId !== tableName
-  ) {
-    throw new Error("Entity ID and table metadata do not align")
   }
-  if (!datasource) {
+
+  if (!json.datasource) {
     throw new Error("No datasource provided for external query")
   }
-  datasource = await sdk.datasources.enrich(datasource)
+
-  const Integration = await getIntegration(datasource.source)
+  const Integration = await getIntegration(json.datasource.source)
+
   // query is the opinionated function
-  if (Integration.prototype.query) {
+  if (!Integration.prototype.query) {
-    const integration = new Integration(datasource.config)
-    return integration.query(json)
-  } else {
     throw "Datasource does not support query."
   }
+
+  const integration = new Integration(json.datasource.config)
+  return integration.query(json)
 }

packages/server/src/integrations/googlesheets.ts

@@ -7,7 +7,6 @@ import {
   Integration,
   Operation,
   PaginationJson,
-  QueryJson,
   QueryType,
   Row,
   Schema,
@@ -18,6 +17,7 @@ import {
   TableSourceType,
   DatasourcePlusQueryResponse,
   BBReferenceFieldSubType,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import {
@@ -381,9 +381,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
     return { tables: externalTables, errors }
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
-    const sheet = json.endpoint.entityId
+    const sheet = json.table.name
-    switch (json.endpoint.operation) {
+    switch (json.operation) {
       case Operation.CREATE:
         return this.create({ sheet, row: json.body as Row })
       case Operation.BULK_CREATE:
@@ -400,7 +400,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
           rowIndex: json.extra?.idFilter?.equal?.rowNumber,
           sheet,
           row: json.body,
-          table: json.meta.table,
+          table: json.table,
         })
       case Operation.DELETE:
         return this.delete({
@@ -426,7 +426,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
         return this.deleteTable(json?.table?.name)
       default:
         throw new Error(
-          `GSheets integration does not support "${json.endpoint.operation}".`
+          `GSheets integration does not support "${json.operation}".`
         )
     }
   }

packages/server/src/integrations/microsoftSqlServer.ts

@@ -4,9 +4,9 @@ import {
   DatasourceFieldType,
   DatasourcePlus,
   DatasourcePlusQueryResponse,
+  EnrichedQueryJson,
   Integration,
   Operation,
-  QueryJson,
   QueryType,
   Schema,
   SourceName,
@@ -342,7 +342,8 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
           ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
           : query.sql
       this.log(sql, query.bindings)
-      return await request.query(sql)
+      const resp = await request.query(sql)
+      return resp
     } catch (err: any) {
       let readableMessage = getReadableErrorMessage(
         SourceName.SQL_SERVER,
@@ -505,23 +506,21 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     return response.recordset || [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const schema = this.config.schema
     await this.connect()
-    if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {
+    if (schema && schema !== DEFAULT_SCHEMA) {
-      json.endpoint.schema = schema
+      json.schema = schema
     }
     const operation = this._operation(json)
     const queryFn = (query: any, op: string) => this.internalQuery(query, op)
     const processFn = (result: any) => {
-      if (json?.meta?.table && result.recordset) {
+      if (result.recordset) {
         return this.convertJsonStringColumns(
-          json.meta.table,
+          json.table,
           result.recordset,
           json.tableAliases
         )
-      } else if (result.recordset) {
-        return result.recordset
       }
       return [{ [operation]: true }]
     }

packages/server/src/integrations/mysql.ts

@@ -2,7 +2,6 @@ import {
   Integration,
   DatasourceFieldType,
   QueryType,
-  QueryJson,
   SqlQuery,
   Table,
   TableSchema,
@@ -15,6 +14,7 @@ import {
   DatasourcePlusQueryResponse,
   SqlQueryBinding,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -390,15 +390,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     return results.length ? results : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     await this.connect()
     try {
       const queryFn = (query: any) =>
         this.internalQuery(query, { connect: false, disableCoercion: true })
       const processFn = (result: any) => {
-        if (json?.meta?.table && Array.isArray(result)) {
+        if (Array.isArray(result)) {
           return this.convertJsonStringColumns(
-            json.meta.table,
+            json.table,
             result,
             json.tableAliases
           )

packages/server/src/integrations/oracle.ts

@@ -3,7 +3,6 @@ import {
   DatasourceFieldType,
   Integration,
   Operation,
-  QueryJson,
   QueryType,
   SqlQuery,
   Table,
@@ -15,6 +14,7 @@ import {
   Row,
   DatasourcePlusQueryResponse,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -545,7 +545,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const operation = this._operation(json)
     const input = this._query(json, { disableReturning: true }) as SqlQuery
     if (Array.isArray(input)) {
@@ -572,13 +572,9 @@ class OracleIntegration extends Sql implements DatasourcePlus {
         return response.rows as Row[]
       } else {
         // get the last row that was updated
-        if (
+        if (response.lastRowid && operation !== Operation.DELETE) {
-          response.lastRowid &&
-          json.endpoint?.entityId &&
-          operation !== Operation.DELETE
-        ) {
           const lastRow = await this.internalQuery({
-            sql: `SELECT * FROM "${json.endpoint.entityId}" WHERE ROWID = '${response.lastRowid}'`,
+            sql: `SELECT * FROM "${json.table.name}" WHERE ROWID = '${response.lastRowid}'`,
           })
           return lastRow.rows as Row[]
         } else {

packages/server/src/integrations/postgres.ts

@@ -3,7 +3,6 @@ import {
   Integration,
   DatasourceFieldType,
   QueryType,
-  QueryJson,
   SqlQuery,
   Table,
   DatasourcePlus,
@@ -14,6 +13,7 @@ import {
   TableSourceType,
   DatasourcePlusQueryResponse,
   SqlClient,
+  EnrichedQueryJson,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -419,7 +419,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     return response.rows.length ? response.rows : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
+  async query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse> {
     const operation = this._operation(json).toLowerCase()
     const input = this._query(json) as SqlQuery
     if (Array.isArray(input)) {

packages/server/src/integrations/tests/sql.spec.ts

@@ -1,268 +0,0 @@
-import {
-  FieldType,
-  Operation,
-  PaginationJson,
-  QueryJson,
-  SearchFilters,
-  SortJson,
-  SqlClient,
-  Table,
-  TableSourceType,
-} from "@budibase/types"
-import { sql } from "@budibase/backend-core"
-import { merge } from "lodash"
-
-const Sql = sql.Sql
-
-const TABLE_NAME = "test"
-const TABLE: Table = {
-  type: "table",
-  sourceType: TableSourceType.EXTERNAL,
-  sourceId: "SOURCE_ID",
-  schema: {
-    id: {
-      name: "id",
-      type: FieldType.NUMBER,
-    },
-  },
-  name: TABLE_NAME,
-  primary: ["id"],
-}
-
-const ORACLE_TABLE: Partial<Table> = {
-  schema: {
-    name: {
-      name: "name",
-      type: FieldType.STRING,
-    },
-  },
-}
-
-function endpoint(table: string, operation: Operation) {
-  return {
-    datasourceId: "Postgres",
-    operation: operation,
-    entityId: table || TABLE_NAME,
-  }
-}
-
-function generateReadJson({
-  table,
-  fields,
-  filters,
-  sort,
-  paginate,
-}: {
-  table?: Partial<Table>
-  fields?: string[]
-  filters?: SearchFilters
-  sort?: SortJson
-  paginate?: PaginationJson
-} = {}): QueryJson {
-  let tableObj: Table = { ...TABLE }
-  if (table) {
-    tableObj = merge(TABLE, table)
-  }
-  return {
-    endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
-    resource: {
-      fields: fields || [],
-    },
-    filters: filters || {},
-    sort: sort || {},
-    paginate: paginate || undefined,
-    meta: {
-      table: tableObj,
-    },
-  }
-}
-
-function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
-  return {
-    endpoint: {
-      datasourceId: "Postgres",
-      entityId: "brands",
-      operation: Operation.READ,
-      schema: config.schema,
-    },
-    resource: {
-      fields: [
-        "brands.brand_id",
-        "brands.brand_name",
-        "products.product_id",
-        "products.product_name",
-        "products.brand_id",
-      ],
-    },
-    filters: {},
-    sort: {},
-    relationships: [
-      {
-        from: "brand_id",
-        to: "brand_id",
-        tableName: "products",
-        column: "products",
-      },
-    ],
-    extra: { idFilter: {} },
-    meta: {
-      table: TABLE,
-    },
-  }
-}
-
-function generateManyRelationshipJson(config: { schema?: string } = {}) {
-  return {
-    endpoint: {
-      datasourceId: "Postgres",
-      entityId: "stores",
-      operation: "READ",
-      schema: config.schema,
-    },
-    resource: {
-      fields: [
-        "stores.store_id",
-        "stores.store_name",
-        "products.product_id",
-        "products.product_name",
-      ],
-    },
-    filters: {},
-    sort: {},
-    paginate: {},
-    relationships: [
-      {
-        from: "store_id",
-        to: "product_id",
-        tableName: "products",
-        column: "products",
-        through: "stocks",
-        fromPrimary: "store_id",
-        toPrimary: "product_id",
-      },
-    ],
-    extra: { idFilter: {} },
-    meta: {
-      table: TABLE,
-    },
-  }
-}
-
-describe("SQL query builder", () => {
-  const relationshipLimit = 500
-  const limit = 500
-  const client = SqlClient.POSTGRES
-  let sql: any
-
-  beforeEach(() => {
-    sql = new Sql(client, limit)
-  })
-
-  it("should add the schema to the LEFT JOIN", () => {
-    const query = sql._query(generateRelationshipJson({ schema: "production" }))
-    expect(query).toEqual({
-      bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
-    })
-  })
-
-  it("should handle if the schema is not present when doing a LEFT JOIN", () => {
-    const query = sql._query(generateRelationshipJson())
-    expect(query).toEqual({
-      bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
-    })
-  })
-
-  it("should add the schema to both the toTable and throughTable in many-to-many join", () => {
-    const query = sql._query(
-      generateManyRelationshipJson({ schema: "production" })
-    )
-    expect(query).toEqual({
-      bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "stores".* from "production"."stores" order by "test"."id" asc limit $1) select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $2) as "products") as "products" from "paginated" as "stores" order by "test"."id" asc`,
-    })
-  })
-
-  it("should lowercase the values for Oracle LIKE statements", () => {
-    let query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        filters: {
-          string: {
-            name: "John",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: ["john%", limit],
-      sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
-    })
-
-    query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        filters: {
-          contains: {
-            age: [20, 25],
-            name: ["John", "Mary"],
-          },
-        },
-      })
-    )
-    const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
-    expect(query).toEqual({
-      bindings: [...filterSet, limit],
-      sql: `select * from (select * from "test" where ((COALESCE(LOWER("test"."age"), '') like :1 and COALESCE(LOWER("test"."age"), '') like :2)) and ((COALESCE(LOWER("test"."name"), '') like :3 and COALESCE(LOWER("test"."name"), '') like :4)) order by "test"."id" asc) where rownum <= :5`,
-    })
-
-    query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        filters: {
-          fuzzy: {
-            name: "Jo",
-          },
-        },
-      })
-    )
-    expect(query).toEqual({
-      bindings: [`%jo%`, limit],
-      sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
-    })
-  })
-
-  it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
-    let query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        table: ORACLE_TABLE,
-        filters: {
-          equal: {
-            name: "John",
-          },
-        },
-      })
-    )
-
-    expect(query).toEqual({
-      bindings: ["John", limit],
-      sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`,
-    })
-  })
-
-  it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
-    let query = new Sql(SqlClient.ORACLE, limit)._query(
-      generateReadJson({
-        table: ORACLE_TABLE,
-        filters: {
-          notEqual: {
-            name: "John",
-          },
-        },
-      })
-    )
-
-    expect(query).toEqual({
-      bindings: ["John", limit],
-      sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") != :1) or to_char("test"."name") is null order by "test"."id" asc) where rownum <= :2`,
-    })
-  })
-})

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -1,12 +1,13 @@
 import {
   Datasource,
   Operation,
-  QueryJson,
   SourceName,
   SqlQuery,
+  SqlClient,
+  EnrichedQueryJson,
+  TableSchema,
   Table,
   TableSourceType,
-  SqlClient,
 } from "@budibase/types"
 import { sql } from "@budibase/backend-core"
 import { join } from "path"
@@ -16,17 +17,21 @@ import sdk from "../../sdk"
 const Sql = sql.Sql
 
 // this doesn't exist strictly
-const TABLE: Table = {
+const TABLE = buildTable("tableName", {})
-  type: "table",
-  sourceType: TableSourceType.EXTERNAL,
-  sourceId: "SOURCE_ID",
-  schema: {},
-  name: "tableName",
-  primary: ["id"],
-}
 
 const AliasTables = sdk.rows.AliasTables
 
+function buildTable(name: string, schema: TableSchema): Table {
+  return {
+    type: "table",
+    sourceType: TableSourceType.EXTERNAL,
+    sourceId: "SOURCE_ID",
+    schema: schema,
+    name: name,
+    primary: ["id"],
+  }
+}
+
 function multiline(sql: string) {
   return sql.replace(/\n/g, "").replace(/ +/g, " ")
 }
@@ -35,8 +40,22 @@ describe("Captures of real examples", () => {
   const relationshipLimit = 500
   const primaryLimit = 100
 
-  function getJson(name: string): QueryJson {
+  function getJson(name: string): EnrichedQueryJson {
-    return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
+    // tables aren't fully specified in the test JSON
+    const base = require(join(__dirname, "sqlQueryJson", name)) as Omit<
+      EnrichedQueryJson,
+      "tables"
+    >
+    const tables: Record<string, Table> = { [base.table.name]: base.table }
+    if (base.relationships) {
+      for (let { tableName } of base.relationships) {
+        tables[tableName] = buildTable(tableName, {})
+      }
+    }
+    return {
+      ...base,
+      tables: tables,
+    }
   }
 
   describe("create", () => {
@@ -63,7 +82,7 @@ describe("Captures of real examples", () => {
         bindings: [primaryLimit, relationshipLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")`
+            `select json_agg(json_build_object('executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'completed',"b"."completed",'completed',"b"."completed",'taskname',"b"."taskname",'taskname',"b"."taskname"`
           )
         ),
       })
@@ -94,7 +113,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
-                 select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) 
+                 select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
                  from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
                  from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
@@ -212,8 +231,7 @@ describe("Captures of real examples", () => {
       }, queryJson)
       expect(returningQuery).toEqual({
         sql: multiline(
-          `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 
+          `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc`
-               and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc`
         ),
         bindings: [1, "Test", 22],
       })
@@ -246,15 +264,17 @@ describe("Captures of real examples", () => {
       }
     }
 
-    function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson {
+    function getQuery(
+      op: Operation,
+      fields: string[] = ["a"]
+    ): EnrichedQueryJson {
       return {
-        endpoint: { datasourceId: "", entityId: "", operation: op },
+        operation: op,
         resource: {
           fields,
         },
-        meta: {
+        table: TABLE,
-          table: TABLE,
+        tables: { [TABLE.name]: TABLE },
-        },
       }
     }
 

packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "READ",
-    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-    "entityId": "persons",
-    "operation": "READ"
-  },
   "resource": {
     "fields": [
       "a.year",
@@ -61,113 +57,111 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
-      "primary": ["personid"],
+    "name": "persons",
-      "name": "persons",
+    "schema": {
-      "schema": {
+      "year": {
-        "year": {
+        "type": "number",
-          "type": "number",
+        "externalType": "integer",
-          "externalType": "integer",
+        "autocolumn": false,
-          "autocolumn": false,
+        "name": "year",
-          "name": "year",
+        "constraints": {
-          "constraints": {
+          "presence": false
-            "presence": false
-          }
-        },
-        "firstname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "firstname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "personid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "personid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "address": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "address",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "age": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": false,
-          "name": "age",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "type": {
-          "type": "options",
-          "externalType": "USER-DEFINED",
-          "autocolumn": false,
-          "name": "type",
-          "constraints": {
-            "presence": false,
-            "inclusion": ["support", "designer", "programmer", "qa"]
-          }
-        },
-        "city": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "city",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "lastname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "lastname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "QA": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "QA",
-          "relationshipType": "many-to-one",
-          "fieldName": "qaid",
-          "type": "link",
-          "main": true,
-          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
-          "foreignKey": "personid"
-        },
-        "executor": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "executor",
-          "relationshipType": "many-to-one",
-          "fieldName": "executorid",
-          "type": "link",
-          "main": true,
-          "_id": "c89530b9770d94bec851e062b5cff3001",
-          "foreignKey": "personid",
-          "tableName": "persons"
         }
       },
-      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "firstname": {
-      "sourceType": "external",
+        "type": "string",
-      "primaryDisplay": "firstname",
+        "externalType": "character varying",
-      "views": {}
+        "autocolumn": false,
-    }
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
   },
   "tableAliases": {
     "persons": "a",

packages/server/src/integrations/tests/sqlQueryJson/createSimple.json

@@ -1,14 +1,7 @@
 {
-  "endpoint": {
+  "operation": "CREATE",
-    "datasourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae",
-    "entityId": "people",
-    "operation": "CREATE"
-  },
   "resource": {
-    "fields": [
+    "fields": ["a.name", "a.age"]
-      "a.name",
-      "a.age"
-    ]
   },
   "filters": {},
   "relationships": [],
@@ -24,41 +17,36 @@
       }
     }
   },
-  "meta": {
+  "table": {
-    "table": {
+    "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people",
-      "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people",
+    "type": "table",
-      "type": "table",
+    "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae",
-      "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae",
+    "sourceType": "external",
-      "sourceType": "external",
+    "primary": ["name", "age"],
-      "primary": [
+    "name": "people",
-        "name",
+    "schema": {
-        "age"
+      "name": {
-      ],
+        "type": "string",
-      "name": "people",
+        "externalType": "varchar",
-      "schema": {
+        "autocolumn": false,
-        "name": {
+        "name": "name",
-          "type": "string",
+        "constraints": {
-          "externalType": "varchar",
+          "presence": true
-          "autocolumn": false,
-          "name": "name",
-          "constraints": {
-            "presence": true
-          }
-        },
-        "age": {
-          "type": "number",
-          "externalType": "int",
-          "autocolumn": false,
-          "name": "age",
-          "constraints": {
-            "presence": false
-          }
         }
       },
-      "primaryDisplay": "name"
+      "age": {
-    }
+        "type": "number",
+        "externalType": "int",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      }
+    },
+    "primaryDisplay": "name"
   },
   "tableAliases": {
     "people": "a"
   }
-}
+}

packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "CREATE",
-    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-    "entityId": "persons",
-    "operation": "CREATE"
-  },
   "resource": {
     "fields": [
       "a.year",
@@ -51,123 +47,114 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
-      "primary": [
+    "name": "persons",
-        "personid"
+    "schema": {
-      ],
+      "year": {
-      "name": "persons",
+        "type": "number",
-      "schema": {
+        "externalType": "integer",
-        "year": {
+        "autocolumn": false,
-          "type": "number",
+        "name": "year",
-          "externalType": "integer",
+        "constraints": {
-          "autocolumn": false,
+          "presence": false
-          "name": "year",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "firstname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "firstname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "personid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "personid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "address": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "address",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "age": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": false,
-          "name": "age",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "type": {
-          "type": "options",
-          "externalType": "USER-DEFINED",
-          "autocolumn": false,
-          "name": "type",
-          "constraints": {
-            "presence": false,
-            "inclusion": [
-              "support",
-              "designer",
-              "programmer",
-              "qa"
-            ]
-          }
-        },
-        "city": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "city",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "lastname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "lastname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "QA": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "QA",
-          "relationshipType": "many-to-one",
-          "fieldName": "qaid",
-          "type": "link",
-          "main": true,
-          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
-          "foreignKey": "personid"
-        },
-        "executor": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "executor",
-          "relationshipType": "many-to-one",
-          "fieldName": "executorid",
-          "type": "link",
-          "main": true,
-          "_id": "c89530b9770d94bec851e062b5cff3001",
-          "foreignKey": "personid",
-          "tableName": "persons"
         }
       },
-      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "firstname": {
-      "sourceType": "external",
+        "type": "string",
-      "primaryDisplay": "firstname",
+        "externalType": "character varying",
-      "views": {}
+        "autocolumn": false,
-    }
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
   },
   "tableAliases": {
     "persons": "a",
     "tasks": "b"
   }
-}
+}

packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json

@@ -1,15 +1,7 @@
 {
-  "endpoint": {
+  "operation": "DELETE",
-    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-    "entityId": "compositetable",
-    "operation": "DELETE"
-  },
   "resource": {
-    "fields": [
+    "fields": ["a.keyparttwo", "a.keypartone", "a.name"]
-      "a.keyparttwo",
-      "a.keypartone",
-      "a.name"
-    ]
   },
   "filters": {
     "equal": {
@@ -26,50 +18,45 @@
       }
     }
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable",
-      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable",
+    "primary": ["keypartone", "keyparttwo"],
-      "primary": [
+    "name": "compositetable",
-        "keypartone",
+    "schema": {
-        "keyparttwo"
+      "keyparttwo": {
-      ],
+        "type": "string",
-      "name": "compositetable",
+        "externalType": "character varying",
-      "schema": {
+        "autocolumn": false,
-        "keyparttwo": {
+        "name": "keyparttwo",
-          "type": "string",
+        "constraints": {
-          "externalType": "character varying",
+          "presence": true
-          "autocolumn": false,
-          "name": "keyparttwo",
-          "constraints": {
-            "presence": true
-          }
-        },
-        "keypartone": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "keypartone",
-          "constraints": {
-            "presence": true
-          }
-        },
-        "name": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "name",
-          "constraints": {
-            "presence": false
-          }
         }
       },
-      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "keypartone": {
-      "sourceType": "external",
+        "type": "string",
-      "primaryDisplay": "keypartone"
+        "externalType": "character varying",
-    }
+        "autocolumn": false,
+        "name": "keypartone",
+        "constraints": {
+          "presence": true
+        }
+      },
+      "name": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "name",
+        "constraints": {
+          "presence": false
+        }
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "keypartone"
   },
   "tableAliases": {
     "compositetable": "a"
   }
-}
+}

packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "READ",
-    "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
-    "entityId": "tasks",
-    "operation": "READ"
-  },
   "resource": {
     "fields": [
       "a.executorid",
@@ -17,10 +13,7 @@
   },
   "filters": {
     "oneOf": {
-      "taskid": [
+      "taskid": [1, 2]
-        1,
-        2
-      ]
     }
   },
   "relationships": [
@@ -42,82 +35,78 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
-      "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
+    "primary": ["taskid"],
-      "primary": [
+    "name": "tasks",
-        "taskid"
+    "schema": {
-      ],
+      "executorid": {
-      "name": "tasks",
+        "type": "number",
-      "schema": {
+        "externalType": "integer",
-        "executorid": {
+        "autocolumn": false,
-          "type": "number",
+        "name": "executorid",
-          "externalType": "integer",
+        "constraints": {
-          "autocolumn": false,
+          "presence": false
-          "name": "executorid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "taskname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "taskname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "taskid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "taskid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "completed": {
-          "type": "boolean",
-          "externalType": "boolean",
-          "autocolumn": false,
-          "name": "completed",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "qaid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": false,
-          "name": "qaid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "products": {
-          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
-          "name": "products",
-          "relationshipType": "many-to-many",
-          "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
-          "type": "link",
-          "_id": "c3b91d00cd36c4cc1a347794725b9adbd",
-          "fieldName": "productid",
-          "throughFrom": "productid",
-          "throughTo": "taskid"
         }
       },
-      "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+      "taskname": {
-      "sourceType": "external",
+        "type": "string",
-      "primaryDisplay": "taskname",
+        "externalType": "character varying",
-      "sql": true,
+        "autocolumn": false,
-      "views": {}
+        "name": "taskname",
-    }
+        "constraints": {
+          "presence": false
+        }
+      },
+      "taskid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "taskid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "completed": {
+        "type": "boolean",
+        "externalType": "boolean",
+        "autocolumn": false,
+        "name": "completed",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "qaid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "qaid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "products": {
+        "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
+        "name": "products",
+        "relationshipType": "many-to-many",
+        "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
+        "type": "link",
+        "_id": "c3b91d00cd36c4cc1a347794725b9adbd",
+        "fieldName": "productid",
+        "throughFrom": "productid",
+        "throughTo": "taskid"
+      }
+    },
+    "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+    "sourceType": "external",
+    "primaryDisplay": "taskname",
+    "sql": true,
+    "views": {}
   },
   "tableAliases": {
     "tasks": "a",
     "products": "b",
     "products_tasks": "c"
   }
-}
+}

packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "READ",
-    "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
-    "entityId": "products",
-    "operation": "READ"
-  },
   "resource": {
     "fields": [
       "a.productname",
@@ -56,48 +52,46 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
-      "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
+    "primary": ["productid"],
-      "primary": ["productid"],
+    "name": "products",
-      "name": "products",
+    "schema": {
-      "schema": {
+      "productname": {
-        "productname": {
+        "type": "string",
-          "type": "string",
+        "externalType": "character varying",
-          "externalType": "character varying",
+        "autocolumn": false,
-          "autocolumn": false,
+        "name": "productname",
-          "name": "productname",
+        "constraints": {
-          "constraints": {
+          "presence": false
-            "presence": false
-          }
-        },
-        "productid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "productid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "tasks": {
-          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
-          "name": "tasks",
-          "relationshipType": "many-to-many",
-          "fieldName": "taskid",
-          "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
-          "throughFrom": "taskid",
-          "throughTo": "productid",
-          "type": "link",
-          "main": true,
-          "_id": "c3b91d00cd36c4cc1a347794725b9adbd"
         }
       },
-      "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+      "productid": {
-      "sourceType": "external",
+        "type": "number",
-      "primaryDisplay": "productname"
+        "externalType": "integer",
-    }
+        "autocolumn": true,
+        "name": "productid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "tasks": {
+        "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
+        "name": "tasks",
+        "relationshipType": "many-to-many",
+        "fieldName": "taskid",
+        "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
+        "throughFrom": "taskid",
+        "throughTo": "productid",
+        "type": "link",
+        "main": true,
+        "_id": "c3b91d00cd36c4cc1a347794725b9adbd"
+      }
+    },
+    "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+    "sourceType": "external",
+    "primaryDisplay": "productname"
   },
   "tableAliases": {
     "products": "a",

packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "READ",
-    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-    "entityId": "products",
-    "operation": "READ"
-  },
   "resource": {
     "fields": [
       "a.productname",
@@ -46,47 +42,45 @@
     "tasks": "b",
     "products": "a"
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products",
-      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products",
+    "primary": ["productid"],
-      "primary": ["productid"],
+    "name": "products",
-      "name": "products",
+    "schema": {
-      "schema": {
+      "productname": {
-        "productname": {
+        "type": "string",
-          "type": "string",
+        "externalType": "character varying",
-          "externalType": "character varying",
+        "autocolumn": false,
-          "autocolumn": false,
+        "name": "productname",
-          "name": "productname",
+        "constraints": {
-          "constraints": {
+          "presence": false
-            "presence": false
-          }
-        },
-        "productid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "productid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "tasks": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "tasks",
-          "relationshipType": "many-to-many",
-          "fieldName": "taskid",
-          "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks",
-          "throughFrom": "taskid",
-          "throughTo": "productid",
-          "type": "link",
-          "main": true,
-          "_id": "ca6862d9ba09146dd8a68e3b5b7055a09"
         }
       },
-      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "productid": {
-      "sourceType": "external",
+        "type": "number",
-      "primaryDisplay": "productname"
+        "externalType": "integer",
-    }
+        "autocolumn": true,
+        "name": "productid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "tasks": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "tasks",
+        "relationshipType": "many-to-many",
+        "fieldName": "taskid",
+        "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks",
+        "throughFrom": "taskid",
+        "throughTo": "productid",
+        "type": "link",
+        "main": true,
+        "_id": "ca6862d9ba09146dd8a68e3b5b7055a09"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "productname"
   }
 }

packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "READ",
-    "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
-    "entityId": "tasks",
-    "operation": "READ"
-  },
   "resource": {
     "fields": [
       "a.executorid",
@@ -102,94 +98,92 @@
   "extra": {
     "idFilter": {}
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
-      "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
+    "primary": ["taskid"],
-      "primary": ["taskid"],
+    "name": "tasks",
-      "name": "tasks",
+    "schema": {
-      "schema": {
+      "executorid": {
-        "executorid": {
+        "type": "number",
-          "type": "number",
+        "externalType": "integer",
-          "externalType": "integer",
+        "name": "executorid",
-          "name": "executorid",
+        "constraints": {
-          "constraints": {
+          "presence": false
-            "presence": false
-          },
-          "autocolumn": true,
-          "autoReason": "foreign_key"
         },
-        "taskname": {
+        "autocolumn": true,
-          "type": "string",
+        "autoReason": "foreign_key"
-          "externalType": "character varying",
+      },
-          "autocolumn": false,
+      "taskname": {
-          "name": "taskname",
+        "type": "string",
-          "constraints": {
+        "externalType": "character varying",
-            "presence": false
+        "autocolumn": false,
-          }
+        "name": "taskname",
-        },
+        "constraints": {
-        "taskid": {
+          "presence": false
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "taskid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "completed": {
-          "type": "boolean",
-          "externalType": "boolean",
-          "autocolumn": false,
-          "name": "completed",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "qaid": {
-          "type": "number",
-          "externalType": "integer",
-          "name": "qaid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "products": {
-          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
-          "name": "products",
-          "relationshipType": "many-to-many",
-          "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
-          "type": "link",
-          "_id": "c3b91d00cd36c4cc1a347794725b9adbd",
-          "fieldName": "productid",
-          "throughFrom": "productid",
-          "throughTo": "taskid"
-        },
-        "tasksToExecute": {
-          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons",
-          "name": "tasksToExecute",
-          "relationshipType": "one-to-many",
-          "type": "link",
-          "_id": "c0f440590bda04f28846242156c1dd60b",
-          "foreignKey": "executorid",
-          "fieldName": "personid"
-        },
-        "tasksToQA": {
-          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons",
-          "name": "tasksToQA",
-          "relationshipType": "one-to-many",
-          "type": "link",
-          "_id": "c5fdf453a0ba743d58e29491d174c974b",
-          "foreignKey": "qaid",
-          "fieldName": "personid"
         }
       },
-      "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+      "taskid": {
-      "sourceType": "external",
+        "type": "number",
-      "primaryDisplay": "taskname",
+        "externalType": "integer",
-      "sql": true,
+        "autocolumn": true,
-      "views": {}
+        "name": "taskid",
-    }
+        "constraints": {
+          "presence": false
+        }
+      },
+      "completed": {
+        "type": "boolean",
+        "externalType": "boolean",
+        "autocolumn": false,
+        "name": "completed",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "qaid": {
+        "type": "number",
+        "externalType": "integer",
+        "name": "qaid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "products": {
+        "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
+        "name": "products",
+        "relationshipType": "many-to-many",
+        "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
+        "type": "link",
+        "_id": "c3b91d00cd36c4cc1a347794725b9adbd",
+        "fieldName": "productid",
+        "throughFrom": "productid",
+        "throughTo": "taskid"
+      },
+      "tasksToExecute": {
+        "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons",
+        "name": "tasksToExecute",
+        "relationshipType": "one-to-many",
+        "type": "link",
+        "_id": "c0f440590bda04f28846242156c1dd60b",
+        "foreignKey": "executorid",
+        "fieldName": "personid"
+      },
+      "tasksToQA": {
+        "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons",
+        "name": "tasksToQA",
+        "relationshipType": "one-to-many",
+        "type": "link",
+        "_id": "c5fdf453a0ba743d58e29491d174c974b",
+        "foreignKey": "qaid",
+        "fieldName": "personid"
+      }
+    },
+    "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+    "sourceType": "external",
+    "primaryDisplay": "taskname",
+    "sql": true,
+    "views": {}
   },
   "tableAliases": {
     "tasks": "a",

packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "UPDATE",
-    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-    "entityId": "persons",
-    "operation": "UPDATE"
-  },
   "resource": {
     "fields": [
       "a.year",
@@ -59,123 +55,114 @@
       }
     }
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
-      "primary": [
+    "name": "persons",
-        "personid"
+    "schema": {
-      ],
+      "year": {
-      "name": "persons",
+        "type": "number",
-      "schema": {
+        "externalType": "integer",
-        "year": {
+        "autocolumn": false,
-          "type": "number",
+        "name": "year",
-          "externalType": "integer",
+        "constraints": {
-          "autocolumn": false,
+          "presence": false
-          "name": "year",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "firstname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "firstname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "personid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "personid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "address": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "address",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "age": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": false,
-          "name": "age",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "type": {
-          "type": "options",
-          "externalType": "USER-DEFINED",
-          "autocolumn": false,
-          "name": "type",
-          "constraints": {
-            "presence": false,
-            "inclusion": [
-              "support",
-              "designer",
-              "programmer",
-              "qa"
-            ]
-          }
-        },
-        "city": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "city",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "lastname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "lastname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "QA": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "QA",
-          "relationshipType": "many-to-one",
-          "fieldName": "qaid",
-          "type": "link",
-          "main": true,
-          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
-          "foreignKey": "personid"
-        },
-        "executor": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "executor",
-          "relationshipType": "many-to-one",
-          "fieldName": "executorid",
-          "type": "link",
-          "main": true,
-          "_id": "c89530b9770d94bec851e062b5cff3001",
-          "foreignKey": "personid",
-          "tableName": "persons"
         }
       },
-      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "firstname": {
-      "sourceType": "external",
+        "type": "string",
-      "primaryDisplay": "firstname",
+        "externalType": "character varying",
-      "views": {}
+        "autocolumn": false,
-    }
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
   },
   "tableAliases": {
     "persons": "a",
     "tasks": "b"
   }
-}
+}

packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json

@@ -1,9 +1,5 @@
 {
-  "endpoint": {
+  "operation": "UPDATE",
-    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-    "entityId": "persons",
-    "operation": "UPDATE"
-  },
   "resource": {
     "fields": [
       "a.year",
@@ -59,123 +55,114 @@
       }
     }
   },
-  "meta": {
+  "table": {
-    "table": {
+    "type": "table",
-      "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
-      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
-      "primary": [
+    "name": "persons",
-        "personid"
+    "schema": {
-      ],
+      "year": {
-      "name": "persons",
+        "type": "number",
-      "schema": {
+        "externalType": "integer",
-        "year": {
+        "autocolumn": false,
-          "type": "number",
+        "name": "year",
-          "externalType": "integer",
+        "constraints": {
-          "autocolumn": false,
+          "presence": false
-          "name": "year",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "firstname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "firstname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "personid": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": true,
-          "name": "personid",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "address": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "address",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "age": {
-          "type": "number",
-          "externalType": "integer",
-          "autocolumn": false,
-          "name": "age",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "type": {
-          "type": "options",
-          "externalType": "USER-DEFINED",
-          "autocolumn": false,
-          "name": "type",
-          "constraints": {
-            "presence": false,
-            "inclusion": [
-              "support",
-              "designer",
-              "programmer",
-              "qa"
-            ]
-          }
-        },
-        "city": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "city",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "lastname": {
-          "type": "string",
-          "externalType": "character varying",
-          "autocolumn": false,
-          "name": "lastname",
-          "constraints": {
-            "presence": false
-          }
-        },
-        "QA": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "QA",
-          "relationshipType": "many-to-one",
-          "fieldName": "qaid",
-          "type": "link",
-          "main": true,
-          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
-          "foreignKey": "personid"
-        },
-        "executor": {
-          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-          "name": "executor",
-          "relationshipType": "many-to-one",
-          "fieldName": "executorid",
-          "type": "link",
-          "main": true,
-          "_id": "c89530b9770d94bec851e062b5cff3001",
-          "foreignKey": "personid",
-          "tableName": "persons"
         }
       },
-      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "firstname": {
-      "sourceType": "external",
+        "type": "string",
-      "primaryDisplay": "firstname",
+        "externalType": "character varying",
-      "views": {}
+        "autocolumn": false,
-    }
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
   },
   "tableAliases": {
     "persons": "a",
     "tasks": "b"
   }
-}
+}

packages/server/src/integrations/tests/utils/index.ts

@@ -102,6 +102,9 @@ function createDummyTest() {
 }
 
 export function datasourceDescribe(opts: DatasourceDescribeOpts) {
+  // tests that call this need a lot longer timeouts
+  jest.setTimeout(120000)
+
   if (process.env.DATASOURCE === "none") {
     createDummyTest()
   }

packages/server/src/integrations/tests/utils/oracle.ts

@@ -25,7 +25,7 @@ export async function getDatasource(): Promise<Datasource> {
         })
         .withWaitStrategy(
           Wait.forLogMessage("DATABASE IS READY TO USE!").withStartupTimeout(
-            20000
+            60000
           )
         )
     )

packages/server/src/middleware/zod-validator.ts

@@ -0,0 +1,43 @@
+import { features } from "@budibase/backend-core"
+import { Ctx, FeatureFlag } from "@budibase/types"
+
+import { AnyZodObject } from "zod"
+import { fromZodError } from "zod-validation-error"
+
+function validate(schema: AnyZodObject, property: "body" | "params") {
+  // Return a Koa middleware function
+  return async (ctx: Ctx, next: any) => {
+    if (!(await features.flags.isEnabled(FeatureFlag.USE_ZOD_VALIDATOR))) {
+      return next()
+    }
+
+    if (!schema) {
+      return next()
+    }
+    let params = null
+    let setClean: ((data: any) => void) | undefined
+    if (ctx[property] != null) {
+      params = ctx[property]
+      setClean = data => (ctx[property] = data)
+    } else if (property === "body" && ctx.request[property] != null) {
+      params = ctx.request[property]
+      setClean = data => (ctx.request[property] = data)
+    } else if (property === "params") {
+      params = ctx.request.query
+      setClean = data => (ctx.request.query = data)
+    }
+
+    const result = schema.safeParse(params)
+    if (!result.success) {
+      ctx.throw(400, fromZodError(result.error))
+    } else {
+      setClean?.(result.data)
+    }
+
+    return next()
+  }
+}
+
+export function validateBody(schema: AnyZodObject) {
+  return validate(schema, "body")
+}

packages/server/src/migrations/functions/backfill/global.ts

@@ -9,11 +9,12 @@ import {
   db as dbUtils,
 } from "@budibase/backend-core"
 import {
-  QuotaUsage,
-  CloudAccount,
   App,
-  TenantBackfillSucceededEvent,
+  CloudAccount,
   Event,
+  Hosting,
+  QuotaUsage,
+  TenantBackfillSucceededEvent,
   User,
 } from "@budibase/types"
 import env from "../../../environment"
@@ -125,7 +126,7 @@ export const run = async (db: any) => {
     try {
       await events.identification.identifyTenantGroup(
         tenantId,
-        account,
+        env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD,
         timestamp
       )
     } catch (e) {

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -1,8 +1,8 @@
 import {
   Aggregation,
   CalculationType,
-  Datasource,
   DocumentType,
+  EnrichedQueryJson,
   FieldType,
   isLogicalSearchOperator,
   Operation,
@@ -38,7 +38,7 @@ import { generateJunctionTableID } from "../../../../../db/utils"
 import AliasTables from "../../sqlAlias"
 import { outputProcessing } from "../../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
-import { processRowCountResponse } from "../../utils"
+import { enrichQueryJson, processRowCountResponse } from "../../utils"
 import {
   dataFilters,
   helpers,
@@ -180,19 +180,6 @@ function cleanupFilters(filters: SearchFilters, allTables: Table[]) {
   return filters
 }
 
-function buildTableMap(tables: Table[]) {
-  const tableMap: Record<string, Table> = {}
-  for (let table of tables) {
-    // update the table name, should never query by name for SQLite
-    table.originalName = table.name
-    table.name = table._id!
-    // need a primary for sorting, lookups etc
-    table.primary = ["_id"]
-    tableMap[table._id!] = table
-  }
-  return tableMap
-}
-
 // table is only needed to handle relationships
 function reverseUserColumnMapping(rows: Row[], table?: Table) {
   const prefixLength = USER_COLUMN_PREFIX.length
@@ -223,30 +210,30 @@ function reverseUserColumnMapping(rows: Row[], table?: Table) {
 }
 
 function runSqlQuery(
-  json: QueryJson,
+  json: EnrichedQueryJson,
   tables: Table[],
   relationships: RelationshipsJson[]
 ): Promise<Row[]>
 function runSqlQuery(
-  json: QueryJson,
+  json: EnrichedQueryJson,
   tables: Table[],
   relationships: RelationshipsJson[],
   opts: { countTotalRows: true }
 ): Promise<number>
 async function runSqlQuery(
-  json: QueryJson,
+  json: EnrichedQueryJson,
   tables: Table[],
   relationships: RelationshipsJson[],
   opts?: { countTotalRows?: boolean }
 ) {
   const relationshipJunctionTableIds = relationships.map(rel => rel.through!)
   const alias = new AliasTables(
-    tables.map(table => table.name).concat(relationshipJunctionTableIds)
+    tables.map(table => table._id!).concat(relationshipJunctionTableIds)
   )
   if (opts?.countTotalRows) {
-    json.endpoint.operation = Operation.COUNT
+    json.operation = Operation.COUNT
   }
-  const processSQLQuery = async (_: Datasource, json: QueryJson) => {
+  const processSQLQuery = async (json: EnrichedQueryJson) => {
     const query = builder._query(json, {
       disableReturning: true,
     })
@@ -281,7 +268,7 @@ async function runSqlQuery(
   if (opts?.countTotalRows) {
     return processRowCountResponse(response)
   } else if (Array.isArray(response)) {
-    return reverseUserColumnMapping(response, json.meta.table)
+    return reverseUserColumnMapping(response, json.table)
   }
   return response
 }
@@ -315,7 +302,11 @@ export async function search(
   }
 
   const allTables = await sdk.tables.getAllInternalTables()
-  const allTablesMap = buildTableMap(allTables)
+  const allTablesMap = allTables.reduce((acc, table) => {
+    acc[table._id!] = table
+    return acc
+  }, {} as Record<string, Table>)
+
   // make sure we have the mapped/latest table
   if (table._id) {
     table = allTablesMap[table._id]
@@ -372,10 +363,7 @@ export async function search(
       operation: Operation.READ,
     },
     filters: searchFilters,
-    table,
     meta: {
-      table,
-      tables: allTablesMap,
       columnPrefix: USER_COLUMN_PREFIX,
     },
     resource: {
@@ -427,11 +415,13 @@ export async function search(
     }
   }
 
+  const enrichedRequest = await enrichQueryJson(request)
+
   try {
     const [rows, totalRows] = await Promise.all([
-      runSqlQuery(request, allTables, relationships),
+      runSqlQuery(enrichedRequest, allTables, relationships),
       options.countRows
-        ? runSqlQuery(request, allTables, relationships, {
+        ? runSqlQuery(enrichedRequest, allTables, relationships, {
             countTotalRows: true,
           })
         : Promise.resolve(undefined),

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -1,22 +1,19 @@
 import {
   Datasource,
   DatasourcePlusQueryResponse,
+  EnrichedQueryJson,
   Operation,
-  QueryJson,
   Row,
   SearchFilters,
   SqlClient,
+  Table,
 } from "@budibase/types"
-import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
-import datasources from "../datasources"
-import { BudibaseInternalDB } from "../../../db/utils"
 import { dataFilters } from "@budibase/shared-core"
 
 type PerformQueryFunction = (
-  datasource: Datasource,
+  json: EnrichedQueryJson
-  json: QueryJson
 ) => Promise<DatasourcePlusQueryResponse>
 
 const WRITE_OPERATIONS: Operation[] = [
@@ -71,13 +68,12 @@ export default class AliasTables {
     this.charSeq = new CharSequence()
   }
 
-  isAliasingEnabled(json: QueryJson, datasource?: Datasource) {
+  isAliasingEnabled(json: EnrichedQueryJson, datasource?: Datasource) {
-    const operation = json.endpoint.operation
     const fieldLength = json.resource?.fields?.length
     if (
       !fieldLength ||
       fieldLength <= 0 ||
-      DISABLED_OPERATIONS.includes(operation)
+      DISABLED_OPERATIONS.includes(json.operation)
     ) {
       return false
     }
@@ -87,7 +83,7 @@ export default class AliasTables {
     }
     try {
       const sqlClient = getSQLClient(datasource)
-      const isWrite = WRITE_OPERATIONS.includes(operation)
+      const isWrite = WRITE_OPERATIONS.includes(json.operation)
       const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient)
       if (isWrite && isDisabledClient) {
         return false
@@ -99,7 +95,11 @@ export default class AliasTables {
     return true
   }
 
-  getAlias(tableName: string) {
+  getAlias(tableName: string | Table) {
+    if (typeof tableName === "object") {
+      tableName = tableName.name
+    }
+
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
     }
@@ -177,17 +177,15 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(
-    json: QueryJson,
+    json: EnrichedQueryJson,
     queryFn: PerformQueryFunction
   ): Promise<DatasourcePlusQueryResponse> {
-    const datasourceId = json.endpoint.datasourceId
+    const datasource = json.datasource
-    const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
+    const isSqs = datasource === undefined
-    let aliasingEnabled: boolean, datasource: Datasource
+    let aliasingEnabled: boolean
     if (isSqs) {
       aliasingEnabled = this.isAliasingEnabled(json)
-      datasource = BudibaseInternalDB
     } else {
-      datasource = await datasources.get(datasourceId)
       aliasingEnabled = this.isAliasingEnabled(json, datasource)
     }
 
@@ -215,24 +213,23 @@ export default class AliasTables {
         }
         json.filters = aliasFilters(json.filters)
       }
-      if (json.meta?.table) {
+
-        this.getAlias(json.meta.table.name)
-      }
-      if (json.meta?.tables) {
-        Object.keys(json.meta.tables).forEach(tableName =>
-          this.getAlias(tableName)
-        )
-      }
       if (json.relationships) {
         json.relationships = json.relationships.map(relationship => ({
           ...relationship,
           aliases: this.aliasMap([
             relationship.through,
             relationship.tableName,
-            json.endpoint.entityId,
+            json.table.name,
           ]),
         }))
       }
+
+      this.getAlias(json.table)
+      for (const tableName of Object.keys(json.tables)) {
+        this.getAlias(tableName)
+      }
+
       // invert and return
       const invertedTableAliases: Record<string, string> = {}
       for (let [key, value] of Object.entries(this.tableAliases)) {
@@ -241,7 +238,7 @@ export default class AliasTables {
       json.tableAliases = invertedTableAliases
     }
 
-    let response: DatasourcePlusQueryResponse = await queryFn(datasource, json)
+    let response = await queryFn(json)
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {

packages/server/src/sdk/app/rows/utils.ts

@@ -14,13 +14,13 @@ import {
   SqlClient,
   ArrayOperator,
   ViewV2,
+  EnrichedQueryJson,
 } from "@budibase/types"
-import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
-import { docIds, sql } from "@budibase/backend-core"
+import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
 import env from "../../../environment"
 
@@ -73,18 +73,58 @@ export function processRowCountResponse(
   }
 }
 
-export async function getDatasourceAndQuery(
+function processInternalTables(tables: Table[]) {
-  json: QueryJson
+  const tableMap: Record<string, Table> = {}
-): Promise<DatasourcePlusQueryResponse> {
+  for (let table of tables) {
-  const datasourceId = json.endpoint.datasourceId
+    // update the table name, should never query by name for SQLite
-  const datasource = await sdk.datasources.get(datasourceId)
+    table.originalName = table.name
-  const table = datasource.entities?.[json.endpoint.entityId]
+    table.name = table._id!
-  if (!json.meta && table) {
+    tableMap[table._id!] = table
-    json.meta = {
+  }
-      table,
+  return tableMap
-    }
+}
+
+export async function enrichQueryJson(
+  json: QueryJson
+): Promise<EnrichedQueryJson> {
+  let datasource: Datasource | undefined = undefined
+
+  if (typeof json.endpoint.datasourceId === "string") {
+    if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) {
+      datasource = await sdk.datasources.get(json.endpoint.datasourceId, {
+        enriched: true,
+      })
+    }
+  } else {
+    datasource = await sdk.datasources.enrich(json.endpoint.datasourceId)
+  }
+
+  let tables: Record<string, Table>
+  if (datasource) {
+    tables = datasource.entities || {}
+  } else {
+    tables = processInternalTables(await sdk.tables.getAllInternalTables())
+  }
+
+  let table: Table
+  if (typeof json.endpoint.entityId === "string") {
+    let entityId = json.endpoint.entityId
+    if (docIds.isDatasourceId(entityId)) {
+      entityId = sql.utils.breakExternalTableId(entityId).tableName
+    }
+    table = tables[entityId]
+  } else {
+    table = json.endpoint.entityId
+  }
+
+  return {
+    operation: json.endpoint.operation,
+    table,
+    tables,
+    datasource,
+    schema: json.endpoint.schema,
+    ...json,
   }
-  return makeExternalQuery(datasource, json)
 }
 
 export function cleanExportRows(

packages/server/src/sdk/app/tables/external/index.ts

@@ -245,7 +245,6 @@ export async function save(
     datasource,
     operation,
     tableToSave,
-    tables,
     oldTable,
     opts?.renaming
   )
@@ -253,7 +252,7 @@ export async function save(
   for (let extraTable of extraTablesToUpdate) {
     const oldExtraTable = oldTables[extraTable.name]
     let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
-    await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
+    await makeTableRequest(datasource, op, extraTable, oldExtraTable)
   }
 
   // make sure the constrained list, all still exist
@@ -292,7 +291,7 @@ export async function destroy(datasourceId: string, table: Table) {
 
   const operation = Operation.DELETE_TABLE
   if (tables) {
-    await makeTableRequest(datasource, operation, table, tables)
+    await makeTableRequest(datasource, operation, table)
     cleanupRelationships(table, tables, { deleting: true })
     delete tables[table.name]
     datasource.entities = tables

packages/server/src/sdk/app/tables/getters.ts

@@ -46,6 +46,7 @@ export async function processTable(table: Table): Promise<Table> {
     const processed: Table = {
       ...table,
       type: "table",
+      primary: ["_id"], // internal tables must always use _id as primary key
       sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
       sourceType: TableSourceType.INTERNAL,
       sql: true,

packages/server/src/tests/utilities/api/datasource.ts

@@ -1,14 +1,16 @@
 import {
-  Datasource,
-  VerifyDatasourceRequest,
-  CreateDatasourceResponse,
-  UpdateDatasourceResponse,
-  UpdateDatasourceRequest,
-  QueryJson,
   BuildSchemaFromSourceResponse,
+  CreateDatasourceResponse,
+  Datasource,
   FetchDatasourceInfoResponse,
+  FieldType,
+  RelationshipType,
+  UpdateDatasourceRequest,
+  UpdateDatasourceResponse,
+  VerifyDatasourceRequest,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
+import { sql } from "@budibase/backend-core"
 
 export class DatasourceAPI extends TestAPI {
   create = async (
@@ -66,16 +68,6 @@ export class DatasourceAPI extends TestAPI {
     return await this._get<Datasource[]>(`/api/datasources`, { expectations })
   }
 
-  query = async (
-    query: Omit<QueryJson, "meta"> & Partial<Pick<QueryJson, "meta">>,
-    expectations?: Expectations
-  ) => {
-    return await this._post<any>(`/api/datasources/query`, {
-      body: query,
-      expectations,
-    })
-  }
-
   fetchSchema = async (
     {
       datasourceId,
@@ -103,4 +95,50 @@ export class DatasourceAPI extends TestAPI {
       }
     )
   }
+
+  addExistingRelationship = async (
+    {
+      one,
+      many,
+    }: {
+      one: { tableId: string; relationshipName: string; foreignKey: string }
+      many: { tableId: string; relationshipName: string; primaryKey: string }
+    },
+    expectations?: Expectations
+  ) => {
+    const oneTableInfo = sql.utils.breakExternalTableId(one.tableId),
+      manyTableInfo = sql.utils.breakExternalTableId(many.tableId)
+    if (oneTableInfo.datasourceId !== manyTableInfo.datasourceId) {
+      throw new Error(
+        "Tables are in different datasources, cannot create relationship."
+      )
+    }
+    const datasource = await this.get(oneTableInfo.datasourceId)
+    const oneTable = datasource.entities?.[oneTableInfo.tableName],
+      manyTable = datasource.entities?.[manyTableInfo.tableName]
+    if (!oneTable || !manyTable) {
+      throw new Error(
+        "Both tables not found in datasource, cannot create relationship."
+      )
+    }
+
+    manyTable.schema[many.relationshipName] = {
+      type: FieldType.LINK,
+      name: many.relationshipName,
+      tableId: oneTable._id!,
+      relationshipType: RelationshipType.MANY_TO_ONE,
+      fieldName: one.foreignKey,
+      foreignKey: many.primaryKey,
+      main: true,
+    }
+    oneTable.schema[one.relationshipName] = {
+      type: FieldType.LINK,
+      name: one.relationshipName,
+      tableId: manyTable._id!,
+      relationshipType: RelationshipType.ONE_TO_MANY,
+      fieldName: many.primaryKey,
+      foreignKey: one.foreignKey,
+    }
+    return await this.update(datasource, expectations)
+  }
 }

packages/server/src/threads/automation.ts

@@ -392,6 +392,7 @@ class Orchestrator {
 
     let iterationCount = 0
     let shouldCleanup = true
+    let reachedMaxIterations = false
 
     for (let loopStepIndex = 0; loopStepIndex < iterations; loopStepIndex++) {
       try {
@@ -419,19 +420,8 @@ class Orchestrator {
         loopStepIndex === env.AUTOMATION_MAX_ITERATIONS ||
         (loopStep.inputs.iterations && loopStepIndex === maxIterations)
       ) {
-        this.updateContextAndOutput(
+        reachedMaxIterations = true
-          pathStepIdx + 1,
+        shouldCleanup = true
-          steps[stepToLoopIndex],
-          {
-            items: this.loopStepOutputs,
-            iterations: loopStepIndex,
-          },
-          {
-            status: AutomationErrors.MAX_ITERATIONS,
-            success: true,
-          }
-        )
-        shouldCleanup = false
         break
       }
 
@@ -485,6 +475,10 @@ class Orchestrator {
               iterations: iterationCount,
             }
 
+      if (reachedMaxIterations && iterations !== 0) {
+        tempOutput.status = AutomationStepStatus.MAX_ITERATIONS
+      }
+
       // Loop Step clean up
       this.executionOutput.steps.splice(pathStepIdx, 0, {
         id: steps[stepToLoopIndex].id,

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -153,10 +153,10 @@ export async function processOutputBBReference(
 }
 
 export async function processOutputBBReferences(
-  value: string | null | undefined,
+  value: string | string[] | null | undefined,
   subtype: BBReferenceFieldSubType
 ): Promise<UserReferenceInfo[] | undefined> {
-  if (!value) {
+  if (!value || (Array.isArray(value) && value.length === 0)) {
     return undefined
   }
   const ids =

packages/server/src/utilities/rowProcessor/index.ts

@@ -29,6 +29,8 @@ import {
 import { isExternalTableID } from "../../integrations/utils"
 import {
   helpers,
+  isExternalColumnName,
+  isInternalColumnName,
   PROTECTED_EXTERNAL_COLUMNS,
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
@@ -200,14 +202,17 @@ export async function inputProcessing(
   const clonedRow = cloneDeep(row)
   const table = await getTableFromSource(source)
 
-  const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
   for (const [key, value] of Object.entries(clonedRow)) {
     const field = table.schema[key]
+    const isBuiltinColumn = isExternalTableID(table._id!)
+      ? isExternalColumnName(key)
+      : isInternalColumnName(key)
     // cleanse fields that aren't in the schema
+    if (!field && !isBuiltinColumn) {
+      delete clonedRow[key]
+    }
+    // field isn't found - might be a built-in column, skip over it
     if (!field) {
-      if (dontCleanseKeys.indexOf(key) === -1) {
-        delete clonedRow[key]
-      }
       continue
     }
     // remove any formula values, they are to be generated

packages/shared-core/src/constants/rows.ts

@@ -12,3 +12,7 @@ export const PROTECTED_EXTERNAL_COLUMNS = ["_id", "_rev", "tableId"] as const
 export function isInternalColumnName(name: string): boolean {
   return (PROTECTED_INTERNAL_COLUMNS as readonly string[]).includes(name)
 }
+
+export function isExternalColumnName(name: string): boolean {
+  return (PROTECTED_EXTERNAL_COLUMNS as readonly string[]).includes(name)
+}

packages/types/package.json

@@ -20,7 +20,8 @@
     "@types/redlock": "4.0.7",
     "rimraf": "3.0.2",
     "typescript": "5.5.2",
-    "koa-useragent": "^4.1.0"
+    "koa-useragent": "^4.1.0",
+    "zod": "^3.23.8"
   },
   "dependencies": {
     "scim-patch": "^0.8.1"

packages/types/src/api/web/app/rows.ts

@@ -1,49 +0,0 @@
-import { SearchFilters, RowSearchParams } from "../../../sdk"
-import { Row } from "../../../documents"
-import { PaginationResponse, SortOrder } from "../../../api"
-import { ReadStream } from "fs"
-
-export interface SaveRowRequest extends Row {}
-
-export interface PatchRowRequest extends Row {
-  _id: string
-  _rev: string
-  tableId: string
-}
-
-export interface PatchRowResponse extends Row {}
-
-export interface SearchRowRequest extends Omit<RowSearchParams, "tableId"> {}
-
-export interface SearchViewRowRequest
-  extends Pick<
-    SearchRowRequest,
-    | "sort"
-    | "sortOrder"
-    | "sortType"
-    | "limit"
-    | "bookmark"
-    | "paginate"
-    | "query"
-    | "countRows"
-  > {}
-
-export interface SearchRowResponse {
-  rows: any[]
-}
-
-export interface PaginatedSearchRowResponse
-  extends SearchRowResponse,
-    PaginationResponse {}
-
-export interface ExportRowsRequest {
-  rows?: string[]
-  columns?: string[]
-  query?: SearchFilters
-  sort?: string
-  sortOrder?: SortOrder
-  delimiter?: string
-  customHeaders?: { [key: string]: string }
-}
-
-export type ExportRowsResponse = ReadStream

packages/types/src/api/web/app/rows/index.ts

@@ -0,0 +1,28 @@
+import { SearchFilters } from "../../../../sdk"
+import { Row } from "../../../../documents"
+import { SortOrder } from "../../../../api/web/pagination"
+import { ReadStream } from "fs"
+
+export * from "./search"
+
+export interface SaveRowRequest extends Row {}
+
+export interface PatchRowRequest extends Row {
+  _id: string
+  _rev: string
+  tableId: string
+}
+
+export interface PatchRowResponse extends Row {}
+
+export interface ExportRowsRequest {
+  rows?: string[]
+  columns?: string[]
+  query?: SearchFilters
+  sort?: string
+  sortOrder?: SortOrder
+  delimiter?: string
+  customHeaders?: { [key: string]: string }
+}
+
+export type ExportRowsResponse = ReadStream

packages/types/src/api/web/app/rows/search.ts

@@ -0,0 +1,100 @@
+import {
+  ArrayOperator,
+  BasicOperator,
+  EmptyFilterOption,
+  InternalSearchFilterOperator,
+  LogicalOperator,
+  RangeOperator,
+  SearchFilterKey,
+} from "../../../../sdk"
+import { Row } from "../../../../documents"
+import {
+  PaginationResponse,
+  SortOrder,
+  SortType,
+} from "../../../../api/web/pagination"
+import { z } from "zod"
+
+const fieldKey = z
+  .string()
+  .refine(s => s !== InternalSearchFilterOperator.COMPLEX_ID_OPERATOR, {
+    message: `Key '${InternalSearchFilterOperator.COMPLEX_ID_OPERATOR}' is not allowed`,
+  })
+
+const stringBasicFilter = z.record(fieldKey, z.string())
+const basicFilter = z.record(fieldKey, z.any())
+const arrayFilter = z.record(fieldKey, z.union([z.any().array(), z.string()]))
+const logicFilter = z.lazy(() =>
+  z.object({
+    conditions: z.array(z.object(queryFilterValidation)),
+  })
+)
+
+const stringOrNumber = z.union([z.string(), z.number()])
+
+const queryFilterValidation: Record<SearchFilterKey, z.ZodTypeAny> = {
+  [BasicOperator.STRING]: stringBasicFilter.optional(),
+  [BasicOperator.FUZZY]: stringBasicFilter.optional(),
+  [RangeOperator.RANGE]: z
+    .record(
+      fieldKey,
+      z.union([
+        z.object({ high: stringOrNumber, low: stringOrNumber }),
+        z.object({ high: stringOrNumber }),
+        z.object({ low: stringOrNumber }),
+      ])
+    )
+    .optional(),
+  [BasicOperator.EQUAL]: basicFilter.optional(),
+  [BasicOperator.NOT_EQUAL]: basicFilter.optional(),
+  [BasicOperator.EMPTY]: basicFilter.optional(),
+  [BasicOperator.NOT_EMPTY]: basicFilter.optional(),
+  [ArrayOperator.ONE_OF]: arrayFilter.optional(),
+  [ArrayOperator.CONTAINS]: arrayFilter.optional(),
+  [ArrayOperator.NOT_CONTAINS]: arrayFilter.optional(),
+  [ArrayOperator.CONTAINS_ANY]: arrayFilter.optional(),
+  [LogicalOperator.AND]: logicFilter.optional(),
+  [LogicalOperator.OR]: logicFilter.optional(),
+}
+
+const searchRowRequest = z.object({
+  query: z
+    .object({
+      allOr: z.boolean().optional(),
+      onEmptyFilter: z.nativeEnum(EmptyFilterOption).optional(),
+      ...queryFilterValidation,
+    })
+    .optional(),
+  paginate: z.boolean().optional(),
+  bookmark: z.union([z.string(), z.number()]).nullish(),
+  limit: z.number().optional(),
+  sort: z.string().nullish(),
+  sortOrder: z.nativeEnum(SortOrder).optional(),
+  sortType: z.nativeEnum(SortType).nullish(),
+  version: z.string().optional(),
+  disableEscaping: z.boolean().optional(),
+  countRows: z.boolean().optional(),
+})
+
+export const searchRowRequestValidator = searchRowRequest
+
+export type SearchRowRequest = z.infer<typeof searchRowRequest>
+export type SearchViewRowRequest = Pick<
+  SearchRowRequest,
+  | "sort"
+  | "sortOrder"
+  | "sortType"
+  | "limit"
+  | "bookmark"
+  | "paginate"
+  | "query"
+  | "countRows"
+>
+
+export interface SearchRowResponse {
+  rows: Row[]
+}
+
+export interface PaginatedSearchRowResponse
+  extends SearchRowResponse,
+    PaginationResponse {}

packages/types/src/documents/app/automation/automation.ts

@@ -174,6 +174,7 @@ export enum AutomationFeature {
 
 export enum AutomationStepStatus {
   NO_ITERATIONS = "no_iterations",
+  MAX_ITERATIONS = "max_iterations_reached",
 }
 
 export enum AutomationStatus {

packages/types/src/documents/global/quotas.ts

@@ -44,7 +44,6 @@ export interface StaticUsage {
 export interface MonthlyUsage {
   [MonthlyQuotaName.QUERIES]: number
   [MonthlyQuotaName.AUTOMATIONS]: number
-  [MonthlyQuotaName.DAY_PASSES]: number
   [MonthlyQuotaName.BUDIBASE_AI_CREDITS]: number
   triggers: {
     [key in MonthlyQuotaName]?: QuotaTriggers

packages/types/src/documents/global/user.ts

@@ -62,7 +62,6 @@ export interface User extends Document {
   password?: string
   status?: UserStatus
   createdAt?: number // override the default createdAt behaviour - users sdk historically set this to Date.now()
-  dayPassRecordedAt?: string
   userGroups?: string[]
   onboardedAt?: string
   freeTrialConfirmedAt?: string

packages/types/src/sdk/datasources.ts

@@ -1,5 +1,5 @@
 import { Table, Row } from "../documents"
-import { QueryJson } from "./search"
+import { EnrichedQueryJson } from "./search"
 
 export const PASSWORD_REPLACEMENT = "--secret-value--"
 
@@ -207,7 +207,7 @@ export interface DatasourcePlus extends IntegrationBase {
   // this returns the format of the identifier
   getBindingIdentifier(): string
   getStringConcat(parts: string[]): string
-  query(json: QueryJson): Promise<DatasourcePlusQueryResponse>
+  query(json: EnrichedQueryJson): Promise<DatasourcePlusQueryResponse>
   buildSchema(
     datasourceId: string,
     entities: Record<string, Table>

packages/types/src/sdk/featureFlag.ts

@@ -1,11 +1,9 @@
 export enum FeatureFlag {
-  PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE",
-  PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
   AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
   AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
   DEFAULT_VALUES = "DEFAULT_VALUES",
-
   BUDIBASE_AI = "BUDIBASE_AI",
+  USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
 }
 
 export interface TenantFeatureFlags {

packages/types/src/sdk/licensing/plan.ts

@@ -38,7 +38,6 @@ export interface AvailablePrice {
 export enum PlanModel {
   PER_USER = "perUser",
   PER_CREATOR_PER_USER = "per_creator_per_user",
-  DAY_PASS = "dayPass",
 }
 
 export interface PurchasedPlan {
@@ -49,7 +48,6 @@ export interface PurchasedPlan {
 }
 
 export interface PurchasedPrice extends AvailablePrice {
-  dayPasses: number | undefined
   /** @deprecated - now at the plan level via model */
   isPerUser: boolean
 }

packages/types/src/sdk/licensing/quota.ts

@@ -23,7 +23,6 @@ export enum StaticQuotaName {
 export enum MonthlyQuotaName {
   QUERIES = "queries",
   AUTOMATIONS = "automations",
-  DAY_PASSES = "dayPasses",
   BUDIBASE_AI_CREDITS = "budibaseAICredits",
 }
 
@@ -63,7 +62,6 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined }
 export type MonthlyQuotas = {
   [MonthlyQuotaName.QUERIES]: Quota
   [MonthlyQuotaName.AUTOMATIONS]: Quota
-  [MonthlyQuotaName.DAY_PASSES]: Quota
   [MonthlyQuotaName.BUDIBASE_AI_CREDITS]: Quota
 }
 

packages/types/src/sdk/search.ts

@@ -1,5 +1,5 @@
 import { Operation } from "./datasources"
-import { Row, Table, DocumentType } from "../documents"
+import { Row, DocumentType, Table, Datasource } from "../documents"
 import { SortOrder, SortType } from "../api"
 import { Knex } from "knex"
 import { Aggregation } from "./row"
@@ -158,8 +158,8 @@ export interface ManyToManyRelationshipJson {
 
 export interface QueryJson {
   endpoint: {
-    datasourceId: string
+    datasourceId: string | Datasource
-    entityId: string
+    entityId: string | Table
     operation: Operation
     schema?: string
   }
@@ -171,11 +171,9 @@ export interface QueryJson {
   sort?: SortJson
   paginate?: PaginationJson
   body?: Row | Row[]
-  table?: Table
+  meta?: {
-  meta: {
-    table: Table
-    tables?: Record<string, Table>
     renamed?: RenameColumn
+    oldTable?: Table
     // can specify something that columns could be prefixed with
     columnPrefix?: string
   }
@@ -186,6 +184,14 @@ export interface QueryJson {
   tableAliases?: Record<string, string>
 }
 
+export interface EnrichedQueryJson extends Omit<QueryJson, "endpoint"> {
+  operation: Operation
+  table: Table
+  tables: Record<string, Table>
+  datasource?: Datasource
+  schema?: string
+}
+
 export interface QueryOptions {
   disableReturning?: boolean
   disableBindings?: boolean

packages/worker/src/api/controllers/global/users.ts

@@ -6,12 +6,12 @@ import {
   AddSSoUserRequest,
   BulkUserRequest,
   BulkUserResponse,
-  CloudAccount,
   CreateAdminUserRequest,
   CreateAdminUserResponse,
   Ctx,
   DeleteInviteUserRequest,
   DeleteInviteUsersRequest,
+  Hosting,
   InviteUserRequest,
   InviteUsersRequest,
   InviteUsersResponse,
@@ -26,7 +26,6 @@ import {
   UserIdentifier,
 } from "@budibase/types"
 import {
-  accounts,
   users,
   cache,
   ErrorCode,
@@ -192,12 +191,10 @@ export const adminUser = async (
         lastName: familyName,
       })
 
-      // events
+      await events.identification.identifyTenantGroup(
-      let account: CloudAccount | undefined
+        tenantId,
-      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+        env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
-        account = await accounts.getAccountByTenantId(tenantId)
+      )
-      }
-      await events.identification.identifyTenantGroup(tenantId, account)
 
       ctx.body = {
         _id: finalUser._id!,