From 8c61359b9d940b6f4eaa6d4840d7f53c2e199066 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Thu, 3 Aug 2023 17:23:15 +0100
Subject: [PATCH 01/88] Allow user specified type casting in MySQL queries

---
 .vscode/launch.json                           | 72 ++++++++-----------
 packages/server/package.json                  |  2 +-
 .../server/src/api/controllers/query/index.ts |  4 +-
 .../tests/{filter.spec.js => filter.spec.ts}  | 23 +++---
 packages/server/src/integrations/mysql.ts     | 11 ++-
 packages/server/src/threads/definitions.ts    |  6 ++
 packages/server/src/threads/query.ts          | 31 +++++++-
 7 files changed, 94 insertions(+), 55 deletions(-)
 rename packages/server/src/automations/tests/{filter.spec.js => filter.spec.ts} (76%)

diff --git a/.vscode/launch.json b/.vscode/launch.json
index 8cb49d5825..6c0089bb6b 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -1,42 +1,32 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Budibase Server",
-            "type": "node",
-            "request": "launch",
-            "runtimeArgs": [
-                "--nolazy",
-                "-r",
-                "ts-node/register/transpile-only"
-            ],
-            "args": [
-                "${workspaceFolder}/packages/server/src/index.ts"
-            ],
-            "cwd": "${workspaceFolder}/packages/server"
-        },
-        {
-            "name": "Budibase Worker",
-            "type": "node",
-            "request": "launch",
-            "runtimeArgs": [
-                "--nolazy",
-                "-r",
-                "ts-node/register/transpile-only"
-            ],
-            "args": [
-                "${workspaceFolder}/packages/worker/src/index.ts"
-            ],
-            "cwd": "${workspaceFolder}/packages/worker"
-        },
-    ],
-    "compounds": [
-        {
-          "name": "Start Budibase",
-          "configurations": ["Budibase Server", "Budibase Worker"]
-        }
-    ]
-}
\ No newline at end of file
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Budibase Server",
+      "type": "node",
+      "request": "launch",
+      "runtimeVersion": "14.20.1",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+      "args": ["${workspaceFolder}/packages/server/src/index.ts"],
+      "cwd": "${workspaceFolder}/packages/server"
+    },
+    {
+      "name": "Budibase Worker",
+      "type": "node",
+      "request": "launch",
+      "runtimeVersion": "14.20.1",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+      "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
+      "cwd": "${workspaceFolder}/packages/worker"
+    }
+  ],
+  "compounds": [
+    {
+      "name": "Start Budibase",
+      "configurations": ["Budibase Server", "Budibase Worker"]
+    }
+  ]
+}
diff --git a/packages/server/package.json b/packages/server/package.json
index e112c1f572..aedab54a81 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -100,7 +100,7 @@
     "memorystream": "0.3.1",
     "mongodb": "5.7",
     "mssql": "9.1.1",
-    "mysql2": "2.3.3",
+    "mysql2": "3.5.2",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
     "open": "8.4.0",
diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts
index c32e17bd92..8e13041bf4 100644
--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@@ -120,7 +120,7 @@ export async function preview(ctx: any) {
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
-  const { fields, parameters, queryVerb, transformer, queryId } = query
+  const { fields, parameters, queryVerb, transformer, queryId, schema } = query
 
   const authConfigCtx: any = getAuthConfig(ctx)
 
@@ -133,6 +133,7 @@ export async function preview(ctx: any) {
       parameters,
       transformer,
       queryId,
+      schema,
       // have to pass down to the thread runner - can't put into context now
       environmentVariables: envVars,
       ctx: {
@@ -228,6 +229,7 @@ async function execute(
         user: ctx.user,
         auth: { ...authConfigCtx },
       },
+      schema: query.schema,
     }
     const runFn = () => Runner.run(inputs)
 
diff --git a/packages/server/src/automations/tests/filter.spec.js b/packages/server/src/automations/tests/filter.spec.ts
similarity index 76%
rename from packages/server/src/automations/tests/filter.spec.js
rename to packages/server/src/automations/tests/filter.spec.ts
index 5fa5e0e202..c2c5d13e2d 100644
--- a/packages/server/src/automations/tests/filter.spec.js
+++ b/packages/server/src/automations/tests/filter.spec.ts
@@ -1,11 +1,18 @@
-const setup = require("./utilities")
-const { FilterConditions } = require("../steps/filter")
+import * as setup from "./utilities"
+import { FilterConditions } from "../steps/filter"
 
 describe("test the filter logic", () => {
-  async function checkFilter(field, condition, value, pass = true) {
-    let res = await setup.runStep(setup.actions.FILTER.stepId,
-      { field, condition, value }
-    )
+  async function checkFilter(
+    field: any,
+    condition: string,
+    value: any,
+    pass = true
+  ) {
+    let res = await setup.runStep(setup.actions.FILTER.stepId, {
+      field,
+      condition,
+      value,
+    })
     expect(res.result).toEqual(pass)
     expect(res.success).toEqual(true)
   }
@@ -36,9 +43,9 @@ describe("test the filter logic", () => {
 
   it("check date coercion", async () => {
     await checkFilter(
-      (new Date()).toISOString(),
+      new Date().toISOString(),
       FilterConditions.GREATER_THAN,
-      (new Date(-10000)).toISOString(),
+      new Date(-10000).toISOString(),
       true
     )
   })
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index a01f6b71f9..cb71a78013 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -148,7 +148,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     this.config = {
       ...config,
       multipleStatements: true,
-      typeCast: function (field: any, next: any) {
+    }
+    if (!this.config.typeCast) {
+      this.config.typeCast = function (field: any, next: any) {
         if (
           field.type == "DATETIME" ||
           field.type === "DATE" ||
@@ -161,7 +163,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
           return field.buffer()?.[0]
         }
         return next()
-      },
+      }
     }
   }
 
@@ -204,7 +206,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
   async internalQuery(
     query: SqlQuery,
-    opts: { connect?: boolean; disableCoercion?: boolean } = {
+    opts: {
+      connect?: boolean
+      disableCoercion?: boolean
+    } = {
       connect: true,
       disableCoercion: false,
     }
diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts
index 21e7ce0b69..dd0891d34a 100644
--- a/packages/server/src/threads/definitions.ts
+++ b/packages/server/src/threads/definitions.ts
@@ -11,6 +11,12 @@ export interface QueryEvent {
   queryId: string
   environmentVariables?: Record<string, string>
   ctx?: any
+  schema?: {
+    [key: string]: {
+      name: string
+      type: string
+    }
+  }
 }
 
 export interface QueryVariable {
diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts
index 9901ddffa6..d3949f7e25 100644
--- a/packages/server/src/threads/query.ts
+++ b/packages/server/src/threads/query.ts
@@ -8,6 +8,7 @@ import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
+import { SourceName } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
@@ -27,6 +28,7 @@ class QueryRunner {
   hasRerun: boolean
   hasRefreshedOAuth: boolean
   hasDynamicVariables: boolean
+  schema: any
 
   constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) {
     this.datasource = input.datasource
@@ -36,6 +38,7 @@ class QueryRunner {
     this.pagination = input.pagination
     this.transformer = input.transformer
     this.queryId = input.queryId
+    this.schema = input.schema
     this.noRecursiveQuery = flags.noRecursiveQuery
     this.cachedVariables = []
     // Additional context items for enrichment
@@ -50,7 +53,7 @@ class QueryRunner {
   }
 
   async execute(): Promise<any> {
-    let { datasource, fields, queryVerb, transformer } = this
+    let { datasource, fields, queryVerb, transformer, schema } = this
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
@@ -67,6 +70,32 @@ class QueryRunner {
       datasourceClone.config.authConfigs = updatedConfigs
     }
 
+    if (datasource.source === SourceName.MYSQL && schema) {
+      datasourceClone.config.typeCast = function (field: any, next: any) {
+        if (schema[field.name]?.name === field.name) {
+          if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) {
+            if (schema[field.name]?.type === "number") {
+              const value = field.string()
+              return value ? Number(value) : null
+            } else {
+              return field.string()
+            }
+          }
+        }
+        if (
+          field.type == "DATETIME" ||
+          field.type === "DATE" ||
+          field.type === "TIMESTAMP"
+        ) {
+          return field.string()
+        }
+        if (field.type === "BIT" && field.length === 1) {
+          return field.buffer()?.[0]
+        }
+        return next()
+      }
+    }
+
     const integration = new Integration(datasourceClone.config)
 
     // pre-query, make sure datasource variables are added to parameters

From 057761fdd902c8455e15b99316be39e0f973b358 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Thu, 3 Aug 2023 18:13:51 +0100
Subject: [PATCH 02/88] Update filters types with schemaFields

---
 .../controls/FilterEditor/FilterDrawer.svelte         | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
index e8bbb8a354..25a03a93de 100644
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
@@ -17,7 +17,7 @@
   import { generate } from "shortid"
   import { LuceneUtils, Constants } from "@budibase/frontend-core"
   import { getFields } from "helpers/searchFields"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
 
   export let schemaFields
   export let filters = []
@@ -64,6 +64,15 @@
       })
   }
 
+  onMount(() => {
+    parseFilters(filters)
+    rawFilters.forEach(filter => {
+      filter.type =
+        schemaFields.find(field => field.name === filter.field)?.type ||
+        filter.type
+    })
+  })
+
   // Add field key prefixes and a special metadata filter object to indicate
   // whether to use the "match all" or "match any" behaviour
   const enrichFilters = (rawFilters, matchAny) => {

From 20f71fadd32fb7f92e8e6ae972ccbf79707e509a Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Thu, 3 Aug 2023 18:52:55 +0100
Subject: [PATCH 03/88] Refactor

---
 packages/server/src/integrations/mysql.ts | 63 +++++++++++++++++------
 packages/server/src/threads/query.ts      | 29 ++---------
 packages/types/src/sdk/datasources.ts     |  6 +++
 3 files changed, 56 insertions(+), 42 deletions(-)

diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index cb71a78013..b88162c59a 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -93,6 +93,21 @@ const SCHEMA: Integration = {
   },
 }
 
+const defaultTypeCasting = function (field: any, next: any) {
+  if (
+    field.type == "DATETIME" ||
+    field.type === "DATE" ||
+    field.type === "TIMESTAMP" ||
+    field.type === "LONGLONG"
+  ) {
+    return field.string()
+  }
+  if (field.type === "BIT" && field.length === 1) {
+    return field.buffer()?.[0]
+  }
+  return next()
+}
+
 export function bindingTypeCoerce(bindings: any[]) {
   for (let i = 0; i < bindings.length; i++) {
     const binding = bindings[i]
@@ -147,24 +162,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     delete config.rejectUnauthorized
     this.config = {
       ...config,
+      typeCast: defaultTypeCasting,
       multipleStatements: true,
     }
-    if (!this.config.typeCast) {
-      this.config.typeCast = function (field: any, next: any) {
-        if (
-          field.type == "DATETIME" ||
-          field.type === "DATE" ||
-          field.type === "TIMESTAMP" ||
-          field.type === "LONGLONG"
-        ) {
-          return field.string()
-        }
-        if (field.type === "BIT" && field.length === 1) {
-          return field.buffer()?.[0]
-        }
-        return next()
-      }
-    }
   }
 
   async testConnection() {
@@ -196,6 +196,37 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     return `concat(${parts.join(", ")})`
   }
 
+  defineTypeCastingFromSchema(schema: {
+    [key: string]: { name: string; type: string }
+  }): void {
+    if (!schema) {
+      return
+    }
+    this.config.typeCast = function (field: any, next: any) {
+      if (schema[field.name]?.name === field.name) {
+        if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) {
+          if (schema[field.name]?.type === "number") {
+            const value = field.string()
+            return value ? Number(value) : null
+          } else {
+            return field.string()
+          }
+        }
+      }
+      if (
+        field.type == "DATETIME" ||
+        field.type === "DATE" ||
+        field.type === "TIMESTAMP"
+      ) {
+        return field.string()
+      }
+      if (field.type === "BIT" && field.length === 1) {
+        return field.buffer()?.[0]
+      }
+      return next()
+    }
+  }
+
   async connect() {
     this.client = await mysql.createConnection(this.config)
   }
diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts
index d3949f7e25..37720c66ba 100644
--- a/packages/server/src/threads/query.ts
+++ b/packages/server/src/threads/query.ts
@@ -70,34 +70,11 @@ class QueryRunner {
       datasourceClone.config.authConfigs = updatedConfigs
     }
 
-    if (datasource.source === SourceName.MYSQL && schema) {
-      datasourceClone.config.typeCast = function (field: any, next: any) {
-        if (schema[field.name]?.name === field.name) {
-          if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) {
-            if (schema[field.name]?.type === "number") {
-              const value = field.string()
-              return value ? Number(value) : null
-            } else {
-              return field.string()
-            }
-          }
-        }
-        if (
-          field.type == "DATETIME" ||
-          field.type === "DATE" ||
-          field.type === "TIMESTAMP"
-        ) {
-          return field.string()
-        }
-        if (field.type === "BIT" && field.length === 1) {
-          return field.buffer()?.[0]
-        }
-        return next()
-      }
-    }
-
     const integration = new Integration(datasourceClone.config)
 
+    // define the type casting from the schema
+    integration.defineTypeCastingFromSchema?.(schema)
+
     // pre-query, make sure datasource variables are added to parameters
     const parameters = await this.addDatasourceVariables()
 
diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts
index 2391f6e878..d6a0d4a7c8 100644
--- a/packages/types/src/sdk/datasources.ts
+++ b/packages/types/src/sdk/datasources.ts
@@ -166,6 +166,12 @@ export interface IntegrationBase {
   delete?(query: any): Promise<any[] | any>
   testConnection?(): Promise<ConnectionInfo>
   getExternalSchema?(): Promise<string>
+  defineTypeCastingFromSchema?(schema: {
+    [key: string]: {
+      name: string
+      type: string
+    }
+  }): void
 }
 
 export interface DatasourcePlus extends IntegrationBase {

From a1d85a831c7118c62918722d3923fc4255945f59 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 4 Aug 2023 16:58:40 +0100
Subject: [PATCH 04/88] Some quick modifications to allow the views to go
 through the standard row CRUD, the view search is still separate for now
 however this may change.

---
 packages/server/src/api/routes/row.ts         | 91 +------------------
 .../server/src/api/routes/tests/row.spec.ts   | 17 ----
 packages/server/src/db/utils.ts               | 12 ++-
 packages/server/src/middleware/noViewData.ts  |  9 --
 .../src/middleware/tests/noViewData.spec.ts   | 83 -----------------
 .../server/src/middleware/trimViewRowInfo.ts  | 14 +--
 packages/server/src/sdk/app/views/index.ts    |  7 +-
 packages/types/src/documents/document.ts      |  1 +
 packages/types/src/sdk/permissions.ts         |  1 -
 9 files changed, 25 insertions(+), 210 deletions(-)
 delete mode 100644 packages/server/src/middleware/noViewData.ts
 delete mode 100644 packages/server/src/middleware/tests/noViewData.spec.ts

diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts
index ac0cd2b4a4..3227df98ed 100644
--- a/packages/server/src/api/routes/row.ts
+++ b/packages/server/src/api/routes/row.ts
@@ -4,9 +4,7 @@ import authorized from "../../middleware/authorized"
 import { paramResource, paramSubResource } from "../../middleware/resourceId"
 import { permissions } from "@budibase/backend-core"
 import { internalSearchValidator } from "./utils/validators"
-import noViewData from "../../middleware/noViewData"
 import trimViewRowInfo from "../../middleware/trimViewRowInfo"
-import * as utils from "../../db/utils"
 const { PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
@@ -177,7 +175,7 @@ router
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
-    noViewData,
+    trimViewRowInfo,
     rowController.save
   )
   /**
@@ -192,7 +190,7 @@ router
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
-    noViewData,
+    trimViewRowInfo,
     rowController.patch
   )
   /**
@@ -245,6 +243,7 @@ router
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
+    trimViewRowInfo,
     rowController.destroy
   )
 
@@ -267,92 +266,10 @@ router
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.exportRows
   )
-
-router
   .get(
     "/api/v2/views/:viewId/search",
-    authorized(PermissionType.VIEW, PermissionLevel.READ),
+    authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.views.searchView
   )
-  /**
-   * @api {post} /api/:tableId/rows Creates a new row
-   * @apiName Creates a new row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API will create a new row based on the supplied body. If the
-   * body includes an "_id" field then it will update an existing row if the field
-   * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
-   * already used by Budibase tables and cannot be used for columns.
-   *
-   * @apiParam {string} tableId The ID of the table to save a row to.
-   *
-   * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
-   * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
-   * must also be provided.
-   * @apiParam (Body) {string} _viewId The ID of the view should be specified in the row body itself.
-   * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself.
-   * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches
-   * a column in the specified table. All other fields will be dropped and not stored.
-   *
-   * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this
-   * is the rows new ID.
-   * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned.
-   * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
-   */
-  .post(
-    "/api/v2/views/:viewId/rows",
-    paramResource("viewId"),
-    authorized(PermissionType.VIEW, PermissionLevel.WRITE),
-    trimViewRowInfo,
-    rowController.save
-  )
-  /**
-   * @api {patch} /api/v2/views/:viewId/rows/:rowId Updates a row
-   * @apiName Update a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint is identical to the row creation endpoint but instead it will
-   * error if an _id isn't provided, it will only function for existing rows.
-   */
-  .patch(
-    "/api/v2/views/:viewId/rows/:rowId",
-    paramResource("viewId"),
-    authorized(PermissionType.VIEW, PermissionLevel.WRITE),
-    trimViewRowInfo,
-    rowController.patch
-  )
-  /**
-   * @api {delete} /api/v2/views/:viewId/rows Delete rows for a view
-   * @apiName Delete rows for a view
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint can delete a single row, or delete them in a bulk
-   * fashion.
-   *
-   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
-   * key of the request body that are to be deleted.
-   * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field.
-   * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its
-   * revision here.
-   *
-   * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array
-   * of the deleted rows, if deleting a single row then the body will contain a "row" property which
-   * is the deleted row.
-   */
-  .delete(
-    "/api/v2/views/:viewId/rows",
-    paramResource("viewId"),
-    authorized(PermissionType.VIEW, PermissionLevel.WRITE),
-    // This is required as the implementation relies on the table id
-    (ctx, next) => {
-      ctx.params.tableId = utils.extractViewInfoFromID(
-        ctx.params.viewId
-      ).tableId
-      return next()
-    },
-    rowController.destroy
-  )
 
 export default router
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index b986ffb945..86c41b8503 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -24,7 +24,6 @@ import {
   structures,
 } from "@budibase/backend-core/tests"
 import trimViewRowInfoMiddleware from "../../../middleware/trimViewRowInfo"
-import noViewDataMiddleware from "../../../middleware/noViewData"
 import router from "../row"
 
 describe("/rows", () => {
@@ -406,14 +405,6 @@ describe("/rows", () => {
         "Table row endpoints cannot contain view info"
       )
     })
-
-    it("should setup the noViewData middleware", async () => {
-      const route = router.stack.find(
-        r => r.methods.includes("POST") && r.path === "/api/:tableId/rows"
-      )
-      expect(route).toBeDefined()
-      expect(route?.stack).toContainEqual(noViewDataMiddleware)
-    })
   })
 
   describe("patch", () => {
@@ -482,14 +473,6 @@ describe("/rows", () => {
         "Table row endpoints cannot contain view info"
       )
     })
-
-    it("should setup the noViewData middleware", async () => {
-      const route = router.stack.find(
-        r => r.methods.includes("PATCH") && r.path === "/api/:tableId/rows"
-      )
-      expect(route).toBeDefined()
-      expect(route?.stack).toContainEqual(noViewDataMiddleware)
-    })
   })
 
   describe("destroy", () => {
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index dda14a9187..e8dc3b3f6c 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -284,10 +284,20 @@ export function getMultiIDParams(ids: string[]) {
  * @returns {string} The new view ID which the view doc can be stored under.
  */
 export function generateViewID(tableId: string) {
-  return `${tableId}${SEPARATOR}${newid()}`
+  return `${DocumentType.VIEW}${SEPARATOR}${tableId}${SEPARATOR}${newid()}`
+}
+
+export function isViewID(viewId: string) {
+  return viewId?.split(SEPARATOR)[0] === DocumentType.VIEW
 }
 
 export function extractViewInfoFromID(viewId: string) {
+  if (!isViewID(viewId)) {
+    throw new Error("Unable to extract table ID, is not a view ID")
+  }
+  const split = viewId.split(SEPARATOR)
+  split.shift()
+  viewId = split.join(SEPARATOR)
   const regex = new RegExp(`^(?<tableId>.+)${SEPARATOR}([^${SEPARATOR}]+)$`)
   const res = regex.exec(viewId)
   return {
diff --git a/packages/server/src/middleware/noViewData.ts b/packages/server/src/middleware/noViewData.ts
deleted file mode 100644
index 809424b6bf..0000000000
--- a/packages/server/src/middleware/noViewData.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-import { Ctx, Row } from "@budibase/types"
-
-export default async (ctx: Ctx<Row>, next: any) => {
-  if (ctx.request.body._viewId) {
-    return ctx.throw(400, "Table row endpoints cannot contain view info")
-  }
-
-  return next()
-}
diff --git a/packages/server/src/middleware/tests/noViewData.spec.ts b/packages/server/src/middleware/tests/noViewData.spec.ts
deleted file mode 100644
index 54b0ca8ff8..0000000000
--- a/packages/server/src/middleware/tests/noViewData.spec.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-import { generator } from "@budibase/backend-core/tests"
-import { BBRequest, FieldType, Row, Table } from "@budibase/types"
-import { Next } from "koa"
-import * as utils from "../../db/utils"
-import noViewDataMiddleware from "../noViewData"
-
-class TestConfiguration {
-  next: Next
-  throw: jest.Mock<(status: number, message: string) => never>
-  middleware: typeof noViewDataMiddleware
-  params: Record<string, any>
-  request?: Pick<BBRequest<Row>, "body">
-
-  constructor() {
-    this.next = jest.fn()
-    this.throw = jest.fn()
-    this.params = {}
-
-    this.middleware = noViewDataMiddleware
-  }
-
-  executeMiddleware(ctxRequestBody: Row) {
-    this.request = {
-      body: ctxRequestBody,
-    }
-    return this.middleware(
-      {
-        request: this.request as any,
-        throw: this.throw as any,
-        params: this.params,
-      } as any,
-      this.next
-    )
-  }
-
-  afterEach() {
-    jest.clearAllMocks()
-  }
-}
-
-describe("noViewData middleware", () => {
-  let config: TestConfiguration
-
-  beforeEach(() => {
-    config = new TestConfiguration()
-  })
-
-  afterEach(() => {
-    config.afterEach()
-  })
-
-  const getRandomData = () => ({
-    _id: generator.guid(),
-    name: generator.name(),
-    age: generator.age(),
-    address: generator.address(),
-  })
-
-  it("it should pass without view id data", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware({
-      ...data,
-    })
-
-    expect(config.next).toBeCalledTimes(1)
-    expect(config.throw).not.toBeCalled()
-  })
-
-  it("it should throw an error if _viewid is provided", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware({
-      _viewId: generator.guid(),
-      ...data,
-    })
-
-    expect(config.throw).toBeCalledTimes(1)
-    expect(config.throw).toBeCalledWith(
-      400,
-      "Table row endpoints cannot contain view info"
-    )
-    expect(config.next).not.toBeCalled()
-  })
-})
diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts
index 2e3ded27f5..763552c3d7 100644
--- a/packages/server/src/middleware/trimViewRowInfo.ts
+++ b/packages/server/src/middleware/trimViewRowInfo.ts
@@ -7,15 +7,15 @@ import { Next } from "koa"
 export default async (ctx: Ctx<Row>, next: Next) => {
   const { body } = ctx.request
   const { _viewId: viewId } = body
-  if (!viewId) {
-    return ctx.throw(400, "_viewId is required")
+
+  const possibleViewId = ctx.params.tableId
+
+  // nothing to do, it is not a view (just a table ID)
+  if (!viewId || !utils.isViewID(possibleViewId)) {
+    return next()
   }
 
-  if (!ctx.params.viewId) {
-    return ctx.throw(400, "viewId path is required")
-  }
-
-  const { tableId } = utils.extractViewInfoFromID(ctx.params.viewId)
+  const { tableId } = utils.extractViewInfoFromID(possibleViewId)
   const { _viewId, ...trimmedView } = await trimViewFields(
     viewId,
     tableId,
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index 7e75f22060..637caa06ee 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -1,17 +1,14 @@
-import { HTTPError, context } from "@budibase/backend-core"
+import { context, HTTPError } from "@budibase/backend-core"
 import { FieldSchema, TableSchema, View, ViewV2 } from "@budibase/types"
 
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
-import merge from "lodash/merge"
 
 export async function get(viewId: string): Promise<ViewV2 | undefined> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
   const table = await sdk.tables.getTable(tableId)
   const views = Object.values(table.views!)
-  const view = views.find(v => isV2(v) && v.id === viewId) as ViewV2 | undefined
-
-  return view
+  return views.find(v => isV2(v) && v.id === viewId) as ViewV2 | undefined
 }
 
 export async function create(
diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts
index 6ba318269b..164aa79ac9 100644
--- a/packages/types/src/documents/document.ts
+++ b/packages/types/src/documents/document.ts
@@ -37,6 +37,7 @@ export enum DocumentType {
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
   AUDIT_LOG = "al",
+  VIEW = "view",
 }
 
 export interface Document {
diff --git a/packages/types/src/sdk/permissions.ts b/packages/types/src/sdk/permissions.ts
index 9e51e4c7b4..9fe1970e44 100644
--- a/packages/types/src/sdk/permissions.ts
+++ b/packages/types/src/sdk/permissions.ts
@@ -14,6 +14,5 @@ export enum PermissionType {
   WEBHOOK = "webhook",
   BUILDER = "builder",
   GLOBAL_BUILDER = "globalBuilder",
-  VIEW = "view",
   QUERY = "query",
 }

From 41a904126817d1f72014dc44c88e8c093ab927af Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 7 Aug 2023 16:36:28 +0100
Subject: [PATCH 05/88] PR fixes.

---
 packages/backend-core/src/security/permissions.ts |  4 ----
 packages/server/src/api/routes/row.ts             | 11 +++++------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts
index 70dae57ae6..aa0b20a30c 100644
--- a/packages/backend-core/src/security/permissions.ts
+++ b/packages/backend-core/src/security/permissions.ts
@@ -78,7 +78,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -87,7 +86,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
     ],
   },
@@ -98,7 +96,6 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.USER, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
     ],
   },
@@ -109,7 +106,6 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
       new Permission(PermissionType.USER, PermissionLevel.ADMIN),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
-      new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
     ],
diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts
index 55906c2ffe..1fd45e0e92 100644
--- a/packages/server/src/api/routes/row.ts
+++ b/packages/server/src/api/routes/row.ts
@@ -267,11 +267,10 @@ router
     rowController.exportRows
   )
 
-router
-  .post(
-    "/api/v2/views/:viewId/search",
-    authorized(PermissionType.TABLE, PermissionLevel.READ),
-    rowController.views.searchView
-  )
+router.post(
+  "/api/v2/views/:viewId/search",
+  authorized(PermissionType.TABLE, PermissionLevel.READ),
+  rowController.views.searchView
+)
 
 export default router

From 4c2e3a54893db23f7c8f0524e079004e7cbf4415 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 7 Aug 2023 16:49:13 +0100
Subject: [PATCH 06/88] Updating last remaining view perms to table perms.

---
 packages/server/src/api/routes/view.ts    | 2 +-
 packages/server/src/utilities/security.ts | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/view.ts b/packages/server/src/api/routes/view.ts
index 18c58134b4..07e31fc701 100644
--- a/packages/server/src/api/routes/view.ts
+++ b/packages/server/src/api/routes/view.ts
@@ -34,7 +34,7 @@ router
     "/api/views/:viewName",
     paramResource("viewName"),
     authorized(
-      permissions.PermissionType.VIEW,
+      permissions.PermissionType.TABLE,
       permissions.PermissionLevel.READ
     ),
     rowController.fetchView
diff --git a/packages/server/src/utilities/security.ts b/packages/server/src/utilities/security.ts
index 694dff4360..54e19007f1 100644
--- a/packages/server/src/utilities/security.ts
+++ b/packages/server/src/utilities/security.ts
@@ -14,6 +14,7 @@ export function getPermissionType(resourceId: string) {
   switch (docType) {
     case DocumentType.TABLE:
     case DocumentType.ROW:
+    case DocumentType.VIEW:
       return permissions.PermissionType.TABLE
     case DocumentType.AUTOMATION:
       return permissions.PermissionType.AUTOMATION
@@ -22,9 +23,6 @@ export function getPermissionType(resourceId: string) {
     case DocumentType.QUERY:
     case DocumentType.DATASOURCE:
       return permissions.PermissionType.QUERY
-    default:
-      // views don't have an ID, will end up here
-      return permissions.PermissionType.VIEW
   }
 }
 

From c6e657a7b45cbf5e8897309b005bdf7d0a2f214d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 7 Aug 2023 20:32:27 +0000
Subject: [PATCH 07/88] Bump version to 2.9.8

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 86af55741a..c33d100e72 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.7",
+  "version": "2.9.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From b925a347d2ffdcd7144c7dfcad62fcd9f3deeb1a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:35:04 +0300
Subject: [PATCH 08/88] Remove fetch-depth

---
 .github/workflows/budibase_ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 08fc2fe0b8..683f8cbfe8 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -173,7 +173,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 0
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Check pro commit

From a882e3341f4927f467b36e436fb017261cf11147 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:43:31 +0300
Subject: [PATCH 09/88] Add since flag

---
 .github/workflows/budibase_ci.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 683f8cbfe8..135fa035f5 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,6 +18,7 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  NX_BASE_BRANCH: ${{ github.base_ref || "origin/master"}}
 
 jobs:
   lint:
@@ -61,9 +62,9 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       # Run build all the projects
-      - run: yarn build
+      - run: yarn build --since=${{ NX_BASE_BRANCH }}
       # Check the types of the projects built via esbuild
-      - run: yarn check:types
+      - run: yarn check:types --since=${{ NX_BASE_BRANCH }}
 
   test-libraries:
     runs-on: ubuntu-latest
@@ -84,7 +85,7 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -111,7 +112,7 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Test worker and server
-        run: yarn test --scope=@budibase/worker --scope=@budibase/server
+        run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -134,7 +135,7 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/pro
+      - run: yarn test --scope=@budibase/pro --since=${{ NX_BASE_BRANCH }}
 
   integration-test:
     runs-on: ubuntu-latest

From 19b3def9b855355b7c31df1a8808eabf19b49d05 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:45:59 +0300
Subject: [PATCH 10/88] Fix env

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 135fa035f5..b12b371f24 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: ${{ github.base_ref || "origin/master"}}
+  NX_BASE_BRANCH: ${{ github.base_ref || origin/master}}
 
 jobs:
   lint:

From 3e7da2bfd10b9cdb495a1d632196f756209d7d44 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:47:10 +0300
Subject: [PATCH 11/88] Fix

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index b12b371f24..3218f69ff6 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: ${{ github.base_ref || origin/master}}
+  NX_BASE_BRANCH: ${{ github.base_ref || 'origin/master'}}
 
 jobs:
   lint:

From 49bc467eb982686af6cd014e8d31dd9908148da6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:48:16 +0300
Subject: [PATCH 12/88] Prefix env

---
 .github/workflows/budibase_ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 3218f69ff6..ea00c9438a 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -62,9 +62,9 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       # Run build all the projects
-      - run: yarn build --since=${{ NX_BASE_BRANCH }}
+      - run: yarn build --since=${{ env.NX_BASE_BRANCH }}
       # Check the types of the projects built via esbuild
-      - run: yarn check:types --since=${{ NX_BASE_BRANCH }}
+      - run: yarn check:types --since=${{ env.NX_BASE_BRANCH }}
 
   test-libraries:
     runs-on: ubuntu-latest
@@ -85,7 +85,7 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ NX_BASE_BRANCH }}
+      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -112,7 +112,7 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Test worker and server
-        run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ NX_BASE_BRANCH }}
+        run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -135,7 +135,7 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/pro --since=${{ NX_BASE_BRANCH }}
+      - run: yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
 
   integration-test:
     runs-on: ubuntu-latest

From d5d5c21ef80afcb09fd9eee992d1adf5bb1cef88 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:56:44 +0300
Subject: [PATCH 13/88] Change build to run in lerna

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 4294e83883..4e4befb5f2 100644
--- a/package.json
+++ b/package.json
@@ -34,7 +34,7 @@
     "preinstall": "node scripts/syncProPackage.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
-    "build": "yarn nx run-many -t=build",
+    "build": "lerna run build --stream",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "check:types": "lerna run check:types",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",

From 987ee1b8b27b612f66ba165291311e002df503b9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 11:58:29 +0300
Subject: [PATCH 14/88] Prefix NX_BASE_BRANCH

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index ea00c9438a..693d81372d 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: ${{ github.base_ref || 'origin/master'}}
+  NX_BASE_BRANCH: origin/${{ github.base_ref || 'master'}}
 
 jobs:
   lint:

From 5eb49fbb293a737afd528e640701b425fd80ff2a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 12:08:39 +0300
Subject: [PATCH 15/88] Fetch remote refs

---
 .github/workflows/budibase_ci.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 693d81372d..52bff87e0c 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -61,6 +61,8 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
+      - name: Fetch remote references
+        run: git fetch origin "+refs/heads/*:refs/remotes/origin/*"
       # Run build all the projects
       - run: yarn build --since=${{ env.NX_BASE_BRANCH }}
       # Check the types of the projects built via esbuild
@@ -85,6 +87,8 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
+      - name: Fetch remote references
+        run: git fetch origin "+refs/heads/*:refs/remotes/origin/*"
       - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
@@ -111,6 +115,8 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
+      - name: Fetch remote references
+        run: git fetch origin "+refs/heads/*:refs/remotes/origin/*"
       - name: Test worker and server
         run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3

From 62bb5bb4ff85ee59296b3e88492920d3e42346e1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 12:14:22 +0300
Subject: [PATCH 16/88] Fetch only required branch

---
 .github/workflows/budibase_ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 52bff87e0c..49d06fc430 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: origin/${{ github.base_ref || 'master'}}
+  NX_BASE_BRANCH: ${{ github.base_ref || 'master'}}
 
 jobs:
   lint:
@@ -62,7 +62,7 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Fetch remote references
-        run: git fetch origin "+refs/heads/*:refs/remotes/origin/*"
+        run: git fetch origin ${{ env.NX_BASE_BRANCH }}
       # Run build all the projects
       - run: yarn build --since=${{ env.NX_BASE_BRANCH }}
       # Check the types of the projects built via esbuild
@@ -88,7 +88,7 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Fetch remote references
-        run: git fetch origin "+refs/heads/*:refs/remotes/origin/*"
+        run: git fetch origin ${{ env.NX_BASE_BRANCH }}
       - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
@@ -116,7 +116,7 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Fetch remote references
-        run: git fetch origin "+refs/heads/*:refs/remotes/origin/*"
+        run: git fetch origin ${{ env.NX_BASE_BRANCH }}
       - name: Test worker and server
         run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3

From 903b1eac09f7bdbdcaa788dcf767933469094306 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 12:27:14 +0300
Subject: [PATCH 17/88] Fetch-depth

---
 .github/workflows/budibase_ci.yml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 49d06fc430..b2e256b044 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -51,9 +51,12 @@ jobs:
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -61,8 +64,7 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Fetch remote references
-        run: git fetch origin ${{ env.NX_BASE_BRANCH }}
+
       # Run build all the projects
       - run: yarn build --since=${{ env.NX_BASE_BRANCH }}
       # Check the types of the projects built via esbuild
@@ -77,9 +79,12 @@ jobs:
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -87,8 +92,6 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Fetch remote references
-        run: git fetch origin ${{ env.NX_BASE_BRANCH }}
       - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3
         with:
@@ -105,9 +108,12 @@ jobs:
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -115,8 +121,6 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Fetch remote references
-        run: git fetch origin ${{ env.NX_BASE_BRANCH }}
       - name: Test worker and server
         run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
       - uses: codecov/codecov-action@v3

From fc6472c0cccd6ad783232d32ed196ab289bee12a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 12:31:46 +0300
Subject: [PATCH 18/88] Prefix

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index b2e256b044..2953aa0575 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: ${{ github.base_ref || 'master'}}
+  NX_BASE_BRANCH: origin/${{ github.base_ref || 'master'}}
 
 jobs:
   lint:

From ee2f3e3157e81b9906225601d1fb48ca45d04be5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 12:39:40 +0300
Subject: [PATCH 19/88] Fix build

---
 .github/workflows/budibase_ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 2953aa0575..cc1244e01a 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -138,6 +138,7 @@ jobs:
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
@@ -166,7 +167,7 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client
+      - run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client
       - name: Run tests
         run: |
           cd qa-core

From b22aeb03c40c7f410e47fdba6287572c7b32aef2 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 11:39:36 +0100
Subject: [PATCH 20/88] Using the self hosted runners in the dev environment to
 build our single image ARM, AMD and AAS images.

---
 .github/workflows/release-selfhost.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml
index 39ee812726..1455f0838f 100644
--- a/.github/workflows/release-selfhost.yml
+++ b/.github/workflows/release-selfhost.yml
@@ -5,7 +5,7 @@ on:
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, dev]
 
     steps:
       - name: Fail if not a tag

From aab04339b82078d3fb46b3e857cac8c246a72fb9 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 8 Aug 2023 11:01:28 +0000
Subject: [PATCH 21/88] Bump version to 2.9.9

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index c33d100e72..fd7dc65c76 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.8",
+  "version": "2.9.9",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 2011e1693e35b4102bdbb131dfd23d2f9abd3632 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 12:06:25 +0100
Subject: [PATCH 22/88] PR comments.

---
 .../server/src/api/controllers/row/utils.ts   |  9 ++-
 packages/server/src/api/routes/row.ts         | 72 +++++++++----------
 packages/server/src/db/utils.ts               |  9 ++-
 packages/server/src/utilities/security.ts     |  6 +-
 packages/types/src/documents/document.ts      |  6 ++
 5 files changed, 57 insertions(+), 45 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index 6ff90f2b25..6cc342b8a0 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -45,13 +45,16 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
 }
 
 export function getTableId(ctx: Ctx) {
-  if (ctx.request.body && ctx.request.body.tableId) {
+  if (ctx.request.body?.tableId) {
     return ctx.request.body.tableId
   }
-  if (ctx.params && ctx.params.tableId) {
+  if (ctx.params?.sourceId) {
+    return ctx.params.sourceId
+  }
+  if (ctx.params?.tableId) {
     return ctx.params.tableId
   }
-  if (ctx.params && ctx.params.viewName) {
+  if (ctx.params?.viewName) {
     return ctx.params.viewName
   }
 }
diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts
index 1fd45e0e92..a4ac8aa3ee 100644
--- a/packages/server/src/api/routes/row.ts
+++ b/packages/server/src/api/routes/row.ts
@@ -11,7 +11,7 @@ const router: Router = new Router()
 
 router
   /**
-   * @api {get} /api/:tableId/:rowId/enrich Get an enriched row
+   * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row
    * @apiName Get an enriched row
    * @apiGroup rows
    * @apiPermission table read access
@@ -25,13 +25,13 @@ router
    * @apiSuccess {object} row The response body will be the enriched row.
    */
   .get(
-    "/api/:tableId/:rowId/enrich",
-    paramSubResource("tableId", "rowId"),
+    "/api/:sourceId/:rowId/enrich",
+    paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetchEnrichedRow
   )
   /**
-   * @api {get} /api/:tableId/rows Get all rows in a table
+   * @api {get} /api/:sourceId/rows Get all rows in a table
    * @apiName Get all rows in a table
    * @apiGroup rows
    * @apiPermission table read access
@@ -40,37 +40,37 @@ router
    * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then
    * will simply stop.
    *
-   * @apiParam {string} tableId The ID of the table to retrieve all rows within.
+   * @apiParam {string} sourceId The ID of the table to retrieve all rows within.
    *
    * @apiSuccess {object[]} rows The response body will be an array of all rows found.
    */
   .get(
-    "/api/:tableId/rows",
-    paramResource("tableId"),
+    "/api/:sourceId/rows",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetch
   )
   /**
-   * @api {get} /api/:tableId/rows/:rowId Retrieve a single row
+   * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row
    * @apiName Retrieve a single row
    * @apiGroup rows
    * @apiPermission table read access
    * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve
    * a row by anything other than its _id field, use the search endpoint.
    *
-   * @apiParam {string} tableId The ID of the table to retrieve a row from.
+   * @apiParam {string} sourceId The ID of the table to retrieve a row from.
    * @apiParam {string} rowId The ID of the row to retrieve.
    *
    * @apiSuccess {object} body The response body will be the row that was found.
    */
   .get(
-    "/api/:tableId/rows/:rowId",
-    paramSubResource("tableId", "rowId"),
+    "/api/:sourceId/rows/:rowId",
+    paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.find
   )
   /**
-   * @api {post} /api/:tableId/search Search for rows in a table
+   * @api {post} /api/:sourceId/search Search for rows in a table
    * @apiName Search for rows in a table
    * @apiGroup rows
    * @apiPermission table read access
@@ -78,7 +78,7 @@ router
    * and data UI in the builder are built atop this. All filtering, sorting and pagination is
    * handled through this, for internal and external (datasource plus, e.g. SQL) tables.
    *
-   * @apiParam {string} tableId The ID of the table to retrieve rows from.
+   * @apiParam {string} sourceId The ID of the table to retrieve rows from.
    *
    * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true,
    * defaults to false.
@@ -133,22 +133,22 @@ router
    * page.
    */
   .post(
-    "/api/:tableId/search",
+    "/api/:sourceId/search",
     internalSearchValidator(),
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
   // DEPRECATED - this is an old API, but for backwards compat it needs to be
   // supported still
   .post(
-    "/api/search/:tableId/rows",
-    paramResource("tableId"),
+    "/api/search/:sourceId/rows",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
   /**
-   * @api {post} /api/:tableId/rows Creates a new row
+   * @api {post} /api/:sourceId/rows Creates a new row
    * @apiName Creates a new row
    * @apiGroup rows
    * @apiPermission table write access
@@ -157,7 +157,7 @@ router
    * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
    * already used by Budibase tables and cannot be used for columns.
    *
-   * @apiParam {string} tableId The ID of the table to save a row to.
+   * @apiParam {string} sourceId The ID of the table to save a row to.
    *
    * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
    * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
@@ -172,14 +172,14 @@ router
    * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
    */
   .post(
-    "/api/:tableId/rows",
-    paramResource("tableId"),
+    "/api/:sourceId/rows",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     trimViewRowInfo,
     rowController.save
   )
   /**
-   * @api {patch} /api/:tableId/rows Updates a row
+   * @api {patch} /api/:sourceId/rows Updates a row
    * @apiName Update a row
    * @apiGroup rows
    * @apiPermission table write access
@@ -187,14 +187,14 @@ router
    * error if an _id isn't provided, it will only function for existing rows.
    */
   .patch(
-    "/api/:tableId/rows",
-    paramResource("tableId"),
+    "/api/:sourceId/rows",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     trimViewRowInfo,
     rowController.patch
   )
   /**
-   * @api {post} /api/:tableId/rows/validate Validate inputs for a row
+   * @api {post} /api/:sourceId/rows/validate Validate inputs for a row
    * @apiName Validate inputs for a row
    * @apiGroup rows
    * @apiPermission table write access
@@ -202,7 +202,7 @@ router
    * given the table schema, this will iterate through all the constraints on the table and
    * check if the request body is valid.
    *
-   * @apiParam {string} tableId The ID of the table the row is to be validated for.
+   * @apiParam {string} sourceId The ID of the table the row is to be validated for.
    *
    * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested
    * against the table schema and constraints.
@@ -214,20 +214,20 @@ router
    * the schema.
    */
   .post(
-    "/api/:tableId/rows/validate",
-    paramResource("tableId"),
+    "/api/:sourceId/rows/validate",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.validate
   )
   /**
-   * @api {delete} /api/:tableId/rows Delete rows
+   * @api {delete} /api/:sourceId/rows Delete rows
    * @apiName Delete rows
    * @apiGroup rows
    * @apiPermission table write access
    * @apiDescription This endpoint can delete a single row, or delete them in a bulk
    * fashion.
    *
-   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
+   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
    *
    * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
    * key of the request body that are to be deleted.
@@ -240,29 +240,29 @@ router
    * is the deleted row.
    */
   .delete(
-    "/api/:tableId/rows",
-    paramResource("tableId"),
+    "/api/:sourceId/rows",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     trimViewRowInfo,
     rowController.destroy
   )
 
   /**
-   * @api {post} /api/:tableId/rows/exportRows Export Rows
+   * @api {post} /api/:sourceId/rows/exportRows Export Rows
    * @apiName Export rows
    * @apiGroup rows
    * @apiPermission table write access
    * @apiDescription This API can export a number of provided rows
    *
-   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
+   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
    *
    * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported
    *
    * @apiSuccess {object[]|object}
    */
   .post(
-    "/api/:tableId/rows/exportRows",
-    paramResource("tableId"),
+    "/api/:sourceId/rows/exportRows",
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.exportRows
   )
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index e8dc3b3f6c..abea725707 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -1,5 +1,7 @@
 import newid from "./newid"
 import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, VirtualDocumentType } from "@budibase/types"
+export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
 type Optional = string | null
 
@@ -19,7 +21,6 @@ export const BudibaseInternalDB = {
 
 export const SEPARATOR = dbCore.SEPARATOR
 export const StaticDatabases = dbCore.StaticDatabases
-export const DocumentType = dbCore.DocumentType
 export const APP_PREFIX = dbCore.APP_PREFIX
 export const APP_DEV_PREFIX = dbCore.APP_DEV_PREFIX
 export const isDevAppID = dbCore.isDevAppID
@@ -284,11 +285,13 @@ export function getMultiIDParams(ids: string[]) {
  * @returns {string} The new view ID which the view doc can be stored under.
  */
 export function generateViewID(tableId: string) {
-  return `${DocumentType.VIEW}${SEPARATOR}${tableId}${SEPARATOR}${newid()}`
+  return `${
+    VirtualDocumentType.VIEW
+  }${SEPARATOR}${tableId}${SEPARATOR}${newid()}`
 }
 
 export function isViewID(viewId: string) {
-  return viewId?.split(SEPARATOR)[0] === DocumentType.VIEW
+  return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW
 }
 
 export function extractViewInfoFromID(viewId: string) {
diff --git a/packages/server/src/utilities/security.ts b/packages/server/src/utilities/security.ts
index 54e19007f1..0da7621773 100644
--- a/packages/server/src/utilities/security.ts
+++ b/packages/server/src/utilities/security.ts
@@ -1,5 +1,5 @@
 import { permissions, roles } from "@budibase/backend-core"
-import { DocumentType } from "../db/utils"
+import { DocumentType, VirtualDocumentType } from "../db/utils"
 
 export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
   permissions.PermissionLevel.WRITE,
@@ -11,10 +11,10 @@ export function getPermissionType(resourceId: string) {
   const docType = Object.values(DocumentType).filter(docType =>
     resourceId.startsWith(docType)
   )[0]
-  switch (docType) {
+  switch (docType as DocumentType | VirtualDocumentType) {
     case DocumentType.TABLE:
     case DocumentType.ROW:
-    case DocumentType.VIEW:
+    case VirtualDocumentType.VIEW:
       return permissions.PermissionType.TABLE
     case DocumentType.AUTOMATION:
       return permissions.PermissionType.AUTOMATION
diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts
index 164aa79ac9..75f55e1367 100644
--- a/packages/types/src/documents/document.ts
+++ b/packages/types/src/documents/document.ts
@@ -37,6 +37,12 @@ export enum DocumentType {
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
   AUDIT_LOG = "al",
+  VIEW = "awd",
+}
+
+// these documents don't really exist, they are part of other
+// documents or enriched into existence as part of get requests
+export enum VirtualDocumentType {
   VIEW = "view",
 }
 

From 0abd1deb3492f80b98b0d34a6709254f393020a6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 13:19:22 +0100
Subject: [PATCH 23/88] Updating test cases, fixes based on running through
 view/row API.

---
 .../src/api/controllers/row/external.ts       | 13 ++--
 .../src/api/controllers/row/internal.ts       | 23 +++---
 .../server/src/api/controllers/row/utils.ts   | 10 ++-
 .../server/src/api/routes/tests/row.spec.ts   | 71 ++-----------------
 .../server/src/middleware/trimViewRowInfo.ts  | 31 +++++---
 .../server/src/tests/utilities/api/row.ts     | 37 ++++++++--
 .../server/src/tests/utilities/api/viewV2.ts  | 46 ------------
 packages/types/src/api/web/app/rows.ts        |  2 +
 8 files changed, 87 insertions(+), 146 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 3ee4ca6edd..bc4edbd661 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -15,6 +15,7 @@ import {
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import * as utils from "./utils"
 
 export async function handleRequest(
   operation: Operation,
@@ -43,7 +44,7 @@ export async function handleRequest(
 }
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const { _id, ...rowData } = ctx.request.body
 
   const validateResult = await sdk.rows.utils.validate({
@@ -70,7 +71,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const validateResult = await sdk.rows.utils.validate({
     row: inputs,
     tableId,
@@ -98,12 +99,12 @@ export async function save(ctx: UserCtx) {
 
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   return sdk.rows.external.getRow(tableId, id)
 }
 
 export async function destroy(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const _id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
     id: breakRowIdField(_id),
@@ -114,7 +115,7 @@ export async function destroy(ctx: UserCtx) {
 
 export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
@@ -130,7 +131,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   const datasource: Datasource = await sdk.datasources.get(datasourceId!)
   if (!tableName) {
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index 2ff1df0933..3432ec80f3 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -13,7 +13,7 @@ import {
 import { FieldTypes } from "../../../constants"
 import * as utils from "./utils"
 import { cloneDeep } from "lodash/fp"
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import {
   UserCtx,
@@ -26,8 +26,8 @@ import {
 import sdk from "../../../sdk"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
+  const tableId = utils.getTableId(ctx)
   const inputs = ctx.request.body
-  const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
   const dbTable = await sdk.tables.getTable(tableId)
@@ -94,7 +94,8 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function save(ctx: UserCtx) {
   let inputs = ctx.request.body
-  inputs.tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
+  inputs.tableId = tableId
 
   if (!inputs._rev && !inputs._id) {
     inputs._id = generateRowID(inputs.tableId)
@@ -132,20 +133,22 @@ export async function save(ctx: UserCtx) {
 }
 
 export async function find(ctx: UserCtx) {
-  const db = dbCore.getDB(ctx.appId)
-  const table = await sdk.tables.getTable(ctx.params.tableId)
-  let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
+  const tableId = utils.getTableId(ctx),
+    rowId = ctx.params.rowId
+  const table = await sdk.tables.getTable(tableId)
+  let row = await utils.findRow(ctx, tableId, rowId)
   row = await outputProcessing(table, row)
   return row
 }
 
 export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
+  const tableId = utils.getTableId(ctx)
   const { _id } = ctx.request.body
   let row = await db.get<Row>(_id)
   let _rev = ctx.request.body._rev || row._rev
 
-  if (row.tableId !== ctx.params.tableId) {
+  if (row.tableId !== tableId) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
   const table = await sdk.tables.getTable(row.tableId)
@@ -163,7 +166,7 @@ export async function destroy(ctx: UserCtx) {
   await updateRelatedFormula(table, row)
 
   let response
-  if (ctx.params.tableId === InternalTables.USER_METADATA) {
+  if (tableId === InternalTables.USER_METADATA) {
     ctx.params = {
       id: _id,
     }
@@ -176,7 +179,7 @@ export async function destroy(ctx: UserCtx) {
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const table = await sdk.tables.getTable(tableId)
   let { rows } = ctx.request.body
 
@@ -216,7 +219,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const rowId = ctx.params.rowId
   // need table to work out where links go in row
   let [table, row] = await Promise.all([
diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index 6cc342b8a0..157f18e231 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -45,15 +45,19 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
 }
 
 export function getTableId(ctx: Ctx) {
-  if (ctx.request.body?.tableId) {
-    return ctx.request.body.tableId
-  }
+  // top priority, use the URL first
   if (ctx.params?.sourceId) {
     return ctx.params.sourceId
   }
+  // check body for a table ID
+  if (ctx.request.body?.tableId) {
+    return ctx.request.body.tableId
+  }
+  // now check for old way of specifying table ID
   if (ctx.params?.tableId) {
     return ctx.params.tableId
   }
+  // now check if a specific view name
   if (ctx.params?.viewName) {
     return ctx.params.viewName
   }
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 5e1616340f..8d4c9a91fd 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -16,15 +16,12 @@ import {
   FieldType,
   SortType,
   SortOrder,
-  DeleteRow,
 } from "@budibase/types"
 import {
   expectAnyInternalColsAttributes,
   generator,
   structures,
 } from "@budibase/backend-core/tests"
-import trimViewRowInfoMiddleware from "../../../middleware/trimViewRowInfo"
-import router from "../row"
 
 describe("/rows", () => {
   let request = setup.getRequest()
@@ -393,18 +390,6 @@ describe("/rows", () => {
       expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
       expect(saved.optsFieldStrKnown).toEqual("Alpha")
     })
-
-    it("should throw an error when creating a table row with view id data", async () => {
-      const res = await request
-        .post(`/api/${row.tableId}/rows`)
-        .send({ ...row, _viewId: generator.guid() })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-      expect(res.body.message).toEqual(
-        "Table row endpoints cannot contain view info"
-      )
-    })
   })
 
   describe("patch", () => {
@@ -454,25 +439,6 @@ describe("/rows", () => {
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
     })
-
-    it("should throw an error when creating a table row with view id data", async () => {
-      const existing = await config.createRow()
-
-      const res = await config.api.row.patch(
-        table._id!,
-        {
-          ...existing,
-          _id: existing._id!,
-          _rev: existing._rev!,
-          tableId: table._id!,
-          _viewId: generator.guid(),
-        },
-        { expectStatus: 400 }
-      )
-      expect(res.body.message).toEqual(
-        "Table row endpoints cannot contain view info"
-      )
-    })
   })
 
   describe("destroy", () => {
@@ -741,7 +707,7 @@ describe("/rows", () => {
       })
       // the environment needs configured for this
       await setup.switchToSelfHosted(async () => {
-        context.doInAppContext(config.getAppId(), async () => {
+        return context.doInAppContext(config.getAppId(), async () => {
           const enriched = await outputProcessing(table, [row])
           expect((enriched as Row[])[0].attachment[0].url).toBe(
             `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
@@ -847,7 +813,7 @@ describe("/rows", () => {
         })
 
         const data = randomRowData()
-        const newRow = await config.api.viewV2.row.create(view.id, {
+        const newRow = await config.api.row.save(view.id, {
           tableId: config.table!._id,
           _viewId: view.id,
           ...data,
@@ -869,16 +835,6 @@ describe("/rows", () => {
         expect(row.body.age).toBeUndefined()
         expect(row.body.jobTitle).toBeUndefined()
       })
-
-      it("should setup the trimViewRowInfo middleware", async () => {
-        const route = router.stack.find(
-          r =>
-            r.methods.includes("POST") &&
-            r.path === "/api/v2/views/:viewId/rows"
-        )
-        expect(route).toBeDefined()
-        expect(route?.stack).toContainEqual(trimViewRowInfoMiddleware)
-      })
     })
 
     describe("patch", () => {
@@ -893,13 +849,13 @@ describe("/rows", () => {
           },
         })
 
-        const newRow = await config.api.viewV2.row.create(view.id, {
+        const newRow = await config.api.row.save(view.id, {
           tableId,
           _viewId: view.id,
           ...randomRowData(),
         })
         const newData = randomRowData()
-        await config.api.viewV2.row.update(view.id, newRow._id!, {
+        await config.api.row.patch(view.id, {
           tableId,
           _viewId: view.id,
           _id: newRow._id!,
@@ -922,16 +878,6 @@ describe("/rows", () => {
         expect(row.body.age).toBeUndefined()
         expect(row.body.jobTitle).toBeUndefined()
       })
-
-      it("should setup the trimViewRowInfo middleware", async () => {
-        const route = router.stack.find(
-          r =>
-            r.methods.includes("PATCH") &&
-            r.path === "/api/v2/views/:viewId/rows/:rowId"
-        )
-        expect(route).toBeDefined()
-        expect(route?.stack).toContainEqual(trimViewRowInfoMiddleware)
-      })
     })
 
     describe("destroy", () => {
@@ -950,10 +896,7 @@ describe("/rows", () => {
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        const body: DeleteRow = {
-          _id: createdRow._id!,
-        }
-        await config.api.viewV2.row.delete(view.id, body)
+        await config.api.row.delete(view.id, [createdRow])
 
         await assertRowUsage(rowUsage - 1)
         await assertQueryUsage(queryUsage + 1)
@@ -982,9 +925,7 @@ describe("/rows", () => {
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        await config.api.viewV2.row.delete(view.id, {
-          rows: [rows[0], rows[2]],
-        })
+        await config.api.row.delete(view.id, [rows[0], rows[2]])
 
         await assertRowUsage(rowUsage - 2)
         await assertQueryUsage(queryUsage + 1)
diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts
index 763552c3d7..5a207936b2 100644
--- a/packages/server/src/middleware/trimViewRowInfo.ts
+++ b/packages/server/src/middleware/trimViewRowInfo.ts
@@ -3,26 +3,35 @@ import * as utils from "../db/utils"
 import sdk from "../sdk"
 import { db } from "@budibase/backend-core"
 import { Next } from "koa"
+import { getTableId } from "../api/controllers/row/utils"
 
 export default async (ctx: Ctx<Row>, next: Next) => {
   const { body } = ctx.request
-  const { _viewId: viewId } = body
+  let { _viewId: viewId } = body
 
-  const possibleViewId = ctx.params.tableId
+  const possibleViewId = getTableId(ctx)
+  if (utils.isViewID(possibleViewId)) {
+    viewId = possibleViewId
+  }
 
   // nothing to do, it is not a view (just a table ID)
-  if (!viewId || !utils.isViewID(possibleViewId)) {
+  if (!viewId) {
     return next()
   }
 
-  const { tableId } = utils.extractViewInfoFromID(possibleViewId)
-  const { _viewId, ...trimmedView } = await trimViewFields(
-    viewId,
-    tableId,
-    body
-  )
-  ctx.request.body = trimmedView
-  ctx.params.tableId = tableId
+  const { tableId } = utils.extractViewInfoFromID(viewId)
+
+  // don't need to trim delete requests
+  if (ctx.method.toLowerCase() !== "delete") {
+    const { _viewId, ...trimmedView } = await trimViewFields(
+      viewId,
+      tableId,
+      body
+    )
+    ctx.request.body = trimmedView
+  }
+
+  ctx.params.sourceId = tableId
 
   return next()
 }
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index c7c72368f5..c6ef4606d2 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -1,4 +1,4 @@
-import { PatchRowRequest } from "@budibase/types"
+import { PatchRowRequest, SaveRowRequest, Row } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -8,12 +8,12 @@ export class RowAPI extends TestAPI {
   }
 
   get = async (
-    tableId: string,
+    sourceId: string,
     rowId: string,
     { expectStatus } = { expectStatus: 200 }
   ) => {
     const request = this.request
-      .get(`/api/${tableId}/rows/${rowId}`)
+      .get(`/api/${sourceId}/rows/${rowId}`)
       .set(this.config.defaultHeaders())
       .expect(expectStatus)
     if (expectStatus !== 404) {
@@ -22,16 +22,43 @@ export class RowAPI extends TestAPI {
     return request
   }
 
+  save = async (
+    sourceId: string,
+    row: SaveRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row> => {
+    const resp = await this.request
+      .post(`/api/${sourceId}/rows`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return resp.body as Row
+  }
+
   patch = async (
-    tableId: string,
+    sourceId: string,
     row: PatchRowRequest,
     { expectStatus } = { expectStatus: 200 }
   ) => {
     return this.request
-      .patch(`/api/${tableId}/rows`)
+      .patch(`/api/${sourceId}/rows`)
       .send(row)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
+
+  delete = async (
+    sourceId: string,
+    rows: Row[],
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    return this.request
+      .delete(`/api/${sourceId}/rows`)
+      .send({ rows })
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
 }
diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts
index 813d2ebfd1..1520154641 100644
--- a/packages/server/src/tests/utilities/api/viewV2.ts
+++ b/packages/server/src/tests/utilities/api/viewV2.ts
@@ -1,10 +1,6 @@
 import {
   CreateViewRequest,
   UpdateViewRequest,
-  DeleteRowRequest,
-  PatchRowRequest,
-  PatchRowResponse,
-  Row,
   ViewV2,
   SearchViewRowRequest,
 } from "@budibase/types"
@@ -90,46 +86,4 @@ export class ViewV2API extends TestAPI {
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
-
-  row = {
-    create: async (
-      viewId: string,
-      row: Row,
-      { expectStatus } = { expectStatus: 200 }
-    ): Promise<Row> => {
-      const result = await this.request
-        .post(`/api/v2/views/${viewId}/rows`)
-        .send(row)
-        .set(this.config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(expectStatus)
-      return result.body as Row
-    },
-    update: async (
-      viewId: string,
-      rowId: string,
-      row: PatchRowRequest,
-      { expectStatus } = { expectStatus: 200 }
-    ): Promise<PatchRowResponse> => {
-      const result = await this.request
-        .patch(`/api/v2/views/${viewId}/rows/${rowId}`)
-        .send(row)
-        .set(this.config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(expectStatus)
-      return result.body as PatchRowResponse
-    },
-    delete: async (
-      viewId: string,
-      body: DeleteRowRequest,
-      { expectStatus } = { expectStatus: 200 }
-    ): Promise<any> => {
-      const result = await this.request
-        .delete(`/api/v2/views/${viewId}/rows`)
-        .send(body)
-        .set(this.config.defaultHeaders())
-        .expect(expectStatus)
-      return result.body
-    },
-  }
 }
diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts
index f1890ef777..2b51c7b203 100644
--- a/packages/types/src/api/web/app/rows.ts
+++ b/packages/types/src/api/web/app/rows.ts
@@ -1,6 +1,8 @@
 import { SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
 
+export interface SaveRowRequest extends Row {}
+
 export interface PatchRowRequest extends Row {
   _id: string
   _rev: string

From c18459d84d7c8979e5402226b6b189033dd386fc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 13:43:13 +0100
Subject: [PATCH 24/88] Updating trim view info test case.

---
 .../middleware/tests/trimViewRowInfo.spec.ts  | 27 ++-----------------
 .../server/src/middleware/trimViewRowInfo.ts  |  2 +-
 2 files changed, 3 insertions(+), 26 deletions(-)

diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
index 427ac9a608..69d1272df9 100644
--- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
+++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
@@ -117,7 +117,7 @@ describe("trimViewRowInfo middleware", () => {
     })
 
     expect(config.request?.body).toEqual(data)
-    expect(config.params.tableId).toEqual(table._id)
+    expect(config.params.sourceId).toEqual(table._id)
 
     expect(config.next).toBeCalledTimes(1)
     expect(config.throw).not.toBeCalled()
@@ -143,32 +143,9 @@ describe("trimViewRowInfo middleware", () => {
       name: data.name,
       address: data.address,
     })
-    expect(config.params.tableId).toEqual(table._id)
+    expect(config.params.sourceId).toEqual(table._id)
 
     expect(config.next).toBeCalledTimes(1)
     expect(config.throw).not.toBeCalled()
   })
-
-  it("it should throw an error if no viewid is provided on the body", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware(viewId, {
-      ...data,
-    })
-
-    expect(config.throw).toBeCalledTimes(1)
-    expect(config.throw).toBeCalledWith(400, "_viewId is required")
-    expect(config.next).not.toBeCalled()
-  })
-
-  it("it should throw an error if no viewid is provided on the parameters", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware(undefined as any, {
-      _viewId: viewId,
-      ...data,
-    })
-
-    expect(config.throw).toBeCalledTimes(1)
-    expect(config.throw).toBeCalledWith(400, "viewId path is required")
-    expect(config.next).not.toBeCalled()
-  })
 })
diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts
index 5a207936b2..cff9dabd37 100644
--- a/packages/server/src/middleware/trimViewRowInfo.ts
+++ b/packages/server/src/middleware/trimViewRowInfo.ts
@@ -22,7 +22,7 @@ export default async (ctx: Ctx<Row>, next: Next) => {
   const { tableId } = utils.extractViewInfoFromID(viewId)
 
   // don't need to trim delete requests
-  if (ctx.method.toLowerCase() !== "delete") {
+  if (ctx?.method?.toLowerCase() !== "delete") {
     const { _viewId, ...trimmedView } = await trimViewFields(
       viewId,
       tableId,

From a220b201fc344275587a589ac31944da7483c81f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 13:45:10 +0100
Subject: [PATCH 25/88] Updating self host single image build

---
 .github/workflows/release-singleimage.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index 1f4a2d0b32..f62e4d2a55 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -9,8 +9,8 @@ env:
   REGISTRY_URL: registry.hub.docker.com
 jobs:
   build-amd64-arm64:
-    name: "build-amd64"
-    runs-on: ubuntu-latest
+    name: "build-amd64-arm64"
+      runs-on: [self-hosted, dev]
     strategy:
       matrix:
         node-version: [14.x]
@@ -74,7 +74,7 @@ jobs:
 
   build-aas:
     name: "build-aas"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, dev]
     strategy:
       matrix:
         node-version: [14.x]

From 73124fab6ae29d8c95a68163f3a6a76ef7b65a76 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 8 Aug 2023 12:45:37 +0000
Subject: [PATCH 26/88] Bump version to 2.9.10

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index fd7dc65c76..7506a3606e 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.9",
+  "version": "2.9.10",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 555e066be04b162b7f9c9a1fe72fb60b7520e89e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 13:49:02 +0100
Subject: [PATCH 27/88] Fixing indentation.

---
 .github/workflows/release-singleimage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index f62e4d2a55..2da80a6b29 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -10,7 +10,7 @@ env:
 jobs:
   build-amd64-arm64:
     name: "build-amd64-arm64"
-      runs-on: [self-hosted, dev]
+    runs-on: [self-hosted, dev]
     strategy:
       matrix:
         node-version: [14.x]

From 378d3ed40f155855a1903fcc8f630dc567d2e92b Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 8 Aug 2023 12:49:40 +0000
Subject: [PATCH 28/88] Bump version to 2.9.11

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 7506a3606e..88644e88d9 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.10",
+  "version": "2.9.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From e4841e8f2c608dff521b723799e813a6a3f48c44 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 15:31:05 +0100
Subject: [PATCH 29/88] Removing the usage of the self hosted runners
 (unavailable in public repo) and attempting to get working using a Docker
 manifest in parallel.

---
 .github/workflows/release-selfhost.yml    |  2 +-
 .github/workflows/release-singleimage.yml | 68 +++++++++++++++++++----
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml
index 1455f0838f..39ee812726 100644
--- a/.github/workflows/release-selfhost.yml
+++ b/.github/workflows/release-selfhost.yml
@@ -5,7 +5,7 @@ on:
 
 jobs:
   release:
-    runs-on: [self-hosted, dev]
+    runs-on: ubuntu-latest
 
     steps:
       - name: Fail if not a tag
diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index 2da80a6b29..a46099908b 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -6,14 +6,18 @@ on:
 env:
   CI: true
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  REGISTRY_URL: registry.hub.docker.com
+  REGISTRY_IMAGE: budibase/budibase
 jobs:
-  build-amd64-arm64:
-    name: "build-amd64-arm64"
-    runs-on: [self-hosted, dev]
+  build-multiarch:
+    name: "build-multiarch"
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version:
+          - 14.x
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
       - name: Fail if not a tag
         run: |
@@ -67,17 +71,61 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ matrix.platform }}
           tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+      - name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build-multiarch
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_API_KEY }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
 
   build-aas:
     name: "build-aas"
-    runs-on: [self-hosted, dev]
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version:
+          - 14.x
     steps:
       - name: Fail if not a tag
         run: |
@@ -101,8 +149,6 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
       - name: Setup Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1

From 4f5d286f62e5d1c6a9f74c6edd22734f5ae2dddd Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 8 Aug 2023 14:39:04 +0000
Subject: [PATCH 30/88] Bump version to 2.9.12

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 88644e88d9..141491ae69 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.11",
+  "version": "2.9.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 00e6a43e3ec2c4968af3fc531ad5aeb465b1d52a Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Tue, 8 Aug 2023 16:39:05 +0100
Subject: [PATCH 31/88] Form step and field group require form

---
 packages/client/manifest.json | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index b6a231917c..925c87e61d 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -2445,6 +2445,7 @@
     "name": "Form Step",
     "icon": "AssetsAdded",
     "hasChildren": true,
+    "requiredAncestors": ["form"],
     "illegalChildren": ["section", "form", "formstep", "formblock"],
     "styles": ["size"],
     "size": {
@@ -2464,6 +2465,7 @@
   "fieldgroup": {
     "name": "Field Group",
     "icon": "Group",
+    "requiredAncestors": ["form"],
     "illegalChildren": ["section"],
     "styles": ["size"],
     "hasChildren": true,

From 0bc7a647dfd278b5b5eb125c89e21eb604038932 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 18:42:39 +0300
Subject: [PATCH 32/88] Types via esbuild

---
 packages/types/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/package.json b/packages/types/package.json
index 96ab8cb095..317342e760 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -14,7 +14,7 @@
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"

From b882a64ad756fa9d39e25f7cafaed5578b33f653 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 18:45:02 +0300
Subject: [PATCH 33/88] Shared-core via esbuild

---
 packages/shared-core/package.json        | 2 +-
 packages/shared-core/tsconfig.build.json | 5 ++++-
 packages/shared-core/tsconfig.json       | 9 ---------
 3 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json
index 98ee89999b..61e97ddc76 100644
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@@ -14,7 +14,7 @@
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"
diff --git a/packages/shared-core/tsconfig.build.json b/packages/shared-core/tsconfig.build.json
index 6930e3cb99..804e86c270 100644
--- a/packages/shared-core/tsconfig.build.json
+++ b/packages/shared-core/tsconfig.build.json
@@ -12,7 +12,10 @@
     "declaration": true,
     "types": ["node"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../../types/src"]
+    }
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [
diff --git a/packages/shared-core/tsconfig.json b/packages/shared-core/tsconfig.json
index f72933ff9b..33e37179d7 100644
--- a/packages/shared-core/tsconfig.json
+++ b/packages/shared-core/tsconfig.json
@@ -1,13 +1,4 @@
 {
   "extends": "./tsconfig.build.json",
-  "compilerOptions": {
-    "baseUrl": ".",
-    "rootDir": "./src",
-    "composite": true,
-    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo",
-    "paths": {
-      "@budibase/types": ["../../types/src"]
-    }
-  },
   "exclude": ["node_modules", "dist"]
 }

From bc5b4297a5e552db92dcd95e85aae7aec08c2da6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 18:56:07 +0300
Subject: [PATCH 34/88] Backend-core via esbuild

---
 packages/backend-core/package.json     | 2 +-
 packages/backend-core/plugins.ts       | 1 -
 packages/backend-core/scripts/build.js | 8 ++++++++
 3 files changed, 9 insertions(+), 2 deletions(-)
 delete mode 100644 packages/backend-core/plugins.ts
 create mode 100644 packages/backend-core/scripts/build.js

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 4631b090fe..25baccbe53 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -14,7 +14,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
diff --git a/packages/backend-core/plugins.ts b/packages/backend-core/plugins.ts
deleted file mode 100644
index 33354eaf64..0000000000
--- a/packages/backend-core/plugins.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from "./src/plugin"
diff --git a/packages/backend-core/scripts/build.js b/packages/backend-core/scripts/build.js
new file mode 100644
index 0000000000..992513755a
--- /dev/null
+++ b/packages/backend-core/scripts/build.js
@@ -0,0 +1,8 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+coreBuild("./src/plugin/index.ts", "./dist/plugins.js")
+coreBuild("./src/index.ts", "./dist/index.js")
+coreBuild("./tests/index.ts", "./dist/tests.js")

From 5ae100d4cc743fa2825f5d8b369ed2ea7f05a6da Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 18:58:50 +0300
Subject: [PATCH 35/88] Remove unneeded nx deps

---
 packages/cli/package.json | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 99771adbd3..a0c885657c 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -63,20 +63,5 @@
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
     "typescript": "4.7.3"
-  },
-  "nx": {
-    "targets": {
-      "build": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/backend-core",
-              "@budibase/string-templates"
-            ],
-            "target": "build"
-          }
-        ]
-      }
-    }
   }
 }

From 93654907c303768ddbe67307347fbf9878823a8f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 19:08:57 +0300
Subject: [PATCH 36/88] Bundle bb dependencies in backend-core

---
 packages/backend-core/tsconfig.build.json | 6 +++++-
 packages/backend-core/tsconfig.json       | 8 --------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/packages/backend-core/tsconfig.build.json b/packages/backend-core/tsconfig.build.json
index bfbed31e23..c714f4d942 100644
--- a/packages/backend-core/tsconfig.build.json
+++ b/packages/backend-core/tsconfig.build.json
@@ -12,7 +12,11 @@
     "declaration": true,
     "types": ["node", "jest"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/shared-core": ["../shared-core/src"]
+    }
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [
diff --git a/packages/backend-core/tsconfig.json b/packages/backend-core/tsconfig.json
index 128814b955..33e37179d7 100644
--- a/packages/backend-core/tsconfig.json
+++ b/packages/backend-core/tsconfig.json
@@ -1,12 +1,4 @@
 {
   "extends": "./tsconfig.build.json",
-  "compilerOptions": {
-    "composite": true,
-    "baseUrl": ".",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/shared-core": ["../shared-core/src"]
-    }
-  },
   "exclude": ["node_modules", "dist"]
 }

From 22b456da5e2562c4efd2c6148e93aa3746132cb8 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Tue, 8 Aug 2023 17:13:40 +0100
Subject: [PATCH 37/88] Allow form step to be bindable

---
 .../ButtonActionEditor/actions/ChangeFormStep.svelte   | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ChangeFormStep.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ChangeFormStep.svelte
index ca2df71c6d..81a2119474 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ChangeFormStep.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ChangeFormStep.svelte
@@ -1,10 +1,12 @@
 <script>
-  import { Select, Label, Stepper } from "@budibase/bbui"
+  import { Select, Label } from "@budibase/bbui"
   import { currentAsset, store } from "builderStore"
   import { getActionProviderComponents } from "builderStore/dataBinding"
   import { onMount } from "svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
+  export let bindings = []
 
   $: actionProviders = getActionProviderComponents(
     $currentAsset,
@@ -51,7 +53,11 @@
   <Select bind:value={parameters.type} options={typeOptions} />
   {#if parameters.type === "specific"}
     <Label small>Number</Label>
-    <Stepper bind:value={parameters.number} />
+    <DrawerBindableInput
+      {bindings}
+      value={parameters.number}
+      on:change={e => (parameters.number = e.detail)}
+    />
   {/if}
 </div>
 

From df29a79a68139f295cfb9e15c2303875c373a65b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 17:26:02 +0100
Subject: [PATCH 38/88] Attempting to get push by digest working correctly for
 single image build.

---
 .github/workflows/release-singleimage.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index a46099908b..6c0a873c8b 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -6,7 +6,6 @@ on:
 env:
   CI: true
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  REGISTRY_IMAGE: budibase/budibase
 jobs:
   build-multiarch:
     name: "build-multiarch"
@@ -72,9 +71,9 @@ jobs:
         with:
           context: .
           platforms: ${{ matrix.platform }}
-          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
+          tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile
-          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          outputs: type=image,name=budibase/budibase,push-by-digest=true,name-canonical=true,push=true
       - name: Export digest
         run: |
           mkdir -p /tmp/digests
@@ -104,7 +103,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.REGISTRY_IMAGE }}
+          images: budibase/budibase
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
@@ -114,10 +113,10 @@ jobs:
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+            $(printf 'budibase/budibase@sha256:%s ' *)
       - name: Inspect image
         run: |
-          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect budibase/budibase:${{ steps.meta.outputs.version }}
 
   build-aas:
     name: "build-aas"

From e468ff066f85a74e36ed607fb117289be4be7010 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 8 Aug 2023 16:26:25 +0000
Subject: [PATCH 39/88] Bump version to 2.9.13

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 141491ae69..31ec0c9979 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.12",
+  "version": "2.9.13",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From b91677f0ceb234428f48e16c6bd453dd014019c7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 19:33:57 +0300
Subject: [PATCH 40/88] Fix declarations

---
 packages/backend-core/package.json | 2 +-
 packages/shared-core/package.json  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 25baccbe53..65577cf4d8 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -14,7 +14,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
-    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
+    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json
index 61e97ddc76..4e7d568fcb 100644
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@@ -14,7 +14,7 @@
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
+    "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"

From a213475041722c66f5503f5e21a4d5ab3149f9a5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 19:34:20 +0300
Subject: [PATCH 41/88] Esbuild, build esm

---
 scripts/build.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/build.js b/scripts/build.js
index b93ae1bbe6..966d5f158c 100755
--- a/scripts/build.js
+++ b/scripts/build.js
@@ -59,9 +59,16 @@ function runBuild(entry, outfile) {
     ],
   }
 
+  const { compilerOptions } = tsconfigPathPluginContent
+  const format =
+    compilerOptions.target === "es6" && compilerOptions.module !== "commonjs"
+      ? "esm"
+      : undefined
+
   build({
     ...sharedConfig,
     platform: "node",
+    format,
     outfile,
   }).then(result => {
     glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {

From 8e655dae6624d9d11cd40b63a18108b460fad317 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 19:36:36 +0300
Subject: [PATCH 42/88] Fix path

---
 packages/shared-core/tsconfig.build.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/shared-core/tsconfig.build.json b/packages/shared-core/tsconfig.build.json
index 804e86c270..31dc1afc10 100644
--- a/packages/shared-core/tsconfig.build.json
+++ b/packages/shared-core/tsconfig.build.json
@@ -14,7 +14,7 @@
     "outDir": "dist",
     "skipLibCheck": true,
     "paths": {
-      "@budibase/types": ["../../types/src"]
+      "@budibase/types": ["../types/src"]
     }
   },
   "include": ["**/*.js", "**/*.ts"],

From da87b28155596dc54945dfadca09c72677f05405 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 8 Aug 2023 19:39:32 +0300
Subject: [PATCH 43/88] Clean nx deps

---
 packages/bbui/package.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index 8a9318ba94..0b87960ab5 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -98,8 +98,7 @@
           {
             "projects": [
               "@budibase/string-templates",
-              "@budibase/shared-core",
-              "@budibase/types"
+              "@budibase/shared-core"
             ],
             "target": "build"
           }

From f754691563bd02db803c64053fcb3004b4f6c2cb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 8 Aug 2023 17:39:51 +0100
Subject: [PATCH 44/88] Reverting the changes to the single image build fully,
 adding in a workflow which should increase the storage capacity of the VM,
 almost doubling it (although it is a bit hacky, should get us by for a
 while).

---
 .github/workflows/release-singleimage.yml | 124 +++-------------------
 1 file changed, 15 insertions(+), 109 deletions(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index 6c0a873c8b..b81d04d089 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -1,4 +1,4 @@
-name: release-singleimage
+name: Deploy Budibase Single Container Image to DockerHub
 
 on:
   workflow_dispatch:
@@ -6,18 +6,20 @@ on:
 env:
   CI: true
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  REGISTRY_URL: registry.hub.docker.com
 jobs:
-  build-multiarch:
-    name: "build-multiarch"
+  build:
+    name: "build"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version:
-          - 14.x
-        platform:
-          - linux/amd64
-          - linux/arm64
+        node-version: [14.x]
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 512
+          swap-size-mb: 1024
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 
@@ -30,12 +32,14 @@ jobs:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
+
       - name: Fail if tag is not in master
         run: |
           if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
             echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
             exit 1
           fi
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
@@ -70,108 +74,10 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          platforms: ${{ matrix.platform }}
+          push: true
+          platforms: linux/amd64,linux/arm64
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile
-          outputs: type=image,name=budibase/budibase,push-by-digest=true,name-canonical=true,push=true
-      - name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      - name: Upload digest
-        uses: actions/upload-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  merge:
-    runs-on: ubuntu-latest
-    needs:
-      - build-multiarch
-    steps:
-      - name: Download digests
-        uses: actions/download-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: budibase/budibase
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf 'budibase/budibase@sha256:%s ' *)
-      - name: Inspect image
-        run: |
-          docker buildx imagetools inspect budibase/budibase:${{ steps.meta.outputs.version }}
-
-  build-aas:
-    name: "build-aas"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version:
-          - 14.x
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build:docker:pre
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:
@@ -180,4 +86,4 @@ jobs:
           platforms: linux/amd64
           build-args: TARGETBUILD=aas
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile
\ No newline at end of file

From 85903a2a30d42b62eedd5d249089fcf3caf75f93 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 8 Aug 2023 16:59:53 +0000
Subject: [PATCH 45/88] Bump version to 2.9.14

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 31ec0c9979..93db547474 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.13",
+  "version": "2.9.14",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 460789bef41a0325c522f2918041570dcd1cede8 Mon Sep 17 00:00:00 2001
From: Mel O'Hagan <mel@budibase.com>
Date: Wed, 9 Aug 2023 08:51:32 +0100
Subject: [PATCH 46/88] Parse specific step as int

---
 packages/client/src/components/app/forms/InnerForm.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte
index c0b4b2b8a8..b4370993a8 100644
--- a/packages/client/src/components/app/forms/InnerForm.svelte
+++ b/packages/client/src/components/app/forms/InnerForm.svelte
@@ -250,7 +250,7 @@
       } else if (type === "first") {
         currentStep.set(1)
       } else if (type === "specific" && number && !isNaN(number)) {
-        currentStep.set(number)
+        currentStep.set(parseInt(number))
       }
     },
     setStep: step => {

From c21da4fbd3a4529477e3993ce375dbd484f2d73b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 11:40:54 +0300
Subject: [PATCH 47/88] Consume packages from builder

---
 packages/builder/package.json   | 12 +++---------
 packages/builder/vite.config.js |  8 ++++++++
 scripts/build.js                |  5 -----
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/packages/builder/package.json b/packages/builder/package.json
index 56834f79dd..db8ef86086 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -133,9 +133,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
-              "@budibase/string-templates",
-              "@budibase/types"
+              "@budibase/string-templates"
             ],
             "target": "build"
           }
@@ -145,9 +143,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
-              "@budibase/string-templates",
-              "@budibase/types"
+              "@budibase/string-templates"
             ],
             "target": "build"
           }
@@ -157,9 +153,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
-              "@budibase/string-templates",
-              "@budibase/types"
+              "@budibase/string-templates"
             ],
             "target": "build"
           }
diff --git a/packages/builder/vite.config.js b/packages/builder/vite.config.js
index 4a0ffca8d4..b7af647916 100644
--- a/packages/builder/vite.config.js
+++ b/packages/builder/vite.config.js
@@ -127,6 +127,14 @@ export default defineConfig(({ mode }) => {
           find: "helpers",
           replacement: path.resolve("./src/helpers"),
         },
+        {
+          find: "@budibase/types",
+          replacement: path.resolve("../types/src"),
+        },
+        {
+          find: "@budibase/shared-core",
+          replacement: path.resolve("../shared-core/src"),
+        },
       ],
     },
   }
diff --git a/scripts/build.js b/scripts/build.js
index 966d5f158c..86f3fb55b9 100755
--- a/scripts/build.js
+++ b/scripts/build.js
@@ -60,15 +60,10 @@ function runBuild(entry, outfile) {
   }
 
   const { compilerOptions } = tsconfigPathPluginContent
-  const format =
-    compilerOptions.target === "es6" && compilerOptions.module !== "commonjs"
-      ? "esm"
-      : undefined
 
   build({
     ...sharedConfig,
     platform: "node",
-    format,
     outfile,
   }).then(result => {
     glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {

From 029b9d7b342b131af334d5fc4f1b7aa3c8f1890c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 11:46:23 +0300
Subject: [PATCH 48/88] Clean

---
 scripts/build.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/build.js b/scripts/build.js
index 86f3fb55b9..b93ae1bbe6 100755
--- a/scripts/build.js
+++ b/scripts/build.js
@@ -59,8 +59,6 @@ function runBuild(entry, outfile) {
     ],
   }
 
-  const { compilerOptions } = tsconfigPathPluginContent
-
   build({
     ...sharedConfig,
     platform: "node",

From 32e09cf131ece2eb67e3fb83f3bbb69660690e6a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 12:13:38 +0300
Subject: [PATCH 49/88] Lint

---
 packages/backend-core/scripts/build.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/backend-core/scripts/build.js b/packages/backend-core/scripts/build.js
index 992513755a..bd00cbc7ff 100644
--- a/packages/backend-core/scripts/build.js
+++ b/packages/backend-core/scripts/build.js
@@ -1,6 +1,4 @@
 #!/usr/bin/node
-const { join } = require("path")
-const fs = require("fs")
 const coreBuild = require("../../../scripts/build")
 
 coreBuild("./src/plugin/index.ts", "./dist/plugins.js")

From a5fb25e41ec179f8d879267ae6e8304b6ce20706 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Wed, 9 Aug 2023 10:38:04 +0100
Subject: [PATCH 50/88] Allow null for firstName and lastName (#11482)

---
 packages/worker/src/api/routes/validation/users.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/worker/src/api/routes/validation/users.ts b/packages/worker/src/api/routes/validation/users.ts
index 30a3d67434..dfc1e6fbbf 100644
--- a/packages/worker/src/api/routes/validation/users.ts
+++ b/packages/worker/src/api/routes/validation/users.ts
@@ -1,12 +1,14 @@
 import { auth } from "@budibase/backend-core"
 import Joi from "joi"
 
+const OPTIONAL_STRING = Joi.string().allow(null, "")
+
 let schema: any = {
-  email: Joi.string().allow(null, ""),
-  password: Joi.string().allow(null, ""),
+  email: OPTIONAL_STRING,
+  password: OPTIONAL_STRING,
   forceResetPassword: Joi.boolean().optional(),
-  firstName: Joi.string().allow(null, ""),
-  lastName: Joi.string().allow(null, ""),
+  firstName: OPTIONAL_STRING,
+  lastName: OPTIONAL_STRING,
   builder: Joi.object({
     global: Joi.boolean().optional(),
     apps: Joi.array().optional(),
@@ -21,8 +23,8 @@ export const buildSelfSaveValidation = () => {
   schema = {
     password: Joi.string().optional(),
     forceResetPassword: Joi.boolean().optional(),
-    firstName: Joi.string().allow("").optional(),
-    lastName: Joi.string().allow("").optional(),
+    firstName: OPTIONAL_STRING,
+    lastName: OPTIONAL_STRING,
     onboardedAt: Joi.string().optional(),
   }
   return auth.joiValidator.body(Joi.object(schema).required().unknown(false))

From 0fc1f13daad722505d5eea904d90acad66e8822a Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 09:38:26 +0000
Subject: [PATCH 51/88] Bump version to 2.9.15

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 93db547474..9a9c4ec560 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.14",
+  "version": "2.9.15",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From c1438b2988ca1459781d1617c15c2152affb77b5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 9 Aug 2023 10:57:05 +0100
Subject: [PATCH 52/88] Attempting to get maximise storage working for our use
 case.

---
 .github/workflows/release-singleimage.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index b81d04d089..7f1cf94704 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -18,8 +18,9 @@ jobs:
       - name: Maximize build space
         uses: easimon/maximize-build-space@master
         with:
-          root-reserve-mb: 512
+          root-reserve-mb: 4096
           swap-size-mb: 1024
+          remove-android: 'true'
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 
@@ -86,4 +87,4 @@ jobs:
           platforms: linux/amd64
           build-args: TARGETBUILD=aas
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
\ No newline at end of file
+          file: ./hosting/single/Dockerfile

From af2a2f4f056cead50f9f8c4cb25a0dc1eebecb20 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 09:58:01 +0000
Subject: [PATCH 53/88] Bump version to 2.9.16

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 9a9c4ec560..bc5c78d3f3 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.15",
+  "version": "2.9.16",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From fd677e5a866233baaa57573706ba301d7e5b8e25 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 10:13:41 +0000
Subject: [PATCH 54/88] Bump version to 2.9.17

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index bc5c78d3f3..d48dbc4b86 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.16",
+  "version": "2.9.17",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From b5e4fdec8d886035fcfe5aa42c662e9148ed79f9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 9 Aug 2023 11:35:35 +0100
Subject: [PATCH 55/88] Up-ing root size.

---
 .github/workflows/release-singleimage.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml
index 7f1cf94704..b879b519d2 100644
--- a/.github/workflows/release-singleimage.yml
+++ b/.github/workflows/release-singleimage.yml
@@ -18,9 +18,10 @@ jobs:
       - name: Maximize build space
         uses: easimon/maximize-build-space@master
         with:
-          root-reserve-mb: 4096
+          root-reserve-mb: 35000
           swap-size-mb: 1024
           remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 

From 56c82cc6971bfb178095ec30cbb38d74d8e55cd0 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 10:35:58 +0000
Subject: [PATCH 56/88] Bump version to 2.9.18

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d48dbc4b86..d506ef3022 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.17",
+  "version": "2.9.18",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 9f842234dc551fb749f4c0b2b0b5f1bcaa94eca6 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 10:40:10 +0000
Subject: [PATCH 57/88] Bump version to 2.9.19-alpha.0

---
 lerna.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lerna.json b/lerna.json
index ed343f9d5e..620924d95e 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.18",
+  "version": "2.9.19-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
@@ -19,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}
\ No newline at end of file

From f55aed2ee524519c0b7c4ba1e337494a1d1dbca5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 9 Aug 2023 13:31:30 +0100
Subject: [PATCH 58/88] Adding NPM ignore files so that doesn't default to the
 .gitignore when publishing to NPM.

---
 packages/backend-core/.npmignore | 7 +++++++
 packages/shared-core/.npmignore  | 3 +++
 2 files changed, 10 insertions(+)
 create mode 100644 packages/backend-core/.npmignore
 create mode 100644 packages/shared-core/.npmignore

diff --git a/packages/backend-core/.npmignore b/packages/backend-core/.npmignore
new file mode 100644
index 0000000000..d1b8eaa79b
--- /dev/null
+++ b/packages/backend-core/.npmignore
@@ -0,0 +1,7 @@
+node_modules/
+tsconfig.build.json
+tsconfig.json
+jest.config.ts
+jest-testcontainers-config.js
+__mocks__
+scripts
diff --git a/packages/shared-core/.npmignore b/packages/shared-core/.npmignore
new file mode 100644
index 0000000000..4926fbc5fd
--- /dev/null
+++ b/packages/shared-core/.npmignore
@@ -0,0 +1,3 @@
+node_modules/
+tsconfig.build.json
+tsconfig.json

From ea3f4e8126c661d3a054d667a588de2209976c61 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 12:44:41 +0000
Subject: [PATCH 59/88] Bump version to 2.9.19

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d506ef3022..8ec6eddf37 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.18",
+  "version": "2.9.19",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From bfdb7854364f10bbf680ef66e9b451f58bd93fe6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 16:36:14 +0300
Subject: [PATCH 60/88] Tweak shared-core content to exclude src

---
 packages/shared-core/.npmignore   |  8 +++++---
 packages/shared-core/package.json | 10 ++--------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/packages/shared-core/.npmignore b/packages/shared-core/.npmignore
index 4926fbc5fd..268f19d4fe 100644
--- a/packages/shared-core/.npmignore
+++ b/packages/shared-core/.npmignore
@@ -1,3 +1,5 @@
-node_modules/
-tsconfig.build.json
-tsconfig.json
+*
+!dist/*
+dist/tsconfig.build.tsbuildinfo
+!package.json
+!LICENCE
\ No newline at end of file
diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json
index 4e7d568fcb..b097a46c57 100644
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@@ -2,14 +2,8 @@
   "name": "@budibase/shared-core",
   "version": "0.0.0",
   "description": "Shared data utils",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "require": "./src/index.ts"
-    }
-  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {

From 90398ced3e8cac5f046fa35445a482a243d7a553 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 16:37:31 +0300
Subject: [PATCH 61/88] Tweak types npm content

---
 packages/types/.npmignore   |  5 +++++
 packages/types/package.json | 10 ++--------
 2 files changed, 7 insertions(+), 8 deletions(-)
 create mode 100644 packages/types/.npmignore

diff --git a/packages/types/.npmignore b/packages/types/.npmignore
new file mode 100644
index 0000000000..268f19d4fe
--- /dev/null
+++ b/packages/types/.npmignore
@@ -0,0 +1,5 @@
+*
+!dist/*
+dist/tsconfig.build.tsbuildinfo
+!package.json
+!LICENCE
\ No newline at end of file
diff --git a/packages/types/package.json b/packages/types/package.json
index 317342e760..bb0163ae13 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -2,14 +2,8 @@
   "name": "@budibase/types",
   "version": "0.0.0",
   "description": "Budibase types",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "require": "./src/index.ts"
-    }
-  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {

From 4c7dfc978e94d80beba303002008026f7990e73e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 16:41:44 +0300
Subject: [PATCH 62/88] Whitelist .npmignore from backend-core

---
 packages/backend-core/.npmignore | 11 ++++-------
 packages/shared-core/.npmignore  |  3 +--
 packages/types/.npmignore        |  3 +--
 3 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/packages/backend-core/.npmignore b/packages/backend-core/.npmignore
index d1b8eaa79b..967c7d11da 100644
--- a/packages/backend-core/.npmignore
+++ b/packages/backend-core/.npmignore
@@ -1,7 +1,4 @@
-node_modules/
-tsconfig.build.json
-tsconfig.json
-jest.config.ts
-jest-testcontainers-config.js
-__mocks__
-scripts
+*
+!dist/*
+dist/tsconfig.build.tsbuildinfo
+!package.json
\ No newline at end of file
diff --git a/packages/shared-core/.npmignore b/packages/shared-core/.npmignore
index 268f19d4fe..967c7d11da 100644
--- a/packages/shared-core/.npmignore
+++ b/packages/shared-core/.npmignore
@@ -1,5 +1,4 @@
 *
 !dist/*
 dist/tsconfig.build.tsbuildinfo
-!package.json
-!LICENCE
\ No newline at end of file
+!package.json
\ No newline at end of file
diff --git a/packages/types/.npmignore b/packages/types/.npmignore
index 268f19d4fe..967c7d11da 100644
--- a/packages/types/.npmignore
+++ b/packages/types/.npmignore
@@ -1,5 +1,4 @@
 *
 !dist/*
 dist/tsconfig.build.tsbuildinfo
-!package.json
-!LICENCE
\ No newline at end of file
+!package.json
\ No newline at end of file

From b441cbe3ec707acce77ac4219eeae4efd7bbde71 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 17:06:05 +0300
Subject: [PATCH 63/88] Include all dist

---
 packages/backend-core/.npmignore | 2 +-
 packages/shared-core/.npmignore  | 2 +-
 packages/types/.npmignore        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/.npmignore b/packages/backend-core/.npmignore
index 967c7d11da..fb547825eb 100644
--- a/packages/backend-core/.npmignore
+++ b/packages/backend-core/.npmignore
@@ -1,4 +1,4 @@
 *
-!dist/*
+!dist/**/*
 dist/tsconfig.build.tsbuildinfo
 !package.json
\ No newline at end of file
diff --git a/packages/shared-core/.npmignore b/packages/shared-core/.npmignore
index 967c7d11da..fb547825eb 100644
--- a/packages/shared-core/.npmignore
+++ b/packages/shared-core/.npmignore
@@ -1,4 +1,4 @@
 *
-!dist/*
+!dist/**/*
 dist/tsconfig.build.tsbuildinfo
 !package.json
\ No newline at end of file
diff --git a/packages/types/.npmignore b/packages/types/.npmignore
index 967c7d11da..fb547825eb 100644
--- a/packages/types/.npmignore
+++ b/packages/types/.npmignore
@@ -1,4 +1,4 @@
 *
-!dist/*
+!dist/**/*
 dist/tsconfig.build.tsbuildinfo
 !package.json
\ No newline at end of file

From 9734776953e98f7c01b600dfbb1d17394652c5e7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 17:20:36 +0300
Subject: [PATCH 64/88] Add required nx dependencies

---
 packages/backend-core/package.json | 15 +++++++++++++++
 packages/shared-core/package.json  | 14 ++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 65577cf4d8..04efe7f5c6 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -88,5 +88,20 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }
diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json
index b097a46c57..382f3fa731 100644
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@@ -20,5 +20,19 @@
     "concurrently": "^7.6.0",
     "rimraf": "3.0.2",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

From 7c12280fe398bce4ef0b3e2c22ef852d344c0f69 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 14:34:26 +0000
Subject: [PATCH 65/88] Bump version to 2.9.20

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 8ec6eddf37..9d89818cbd 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.19",
+  "version": "2.9.20",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 1c6b17ee326883aa3e4e545495210f411d1a287e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 18:02:45 +0300
Subject: [PATCH 66/88] Use affected only on PRs, excluding master

---
 .github/workflows/budibase_ci.yml | 43 ++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index cc1244e01a..af135b757b 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,8 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: origin/${{ github.base_ref || 'master'}}
+  NX_BASE_BRANCH: origin/${{ github.base_ref }}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
 
 jobs:
   lint:
@@ -66,9 +67,21 @@ jobs:
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
-      - run: yarn build --since=${{ env.NX_BASE_BRANCH }}
+      - name: Build
+        run: |
+          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+            yarn build --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn build
+          fi
       # Check the types of the projects built via esbuild
-      - run: yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+      - name: Check types
+        run: |
+          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn check:types
+          fi
 
   test-libraries:
     runs-on: ubuntu-latest
@@ -92,7 +105,13 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+      - name: Test
+        run: |
+          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+          fi
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -122,7 +141,13 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Test worker and server
-        run: yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+        run: |
+          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/worker --scope=@budibase/server
+          fi
+
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -146,7 +171,13 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+      - name: Test
+        run: |
+          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+            yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/pro
+          fi
 
   integration-test:
     runs-on: ubuntu-latest

From 63006e36d4b1203f826a66f3c9a3aff341e357e5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 9 Aug 2023 16:24:57 +0100
Subject: [PATCH 67/88] Adding the ability to disable an automation, as well
 making sure that automations have steps before running.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts | 14 ++++++
 .../server/src/api/controllers/application.ts |  4 +-
 .../server/src/api/controllers/automation.ts  | 29 ++++++------
 packages/server/src/automations/triggers.ts   |  4 +-
 packages/server/src/automations/utils.ts      | 25 +++-------
 .../src/sdk/app/automations/automations.ts    | 38 +++++++++++++++
 .../server/src/sdk/app/automations/index.ts   |  2 +
 packages/server/src/threads/automation.ts     | 47 ++++++++++++++-----
 .../types/src/documents/app/automation.ts     |  5 ++
 packages/types/src/sdk/db.ts                  |  6 +++
 10 files changed, 124 insertions(+), 50 deletions(-)
 create mode 100644 packages/server/src/sdk/app/automations/automations.ts

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 29ca4123f5..89f76769b3 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -8,6 +8,7 @@ import {
   DatabasePutOpts,
   DatabaseCreateIndexOpts,
   DatabaseDeleteIndexOpts,
+  DocExistsResponse,
   Document,
   isDocument,
 } from "@budibase/types"
@@ -120,6 +121,19 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
+  async docExists(docId: string): Promise<DocExistsResponse> {
+    const db = await this.checkSetup()
+    let _rev, exists
+    try {
+      const { etag } = await db.head(docId)
+      _rev = etag
+      exists = true
+    } catch (err) {
+      exists = false
+    }
+    return { _rev, exists }
+  }
+
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index c5556479fe..f8b6bd6d16 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -39,9 +39,8 @@ import {
 } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream } from "../../utilities"
-import { doesUserHaveLock, getLocksById } from "../../utilities/redis"
+import { doesUserHaveLock } from "../../utilities/redis"
 import { cleanupAutomations } from "../../automations/utils"
-import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { groups, licensing, quotas } from "@budibase/pro"
 import {
@@ -51,7 +50,6 @@ import {
   PlanType,
   Screen,
   UserCtx,
-  ContextUser,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts
index a4697f99f2..c861782a02 100644
--- a/packages/server/src/api/controllers/automation.ts
+++ b/packages/server/src/api/controllers/automation.ts
@@ -20,7 +20,7 @@ import {
   Automation,
   AutomationActionStepId,
   AutomationResults,
-  BBContext,
+  Ctx,
 } from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
 import sdk from "../../sdk"
@@ -73,7 +73,7 @@ function cleanAutomationInputs(automation: Automation) {
   return automation
 }
 
-export async function create(ctx: BBContext) {
+export async function create(ctx: Ctx) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -142,7 +142,7 @@ export async function handleStepEvents(
   }
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: Ctx) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -193,7 +193,7 @@ export async function update(ctx: BBContext) {
   builderSocket?.emitAutomationUpdate(ctx, automation)
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: Ctx) {
   const db = context.getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
@@ -203,12 +203,11 @@ export async function fetch(ctx: BBContext) {
   ctx.body = response.rows.map(row => row.doc)
 }
 
-export async function find(ctx: BBContext) {
-  const db = context.getAppDB()
-  ctx.body = await db.get(ctx.params.id)
+export async function find(ctx: Ctx) {
+  ctx.body = await sdk.automations.get(ctx.params.id)
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get<Automation>(automationId)
@@ -222,11 +221,11 @@ export async function destroy(ctx: BBContext) {
   builderSocket?.emitAutomationDeletion(ctx, automationId)
 }
 
-export async function logSearch(ctx: BBContext) {
+export async function logSearch(ctx: Ctx) {
   ctx.body = await automations.logs.logSearch(ctx.request.body)
 }
 
-export async function clearLogError(ctx: BBContext) {
+export async function clearLogError(ctx: Ctx) {
   const { automationId, appId } = ctx.request.body
   await context.doInAppContext(appId, async () => {
     const db = context.getProdAppDB()
@@ -245,15 +244,15 @@ export async function clearLogError(ctx: BBContext) {
   })
 }
 
-export async function getActionList(ctx: BBContext) {
+export async function getActionList(ctx: Ctx) {
   ctx.body = await getActionDefinitions()
 }
 
-export async function getTriggerList(ctx: BBContext) {
+export async function getTriggerList(ctx: Ctx) {
   ctx.body = getTriggerDefinitions()
 }
 
-export async function getDefinitionList(ctx: BBContext) {
+export async function getDefinitionList(ctx: Ctx) {
   ctx.body = {
     trigger: getTriggerDefinitions(),
     action: await getActionDefinitions(),
@@ -266,7 +265,7 @@ export async function getDefinitionList(ctx: BBContext) {
  *                   *
  *********************/
 
-export async function trigger(ctx: BBContext) {
+export async function trigger(ctx: Ctx) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
 
@@ -311,7 +310,7 @@ function prepareTestInput(input: any) {
   return input
 }
 
-export async function test(ctx: BBContext) {
+export async function test(ctx: Ctx) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
   await setTestFlag(automation._id!)
diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts
index 922bc10343..9cc8c6b077 100644
--- a/packages/server/src/automations/triggers.ts
+++ b/packages/server/src/automations/triggers.ts
@@ -6,11 +6,11 @@ import { isDevAppID } from "../db/utils"
 // need this to call directly, so we can get a response
 import { automationQueue } from "./bullboard"
 import { checkTestFlag } from "../utilities/redis"
-import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"
 import { executeSynchronously } from "../threads/automation"
+import sdk from "../sdk"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -142,7 +142,7 @@ export async function rebootTrigger() {
       let automations = await getAllAutomations()
       let rebootEvents = []
       for (let automation of automations) {
-        if (utils.isRebootTrigger(automation)) {
+        if (sdk.automations.isReboot(automation)) {
           const job = {
             automation,
             event: {
diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index 194eff5b32..add5637bfd 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -128,19 +128,6 @@ export async function clearMetadata() {
   await db.bulkDocs(automationMetadata)
 }
 
-export function isCronTrigger(auto: Automation) {
-  return (
-    auto &&
-    auto.definition.trigger &&
-    auto.definition.trigger.stepId === CRON_STEP_ID
-  )
-}
-
-export function isRebootTrigger(auto: Automation) {
-  const trigger = auto ? auto.definition.trigger : null
-  return isCronTrigger(auto) && trigger?.inputs.cron === REBOOT_CRON
-}
-
 /**
  * This function handles checking of any cron jobs that need to be enabled/updated.
  * @param {string} appId The ID of the app in which we are checking for webhooks
@@ -148,13 +135,15 @@ export function isRebootTrigger(auto: Automation) {
  */
 export async function enableCronTrigger(appId: any, automation: Automation) {
   const trigger = automation ? automation.definition.trigger : null
+  const validCron =
+    sdk.automations.isCron(automation) &&
+    trigger?.inputs.cron
+  const needsCreated =
+    !sdk.automations.isReboot(automation) &&
+    !sdk.automations.disabled(automation)
 
   // need to create cron job
-  if (
-    isCronTrigger(automation) &&
-    !isRebootTrigger(automation) &&
-    trigger?.inputs.cron
-  ) {
+  if (validCron && needsCreated) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
     const job: any = await automationQueue.add(
diff --git a/packages/server/src/sdk/app/automations/automations.ts b/packages/server/src/sdk/app/automations/automations.ts
new file mode 100644
index 0000000000..80f84c90e2
--- /dev/null
+++ b/packages/server/src/sdk/app/automations/automations.ts
@@ -0,0 +1,38 @@
+import { context } from "@budibase/backend-core"
+import { Automation, AutomationState, DocumentType } from "@budibase/types"
+import { definitions } from "../../../automations/triggerInfo"
+
+const REBOOT_CRON = "@reboot"
+
+export async function exists(automationId: string) {
+  if (!automationId?.startsWith(DocumentType.AUTOMATION)) {
+    throw new Error("Invalid automation ID.")
+  }
+  const db = context.getAppDB()
+  return db.docExists(automationId)
+}
+
+export async function get(automationId: string) {
+  const db = context.getAppDB()
+  return (await db.get(automationId)) as Automation
+}
+
+export function disabled(automation: Automation) {
+  return automation.state === AutomationState.DISABLED || !hasSteps(automation)
+}
+
+export function isCron(automation: Automation) {
+  return (
+    automation?.definition.trigger &&
+    automation?.definition.trigger.stepId === definitions.CRON.stepId
+  )
+}
+
+export function isReboot(automation: Automation) {
+  const trigger = automation?.definition.trigger
+  return isCron(automation) && trigger?.inputs.cron === REBOOT_CRON
+}
+
+export function hasSteps(automation: Automation) {
+  return automation?.definition?.steps?.length > 0
+}
diff --git a/packages/server/src/sdk/app/automations/index.ts b/packages/server/src/sdk/app/automations/index.ts
index 16530cf085..540d5545fc 100644
--- a/packages/server/src/sdk/app/automations/index.ts
+++ b/packages/server/src/sdk/app/automations/index.ts
@@ -1,7 +1,9 @@
 import * as webhook from "./webhook"
 import * as utils from "./utils"
+import * as automations from "./automations"
 
 export default {
   webhook,
   utils,
+  ...automations,
 }
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index bc0629a939..5a79444531 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -1,10 +1,9 @@
 import { default as threadUtils } from "./utils"
 import { Job } from "bull"
-threadUtils.threadSetup()
 import {
-  isRecurring,
   disableCronById,
   isErrorInOutput,
+  isRecurring,
 } from "../automations/utils"
 import * as actions from "../automations/actions"
 import * as automationUtils from "../automations/automationUtils"
@@ -15,17 +14,17 @@ import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
 import { storeLog } from "../automations/logging"
 import {
   Automation,
-  AutomationStep,
-  AutomationStatus,
-  AutomationMetadata,
-  AutomationJob,
   AutomationData,
+  AutomationJob,
+  AutomationMetadata,
+  AutomationStatus,
+  AutomationStep,
 } from "@budibase/types"
 import {
-  LoopStep,
-  LoopInput,
-  TriggerOutput,
   AutomationContext,
+  LoopInput,
+  LoopStep,
+  TriggerOutput,
 } from "../definitions/automations"
 import { WorkerCallback } from "./definitions"
 import { context, logging } from "@budibase/backend-core"
@@ -34,6 +33,10 @@ import { cloneDeep } from "lodash/fp"
 import { performance } from "perf_hooks"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
+import sdk from "../sdk"
+
+threadUtils.threadSetup()
+
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
@@ -514,7 +517,8 @@ class Orchestrator {
 
 export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
   const appId = job.data.event.appId
-  const automationId = job.data.automation._id
+  const automation = job.data.automation
+  const automationId = automation._id
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
@@ -525,10 +529,30 @@ export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
     appId,
     automationId,
     task: async () => {
+      let automation = job.data.automation,
+        isCron = sdk.automations.isCron(job.data.automation),
+        notFound = false
+      try {
+        automation = await sdk.automations.get(automationId)
+      } catch (err: any) {
+        // automation no longer exists
+        notFound = err
+      }
+      const disabled = sdk.automations.disabled(automation)
+      const stopAutomation = disabled || notFound
       const envVars = await sdkUtils.getEnvironmentVariables()
       // put into automation thread for whole context
       await context.doInEnvironmentContext(envVars, async () => {
         const automationOrchestrator = new Orchestrator(job)
+        // hard stop on automations
+        if (isCron && stopAutomation) {
+          await automationOrchestrator.stopCron(
+            disabled ? "disabled" : "not_found"
+          )
+        }
+        if (stopAutomation) {
+          return
+        }
         try {
           const response = await automationOrchestrator.execute()
           callback(null, response)
@@ -557,11 +581,10 @@ export function executeSynchronously(job: Job) {
     // put into automation thread for whole context
     return context.doInEnvironmentContext(envVars, async () => {
       const automationOrchestrator = new Orchestrator(job)
-      const response = await Promise.race([
+      return await Promise.race([
         automationOrchestrator.execute(),
         timeoutPromise,
       ])
-      return response
     })
   })
 }
diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts
index d1dbbec21b..05ac76d5e9 100644
--- a/packages/types/src/documents/app/automation.ts
+++ b/packages/types/src/documents/app/automation.ts
@@ -100,6 +100,10 @@ export const AutomationStepIdArray = [
   ...Object.values(AutomationTriggerStepId),
 ]
 
+export enum AutomationState {
+  DISABLED = "disabled",
+}
+
 export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
@@ -112,6 +116,7 @@ export interface Automation extends Document {
   name: string
   internal?: boolean
   type?: string
+  state?: AutomationState
 }
 
 interface BaseIOStructure {
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index e07cc82fd2..9ac8ff26d4 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -40,6 +40,11 @@ export type DatabasePutOpts = {
   force?: boolean
 }
 
+export type DocExistsResponse = {
+  _rev?: string
+  exists: boolean
+}
+
 export type DatabaseCreateIndexOpts = {
   index: {
     fields: string[]
@@ -90,6 +95,7 @@ export interface Database {
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
   get<T>(id?: string): Promise<T>
+  docExists(id: string): Promise<DocExistsResponse>
   remove(
     id: string | Document,
     rev?: string

From dbfdc242d1afcb7952ecb5366a43722521448213 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 15:25:47 +0000
Subject: [PATCH 68/88] Bump version to 2.9.21-alpha.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 9d89818cbd..eb3f1bd6aa 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.20",
+  "version": "2.9.21-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 2622fecd93389144b2f6bb1620352a12f05aadad Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 9 Aug 2023 16:33:37 +0100
Subject: [PATCH 69/88] Quick fix.

---
 packages/server/src/threads/automation.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 5a79444531..8bf0b506fe 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -1,5 +1,6 @@
 import { default as threadUtils } from "./utils"
 import { Job } from "bull"
+threadUtils.threadSetup()
 import {
   disableCronById,
   isErrorInOutput,
@@ -35,8 +36,6 @@ import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
 import sdk from "../sdk"
 
-threadUtils.threadSetup()
-
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId

From a3a8d35b8d12ef83044989f62a65e703e8166f59 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 9 Aug 2023 16:58:46 +0100
Subject: [PATCH 70/88] Adding logging of start, end and duration from
 automation logs.

---
 packages/server/src/automations/utils.ts | 48 +++++++++++++++++++-----
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index add5637bfd..7b0ab4f2fd 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -16,13 +16,14 @@ import {
 } from "@budibase/types"
 import sdk from "../sdk"
 
-const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
-const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
 
-function loggingArgs(job: AutomationJob) {
-  return [
+function loggingArgs(
+  job: AutomationJob,
+  timing?: { start: number; complete?: boolean }
+) {
+  const logs: any[] = [
     {
       _logKey: "automation",
       trigger: job.data.automation.definition.trigger.event,
@@ -32,24 +33,53 @@ function loggingArgs(job: AutomationJob) {
       jobId: job.id,
     },
   ]
+  if (timing?.start) {
+    logs.push({
+      _logKey: "startTime",
+      start: timing.start,
+    })
+  }
+  if (timing?.start && timing?.complete) {
+    const end = new Date().getTime()
+    const duration = end - timing.start
+    logs.push({
+      _logKey: "endTime",
+      end,
+    })
+    logs.push({
+      _logKey: "duration",
+      duration,
+    })
+  }
+  return logs
 }
 
 export async function processEvent(job: AutomationJob) {
   const appId = job.data.event.appId!
   const automationId = job.data.automation._id!
+  const start = new Date().getTime()
   const task = async () => {
     try {
       // need to actually await these so that an error can be captured properly
-      console.log("automation running", ...loggingArgs(job))
+      console.log("automation running", ...loggingArgs(job, { start }))
 
       const runFn = () => Runner.run(job)
       const result = await quotas.addAutomation(runFn, {
         automationId,
       })
-      console.log("automation completed", ...loggingArgs(job))
+      const end = new Date().getTime()
+      const duration = end - start
+      console.log(
+        "automation completed",
+        ...loggingArgs(job, { start, complete: true })
+      )
       return result
     } catch (err) {
-      console.error(`automation was unable to run`, err, ...loggingArgs(job))
+      console.error(
+        `automation was unable to run`,
+        err,
+        ...loggingArgs(job, { start, complete: true })
+      )
       return { err }
     }
   }
@@ -135,9 +165,7 @@ export async function clearMetadata() {
  */
 export async function enableCronTrigger(appId: any, automation: Automation) {
   const trigger = automation ? automation.definition.trigger : null
-  const validCron =
-    sdk.automations.isCron(automation) &&
-    trigger?.inputs.cron
+  const validCron = sdk.automations.isCron(automation) && trigger?.inputs.cron
   const needsCreated =
     !sdk.automations.isReboot(automation) &&
     !sdk.automations.disabled(automation)

From 91f9338741af8e821c06358f53266c2cd96e9361 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 9 Aug 2023 16:43:47 +0000
Subject: [PATCH 71/88] Bump version to 2.9.21

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 9d89818cbd..05fb4a3b9d 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.20",
+  "version": "2.9.21",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From fb4b439f7b10f0c2a3ac7410e4158ca46f3080c0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 11:10:47 +0300
Subject: [PATCH 72/88] Fix ifs

---
 .github/workflows/budibase_ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index af135b757b..a94c56407f 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -69,7 +69,7 @@ jobs:
       # Run build all the projects
       - name: Build
         run: |
-          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+          if ${{ env.USE_NX_AFFECTED }}; then
             yarn build --since=${{ env.NX_BASE_BRANCH }}
           else
             yarn build
@@ -77,7 +77,7 @@ jobs:
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |
-          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+          if ${{ env.USE_NX_AFFECTED }}; then
             yarn check:types --since=${{ env.NX_BASE_BRANCH }}
           else
             yarn check:types
@@ -107,7 +107,7 @@ jobs:
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
-          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+          if ${{ env.USE_NX_AFFECTED }}; then
             yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
           else
             yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
@@ -142,7 +142,7 @@ jobs:
       - run: yarn --frozen-lockfile
       - name: Test worker and server
         run: |
-          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+          if ${{ env.USE_NX_AFFECTED }}; then
             yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
           else
             yarn test --scope=@budibase/worker --scope=@budibase/server
@@ -173,7 +173,7 @@ jobs:
       - run: yarn --frozen-lockfile
       - name: Test
         run: |
-          if [[ ${{ env.USE_NX_AFFECTED }} ]]; then
+          if ${{ env.USE_NX_AFFECTED }}; then
             yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
           else
             yarn test --scope=@budibase/pro

From 97e64b88aabd7ac9367f8eaa3b7ded3e8e7a6cd2 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 08:23:37 +0000
Subject: [PATCH 73/88] Bump version to 2.9.21-alpha.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index eb3f1bd6aa..a7edba2921 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.21-alpha.0",
+  "version": "2.9.21-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From f226d7d093fc42e186e7386447aabd47981591a8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 10:07:05 +0100
Subject: [PATCH 74/88] Update package.json

---
 packages/backend-core/package.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 04efe7f5c6..1d6aa5b9fd 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -2,11 +2,11 @@
   "name": "@budibase/backend-core",
   "version": "0.0.0",
   "description": "Budibase backend core libraries used in server and worker",
-  "main": "dist/src/index.js",
+  "main": "dist/index.js",
   "types": "dist/src/index.d.ts",
   "exports": {
-    ".": "./dist/src/index.js",
-    "./tests": "./dist/tests/index.js",
+    ".": "./dist/index.js",
+    "./tests": "./dist/tests.js",
     "./*": "./dist/*.js"
   },
   "author": "Budibase",

From a7be5b9ff7d7e8be88d653575a6f50ca3469d881 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 09:35:57 +0000
Subject: [PATCH 75/88] Bump version to 2.9.22

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 05fb4a3b9d..c90d8fb9cf 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.21",
+  "version": "2.9.22",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 90e9263dc73921df8e1a38099bbee85d9451c5da Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 13:00:02 +0300
Subject: [PATCH 76/88] Update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 3495feff36..6dcfd9833c 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 3495feff36b73ab9abf2cc45df39900e79d38e27
+Subproject commit 6dcfd9833c4c4fbd87d2e9a064d5af09eebc22b2

From d6da826c9fdbdc4c65f86e54406fe622dfd634dc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 10 Aug 2023 11:07:25 +0100
Subject: [PATCH 77/88] Further PR comments.

---
 packages/server/src/api/controllers/row/utils.ts | 8 ++++----
 packages/types/src/documents/document.ts         | 1 -
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index 157f18e231..e85ec4553c 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -49,14 +49,14 @@ export function getTableId(ctx: Ctx) {
   if (ctx.params?.sourceId) {
     return ctx.params.sourceId
   }
-  // check body for a table ID
-  if (ctx.request.body?.tableId) {
-    return ctx.request.body.tableId
-  }
   // now check for old way of specifying table ID
   if (ctx.params?.tableId) {
     return ctx.params.tableId
   }
+  // check body for a table ID
+  if (ctx.request.body?.tableId) {
+    return ctx.request.body.tableId
+  }
   // now check if a specific view name
   if (ctx.params?.viewName) {
     return ctx.params.viewName
diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts
index 75f55e1367..03e01907b8 100644
--- a/packages/types/src/documents/document.ts
+++ b/packages/types/src/documents/document.ts
@@ -37,7 +37,6 @@ export enum DocumentType {
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
   AUDIT_LOG = "al",
-  VIEW = "awd",
 }
 
 // these documents don't really exist, they are part of other

From febf51a9510c38c57eee4456f2cd37b2c5d13bd2 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 10:19:59 +0000
Subject: [PATCH 78/88] Bump version to 2.9.21-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a7edba2921..e489d6e5ee 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.21-alpha.1",
+  "version": "2.9.21-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 5c66a10f36f56e6a419b17a006ae033bc5be18e9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 14:19:01 +0300
Subject: [PATCH 79/88] Update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 6dcfd9833c..de3bd2eda7 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 6dcfd9833c4c4fbd87d2e9a064d5af09eebc22b2
+Subproject commit de3bd2eda75cdd6993c8208998dda5e0c3bccc8a

From b4615af979c05d9fdf858a045478f634808d748e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 14:26:33 +0300
Subject: [PATCH 80/88] Update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index de3bd2eda7..02626390cd 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit de3bd2eda75cdd6993c8208998dda5e0c3bccc8a
+Subproject commit 02626390cde905a248cb60729968667c9e49fae9

From ad98588f306abc6ff8a5d3a1f694ac030204b43d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 11:39:18 +0000
Subject: [PATCH 81/88] Bump version to 2.9.23

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index c90d8fb9cf..0df96afd31 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.22",
+  "version": "2.9.23",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 415c5de46769cc04e3c7e96045f9b9085a804af2 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 12:43:22 +0000
Subject: [PATCH 82/88] Bump version to 2.9.24-alpha.0

---
 lerna.json | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lerna.json b/lerna.json
index 84e3a6b634..b03609303e 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,7 +1,9 @@
 {
-  "version": "2.9.23",
+  "version": "2.9.24-alpha.0",
   "npmClient": "yarn",
-  "packages": ["packages/*"],
+  "packages": [
+    "packages/*"
+  ],
   "useNx": true,
   "command": {
     "publish": {
@@ -17,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}
\ No newline at end of file

From 65e5e40963bbf7b737babaad390073657f76c096 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 17:24:48 +0300
Subject: [PATCH 83/88] Fix python installation in server docker and single
 image

---
 hosting/single/Dockerfile  | 2 +-
 packages/server/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile
index 6fc1f96d28..9fdf2449d1 100644
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@@ -1,7 +1,7 @@
 FROM node:18-slim as build
 
 # install node-gyp dependencies
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python3
 
 # add pin script
 WORKDIR /
diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile
index 61f064ecfb..e1b3b208c7 100644
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@@ -18,7 +18,7 @@ ENV TOP_LEVEL_PATH=/
 
 # handle node-gyp
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ make python
+    && apt-get install -y --no-install-recommends g++ make python3
 RUN yarn global add pm2
 
 # Install client for oracle datasource

From 5ff26ab2da925032946b9fc4715b083d10c3498c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 17:33:31 +0300
Subject: [PATCH 84/88] Do not build using affected

---
 .github/workflows/budibase_ci.yml | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index a94c56407f..b754587d34 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -69,19 +69,11 @@ jobs:
       # Run build all the projects
       - name: Build
         run: |
-          if ${{ env.USE_NX_AFFECTED }}; then
-            yarn build --since=${{ env.NX_BASE_BRANCH }}
-          else
-            yarn build
-          fi
+          yarn build
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |
-          if ${{ env.USE_NX_AFFECTED }}; then
-            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
-          else
-            yarn check:types
-          fi
+          yarn check:types
 
   test-libraries:
     runs-on: ubuntu-latest

From 393bdecd0dc963947dadc301a126f1a773acbdf2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 10 Aug 2023 17:34:51 +0300
Subject: [PATCH 85/88] Check:types using affected

---
 .github/workflows/budibase_ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index b754587d34..6cdfba068b 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -73,7 +73,11 @@ jobs:
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |
-          yarn check:types
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn check:types
+          fi
 
   test-libraries:
     runs-on: ubuntu-latest

From b0191a58470af5c4418df5614912d8ac7a2105a6 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 14:51:01 +0000
Subject: [PATCH 86/88] Bump version to 2.9.24-alpha.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index b03609303e..7dc6b51562 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.24-alpha.0",
+  "version": "2.9.24-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 3f26bb2aae737c233953d2a752f6c1a5fc58b2d4 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 10 Aug 2023 16:09:14 +0000
Subject: [PATCH 87/88] Bump version to 2.9.24-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 7dc6b51562..721efe58a4 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.24-alpha.1",
+  "version": "2.9.24-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From a444d6a7328447d9ef18ba49390c9a9a324ebfeb Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 11 Aug 2023 06:52:17 +0000
Subject: [PATCH 88/88] Bump version to 2.9.24-alpha.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 721efe58a4..8a0649b8bb 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.9.24-alpha.2",
+  "version": "2.9.24-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"