From d03fdb6df94f6d17ad0c1a79ed38f1196e70f535 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 3 Oct 2023 17:35:00 +0100
Subject: [PATCH 01/95] Add initial support for query datasources in grids

---
 packages/client/manifest.json                 |   5 +-
 .../src/components/app/GridBlock.svelte       |   7 +-
 .../src/components/grid/cells/DataCell.svelte |   2 +-
 .../src/components/grid/layout/NewRow.svelte  |   6 +-
 .../src/components/grid/stores/config.js      |  11 +-
 .../src/components/grid/stores/datasource.js  |  25 +++-
 .../src/components/grid/stores/index.js       |   2 +
 .../src/components/grid/stores/query.js       | 108 ++++++++++++++++++
 .../src/components/grid/stores/rows.js        |  10 +-
 .../src/components/grid/stores/table.js       |   7 +-
 .../src/components/grid/stores/viewV2.js      |  16 +--
 .../src/fetch/{fetchData.js => index.js}      |  11 ++
 packages/frontend-core/src/index.js           |   2 +-
 13 files changed, 178 insertions(+), 34 deletions(-)
 create mode 100644 packages/frontend-core/src/components/grid/stores/query.js
 rename packages/frontend-core/src/fetch/{fetchData.js => index.js} (72%)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 4e56ca758d..b1391e36fd 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5545,12 +5545,11 @@
       "width": 600,
       "height": 400
     },
-    "info": "Grid Blocks are only compatible with internal or SQL tables",
     "settings": [
       {
-        "type": "table",
+        "type": "dataSource",
         "label": "Data",
-        "key": "table",
+        "key": "datasource",
         "required": true
       },
       {
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 9bdea52124..1cdbcf00ff 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -4,7 +4,7 @@
   import { getContext } from "svelte"
   import { Grid } from "@budibase/frontend-core"
 
-  export let table
+  export let datasource
   export let allowAddRows = true
   export let allowEditRows = true
   export let allowDeleteRows = true
@@ -15,6 +15,9 @@
   export let fixedRowHeight = null
   export let columns = null
 
+  // Legacy settings
+  export let table
+
   const component = getContext("component")
   const { styleable, API, builderStore, notificationStore } = getContext("sdk")
 
@@ -38,7 +41,7 @@
   class:in-builder={$builderStore.inBuilder}
 >
   <Grid
-    datasource={table}
+    datasource={datasource || table}
     {API}
     {stripeRows}
     {initialFilter}
diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
index 19bdf1bd74..f9cdef3756 100644
--- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
@@ -34,7 +34,7 @@
     column.schema.autocolumn ||
     column.schema.disabled ||
     column.schema.type === "formula" ||
-    (!$config.canEditRows && row._id)
+    (!$config.canEditRows && !row._isNewRow)
 
   // Register this cell API if the row is focused
   $: {
diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte
index 6a926ca02c..3f5b9617bd 100644
--- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte
+++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte
@@ -33,7 +33,7 @@
 
   let visible = false
   let isAdding = false
-  let newRow = {}
+  let newRow = { _isNewRow: true }
   let offset = 0
 
   $: firstColumn = $stickyColumn || $renderedColumns[0]
@@ -58,7 +58,9 @@
 
     // Create row
     const newRowIndex = offset ? undefined : 0
-    const savedRow = await rows.actions.addRow(newRow, newRowIndex)
+    let rowToCreate = { ...newRow }
+    delete rowToCreate._isNewRow
+    const savedRow = await rows.actions.addRow(rowToCreate, newRowIndex)
     if (savedRow) {
       // Reset state
       clear()
diff --git a/packages/frontend-core/src/components/grid/stores/config.js b/packages/frontend-core/src/components/grid/stores/config.js
index ae995b4ac7..0efdc2104e 100644
--- a/packages/frontend-core/src/components/grid/stores/config.js
+++ b/packages/frontend-core/src/components/grid/stores/config.js
@@ -37,9 +37,10 @@ export const deriveStores = context => {
     [props, hasNonAutoColumn],
     ([$props, $hasNonAutoColumn]) => {
       let config = { ...$props }
+      const type = $props.datasource?.type
 
       // Disable some features if we're editing a view
-      if ($props.datasource?.type === "viewV2") {
+      if (type === "viewV2") {
         config.canEditColumns = false
       }
 
@@ -48,6 +49,14 @@ export const deriveStores = context => {
         config.canAddRows = false
       }
 
+      // Disable features for non DS+
+      if (!["table", "viewV2"].includes(type)) {
+        config.canAddRows = false
+        config.canEditRows = false
+        config.canDeleteRows = false
+        config.canExpandRows = false
+      }
+
       return config
     }
   )
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 3f4347953e..e56b37a5f3 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -1,4 +1,5 @@
 import { derived, get, writable } from "svelte/store"
+import { getDatasourceDefinition } from "../../../fetch"
 
 export const createStores = () => {
   const definition = writable(null)
@@ -19,6 +20,14 @@ export const deriveStores = context => {
       }
       let newSchema = { ...$definition?.schema }
 
+      // Ensure schema is configured as objects.
+      // Certain datasources like queries use primitives.
+      Object.keys(newSchema).forEach(key => {
+        if (typeof newSchema[key] !== "object") {
+          newSchema[key] = { type: newSchema[key] }
+        }
+      })
+
       // Apply schema overrides
       Object.keys($schemaOverrides || {}).forEach(field => {
         if (newSchema[field]) {
@@ -48,7 +57,16 @@ export const deriveStores = context => {
 }
 
 export const createActions = context => {
-  const { datasource, definition, config, dispatch, table, viewV2 } = context
+  const {
+    API,
+    datasource,
+    definition,
+    config,
+    dispatch,
+    table,
+    viewV2,
+    query,
+  } = context
 
   // Gets the appropriate API for the configured datasource type
   const getAPI = () => {
@@ -58,6 +76,8 @@ export const createActions = context => {
         return table
       case "viewV2":
         return viewV2
+      case "query":
+        return query
       default:
         return null
     }
@@ -65,7 +85,8 @@ export const createActions = context => {
 
   // Refreshes the datasource definition
   const refreshDefinition = async () => {
-    return await getAPI()?.actions.refreshDefinition()
+    const def = await getDatasourceDefinition({ API, datasource })
+    definition.set(def)
   }
 
   // Saves the datasource definition
diff --git a/packages/frontend-core/src/components/grid/stores/index.js b/packages/frontend-core/src/components/grid/stores/index.js
index 7b73ea8be6..5a46227b49 100644
--- a/packages/frontend-core/src/components/grid/stores/index.js
+++ b/packages/frontend-core/src/components/grid/stores/index.js
@@ -17,6 +17,7 @@ import * as Filter from "./filter"
 import * as Notifications from "./notifications"
 import * as Table from "./table"
 import * as ViewV2 from "./viewV2"
+import * as Query from "./query"
 import * as Datasource from "./datasource"
 
 const DependencyOrderedStores = [
@@ -26,6 +27,7 @@ const DependencyOrderedStores = [
   Scroll,
   Table,
   ViewV2,
+  Query,
   Datasource,
   Columns,
   Rows,
diff --git a/packages/frontend-core/src/components/grid/stores/query.js b/packages/frontend-core/src/components/grid/stores/query.js
new file mode 100644
index 0000000000..6ac0547d27
--- /dev/null
+++ b/packages/frontend-core/src/components/grid/stores/query.js
@@ -0,0 +1,108 @@
+import { get } from "svelte/store"
+
+export const createActions = context => {
+  const { API, columns, stickyColumn } = context
+
+  const saveDefinition = async newDefinition => {
+    await API.saveQuery(newDefinition)
+  }
+
+  const saveRow = async () => {
+    throw "Rows cannot be updated through queries"
+  }
+
+  const deleteRows = async () => {
+    throw "Rows cannot be deleted through queries"
+  }
+
+  const getRow = () => {
+    throw "Queries don't support fetching individual rows"
+  }
+
+  const isDatasourceValid = datasource => {
+    return datasource?.type === "query" && datasource?._id
+  }
+
+  const canUseColumn = name => {
+    const $columns = get(columns)
+    const $sticky = get(stickyColumn)
+    return $columns.some(col => col.name === name) || $sticky?.name === name
+  }
+
+  return {
+    query: {
+      actions: {
+        saveDefinition,
+        addRow: saveRow,
+        updateRow: saveRow,
+        deleteRows,
+        getRow,
+        isDatasourceValid,
+        canUseColumn,
+      },
+    },
+  }
+}
+
+export const initialise = context => {
+  const {
+    datasource,
+    sort,
+    filter,
+    query,
+    initialFilter,
+    initialSortColumn,
+    initialSortOrder,
+    fetch,
+  } = context
+
+  // Keep a list of subscriptions so that we can clear them when the datasource
+  // config changes
+  let unsubscribers = []
+
+  // Observe datasource changes and apply logic for view V2 datasources
+  datasource.subscribe($datasource => {
+    // Clear previous subscriptions
+    unsubscribers?.forEach(unsubscribe => unsubscribe())
+    unsubscribers = []
+    if (!query.actions.isDatasourceValid($datasource)) {
+      return
+    }
+
+    // Wipe state
+    filter.set(get(initialFilter))
+    sort.set({
+      column: get(initialSortColumn),
+      order: get(initialSortOrder) || "ascending",
+    })
+
+    // Update fetch when filter changes
+    unsubscribers.push(
+      filter.subscribe($filter => {
+        // Ensure we're updating the correct fetch
+        const $fetch = get(fetch)
+        if ($fetch?.options?.datasource?._id !== $datasource._id) {
+          return
+        }
+        $fetch.update({
+          filter: $filter,
+        })
+      })
+    )
+
+    // Update fetch when sorting changes
+    unsubscribers.push(
+      sort.subscribe($sort => {
+        // Ensure we're updating the correct fetch
+        const $fetch = get(fetch)
+        if ($fetch?.options?.datasource?._id !== $datasource._id) {
+          return
+        }
+        $fetch.update({
+          sortOrder: $sort.order || "ascending",
+          sortColumn: $sort.column,
+        })
+      })
+    )
+  })
+}
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 392bf392e8..2e2832f209 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -1,7 +1,8 @@
 import { writable, derived, get } from "svelte/store"
-import { fetchData } from "../../../fetch/fetchData"
+import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import { tick } from "svelte"
+import { Helpers } from "@budibase/bbui"
 
 export const createStores = () => {
   const rows = writable([])
@@ -413,6 +414,13 @@ export const createActions = context => {
     let newRow
     for (let i = 0; i < newRows.length; i++) {
       newRow = newRows[i]
+
+      // Ensure we have a unique _id.
+      // This means generating one for non DS+.
+      if (!newRow._id) {
+        newRow._id = Helpers.hashString(JSON.stringify(newRow))
+      }
+
       if (!rowCacheMap[newRow._id]) {
         rowCacheMap[newRow._id] = true
         rowsToAppend.push(newRow)
diff --git a/packages/frontend-core/src/components/grid/stores/table.js b/packages/frontend-core/src/components/grid/stores/table.js
index ed13609f45..9a4058ac8f 100644
--- a/packages/frontend-core/src/components/grid/stores/table.js
+++ b/packages/frontend-core/src/components/grid/stores/table.js
@@ -3,11 +3,7 @@ import { get } from "svelte/store"
 const SuppressErrors = true
 
 export const createActions = context => {
-  const { definition, API, datasource, columns, stickyColumn } = context
-
-  const refreshDefinition = async () => {
-    definition.set(await API.fetchTableDefinition(get(datasource).tableId))
-  }
+  const { API, datasource, columns, stickyColumn } = context
 
   const saveDefinition = async newDefinition => {
     await API.saveTable(newDefinition)
@@ -52,7 +48,6 @@ export const createActions = context => {
   return {
     table: {
       actions: {
-        refreshDefinition,
         saveDefinition,
         addRow: saveRow,
         updateRow: saveRow,
diff --git a/packages/frontend-core/src/components/grid/stores/viewV2.js b/packages/frontend-core/src/components/grid/stores/viewV2.js
index b9a4bc099b..000727b262 100644
--- a/packages/frontend-core/src/components/grid/stores/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/viewV2.js
@@ -3,20 +3,7 @@ import { get } from "svelte/store"
 const SuppressErrors = true
 
 export const createActions = context => {
-  const { definition, API, datasource, columns, stickyColumn } = context
-
-  const refreshDefinition = async () => {
-    const $datasource = get(datasource)
-    if (!$datasource) {
-      definition.set(null)
-      return
-    }
-    const table = await API.fetchTableDefinition($datasource.tableId)
-    const view = Object.values(table?.views || {}).find(
-      view => view.id === $datasource.id
-    )
-    definition.set(view)
-  }
+  const { API, datasource, columns, stickyColumn } = context
 
   const saveDefinition = async newDefinition => {
     await API.viewV2.update(newDefinition)
@@ -61,7 +48,6 @@ export const createActions = context => {
   return {
     viewV2: {
       actions: {
-        refreshDefinition,
         saveDefinition,
         addRow: saveRow,
         updateRow: saveRow,
diff --git a/packages/frontend-core/src/fetch/fetchData.js b/packages/frontend-core/src/fetch/index.js
similarity index 72%
rename from packages/frontend-core/src/fetch/fetchData.js
rename to packages/frontend-core/src/fetch/index.js
index 063dd02cbf..0edd07762b 100644
--- a/packages/frontend-core/src/fetch/fetchData.js
+++ b/packages/frontend-core/src/fetch/index.js
@@ -24,7 +24,18 @@ const DataFetchMap = {
   jsonarray: JSONArrayFetch,
 }
 
+// Constructs a new fetch model for a certain datasource
 export const fetchData = ({ API, datasource, options }) => {
   const Fetch = DataFetchMap[datasource?.type] || TableFetch
   return new Fetch({ API, datasource, ...options })
 }
+
+// Fetches the definition of any type of datasource
+export const getDatasourceDefinition = async ({ API, datasource }) => {
+  const handler = DataFetchMap[datasource?.type]
+  if (!handler) {
+    return null
+  }
+  const instance = new handler({ API })
+  return await instance.getDefinition(datasource)
+}
diff --git a/packages/frontend-core/src/index.js b/packages/frontend-core/src/index.js
index b0afc0c25d..f51be616f8 100644
--- a/packages/frontend-core/src/index.js
+++ b/packages/frontend-core/src/index.js
@@ -1,5 +1,5 @@
 export { createAPIClient } from "./api"
-export { fetchData } from "./fetch/fetchData"
+export { fetchData } from "./fetch"
 export { Utils } from "./utils"
 export * as Constants from "./constants"
 export * from "./stores"

From c4a516ccb37e838038011ba39d198e65a1ca2c79 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Oct 2023 09:25:35 +0100
Subject: [PATCH 02/95] Ensure grid schema structure is predictable and prevent
 copying IDs for query datasources

---
 .../components/grid/cells/HeaderCell.svelte   |  7 +--
 .../grid/overlays/MenuOverlay.svelte          |  4 +-
 .../src/components/grid/stores/columns.js     | 33 +++++++------
 .../src/components/grid/stores/datasource.js  | 48 +++++++++++--------
 .../src/components/grid/stores/rows.js        |  2 +-
 .../src/components/grid/stores/sort.js        | 15 +++---
 6 files changed, 61 insertions(+), 48 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index a58b4c8fe4..5ac70c93c8 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -22,6 +22,7 @@
     columns,
     definition,
     datasource,
+    schema,
   } = getContext("grid")
 
   const bannedDisplayColumnTypes = [
@@ -126,16 +127,16 @@
     // Generate new name
     let newName = `${column.name} copy`
     let attempts = 2
-    while ($definition.schema[newName]) {
+    while ($schema[newName]) {
       newName = `${column.name} copy ${attempts++}`
     }
 
     // Save schema with new column
-    const existingColumnDefinition = $definition.schema[column.name]
+    const existingColumnDefinition = $schema[column.name]
     await datasource.actions.saveDefinition({
       ...$definition,
       schema: {
-        ...$definition.schema,
+        ...$schema,
         [newName]: {
           ...existingColumnDefinition,
           name: newName,
diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
index cbf2c6ee4e..7fb2bb138d 100644
--- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
@@ -75,7 +75,9 @@
       </MenuItem>
       <MenuItem
         icon="Copy"
-        disabled={isNewRow || !$focusedRow?._id}
+        disabled={isNewRow ||
+          !$focusedRow?._id ||
+          $focusedRow?._id?.startsWith("fake-")}
         on:click={() => copyToClipboard($focusedRow?._id)}
         on:click={menu.actions.close}
       >
diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 629d5dd893..c36afea7a1 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -69,7 +69,7 @@ export const deriveStores = context => {
 }
 
 export const createActions = context => {
-  const { columns, stickyColumn, datasource, definition } = context
+  const { columns, stickyColumn, datasource, definition, schema } = context
 
   // Updates the datasources primary display column
   const changePrimaryDisplay = async column => {
@@ -101,7 +101,7 @@ export const createActions = context => {
     const $columns = get(columns)
     const $definition = get(definition)
     const $stickyColumn = get(stickyColumn)
-    const newSchema = cloneDeep($definition.schema)
+    let newSchema = cloneDeep(get(schema)) || {}
 
     // Build new updated datasource schema
     Object.keys(newSchema).forEach(column => {
@@ -142,11 +142,11 @@ export const createActions = context => {
 }
 
 export const initialise = context => {
-  const { definition, columns, stickyColumn, schema } = context
+  const { definition, columns, stickyColumn, enrichedSchema } = context
 
   // Merge new schema fields with existing schema in order to preserve widths
-  schema.subscribe($schema => {
-    if (!$schema) {
+  enrichedSchema.subscribe($enrichedSchema => {
+    if (!$enrichedSchema) {
       columns.set([])
       stickyColumn.set(null)
       return
@@ -155,13 +155,16 @@ export const initialise = context => {
 
     // Find primary display
     let primaryDisplay
-    if ($definition.primaryDisplay && $schema[$definition.primaryDisplay]) {
+    if (
+      $definition.primaryDisplay &&
+      $enrichedSchema[$definition.primaryDisplay]
+    ) {
       primaryDisplay = $definition.primaryDisplay
     }
 
     // Get field list
     let fields = []
-    Object.keys($schema).forEach(field => {
+    Object.keys($enrichedSchema).forEach(field => {
       if (field !== primaryDisplay) {
         fields.push(field)
       }
@@ -172,11 +175,11 @@ export const initialise = context => {
       fields
         .map(field => ({
           name: field,
-          label: $schema[field].displayName || field,
-          schema: $schema[field],
-          width: $schema[field].width || DefaultColumnWidth,
-          visible: $schema[field].visible ?? true,
-          order: $schema[field].order,
+          label: $enrichedSchema[field].displayName || field,
+          schema: $enrichedSchema[field],
+          width: $enrichedSchema[field].width || DefaultColumnWidth,
+          visible: $enrichedSchema[field].visible ?? true,
+          order: $enrichedSchema[field].order,
         }))
         .sort((a, b) => {
           // Sort by order first
@@ -207,9 +210,9 @@ export const initialise = context => {
     }
     stickyColumn.set({
       name: primaryDisplay,
-      label: $schema[primaryDisplay].displayName || primaryDisplay,
-      schema: $schema[primaryDisplay],
-      width: $schema[primaryDisplay].width || DefaultColumnWidth,
+      label: $enrichedSchema[primaryDisplay].displayName || primaryDisplay,
+      schema: $enrichedSchema[primaryDisplay],
+      width: $enrichedSchema[primaryDisplay].width || DefaultColumnWidth,
       visible: true,
       order: 0,
       left: GutterWidth,
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index e56b37a5f3..0d70424550 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -12,27 +12,36 @@ export const createStores = () => {
 export const deriveStores = context => {
   const { definition, schemaOverrides, columnWhitelist } = context
 
-  const schema = derived(
-    [definition, schemaOverrides, columnWhitelist],
-    ([$definition, $schemaOverrides, $columnWhitelist]) => {
-      if (!$definition?.schema) {
+  const schema = derived(definition, $definition => {
+    let schema = $definition?.schema
+    if (!schema) {
+      return null
+    }
+
+    // Ensure schema is configured as objects.
+    // Certain datasources like queries use primitives.
+    Object.keys(schema || {}).forEach(key => {
+      if (typeof schema[key] !== "object") {
+        schema[key] = { type: schema[key] }
+      }
+    })
+
+    return schema
+  })
+
+  const enrichedSchema = derived(
+    [schema, schemaOverrides, columnWhitelist],
+    ([$schema, $schemaOverrides, $columnWhitelist]) => {
+      if (!$schema) {
         return null
       }
-      let newSchema = { ...$definition?.schema }
-
-      // Ensure schema is configured as objects.
-      // Certain datasources like queries use primitives.
-      Object.keys(newSchema).forEach(key => {
-        if (typeof newSchema[key] !== "object") {
-          newSchema[key] = { type: newSchema[key] }
-        }
-      })
+      let enrichedSchema = { ...$schema }
 
       // Apply schema overrides
       Object.keys($schemaOverrides || {}).forEach(field => {
-        if (newSchema[field]) {
-          newSchema[field] = {
-            ...newSchema[field],
+        if (enrichedSchema[field]) {
+          enrichedSchema[field] = {
+            ...enrichedSchema[field],
             ...$schemaOverrides[field],
           }
         }
@@ -40,19 +49,20 @@ export const deriveStores = context => {
 
       // Apply whitelist if specified
       if ($columnWhitelist?.length) {
-        Object.keys(newSchema).forEach(key => {
+        Object.keys(enrichedSchema).forEach(key => {
           if (!$columnWhitelist.includes(key)) {
-            delete newSchema[key]
+            delete enrichedSchema[key]
           }
         })
       }
 
-      return newSchema
+      return enrichedSchema
     }
   )
 
   return {
     schema,
+    enrichedSchema,
   }
 }
 
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 2e2832f209..1eafb20756 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -418,7 +418,7 @@ export const createActions = context => {
       // Ensure we have a unique _id.
       // This means generating one for non DS+.
       if (!newRow._id) {
-        newRow._id = Helpers.hashString(JSON.stringify(newRow))
+        newRow._id = `fake-${Helpers.hashString(JSON.stringify(newRow))}`
       }
 
       if (!rowCacheMap[newRow._id]) {
diff --git a/packages/frontend-core/src/components/grid/stores/sort.js b/packages/frontend-core/src/components/grid/stores/sort.js
index 734a876eed..336570d012 100644
--- a/packages/frontend-core/src/components/grid/stores/sort.js
+++ b/packages/frontend-core/src/components/grid/stores/sort.js
@@ -17,7 +17,7 @@ export const createStores = context => {
 }
 
 export const initialise = context => {
-  const { sort, initialSortColumn, initialSortOrder, definition } = context
+  const { sort, initialSortColumn, initialSortOrder, schema } = context
 
   // Reset sort when initial sort props change
   initialSortColumn.subscribe(newSortColumn => {
@@ -28,15 +28,12 @@ export const initialise = context => {
   })
 
   // Derive if the current sort column exists in the schema
-  const sortColumnExists = derived(
-    [sort, definition],
-    ([$sort, $definition]) => {
-      if (!$sort?.column || !$definition) {
-        return true
-      }
-      return $definition.schema?.[$sort.column] != null
+  const sortColumnExists = derived([sort, schema], ([$sort, $schema]) => {
+    if (!$sort?.column || !$schema) {
+      return true
     }
-  )
+    return $schema[$sort.column] != null
+  })
 
   // Clear sort state if our sort column does not exist
   sortColumnExists.subscribe(exists => {

From 77f87af87f3f169b0b90b9fa39b4c102991bb956 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 4 Oct 2023 09:36:42 +0100
Subject: [PATCH 03/95] Improve grid handling of invalid datasources and fix
 potential error when encoutering invalid datasources

---
 .../src/components/grid/layout/Grid.svelte       | 16 ++++++++--------
 .../src/components/grid/stores/rows.js           |  2 ++
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index 44e0a972f1..e2ecd0f968 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -141,7 +141,14 @@
       </div>
     </div>
   {/if}
-  {#if $loaded}
+  {#if $error}
+    <div class="grid-error">
+      <div class="grid-error-title">There was a problem loading your grid</div>
+      <div class="grid-error-subtitle">
+        {$error}
+      </div>
+    </div>
+  {:else if $loaded}
     <div class="grid-data-outer" use:clickOutside={ui.actions.blur}>
       <div class="grid-data-inner">
         <StickyColumn>
@@ -171,13 +178,6 @@
         </div>
       </div>
     </div>
-  {:else if $error}
-    <div class="grid-error">
-      <div class="grid-error-title">There was a problem loading your grid</div>
-      <div class="grid-error-subtitle">
-        {$error}
-      </div>
-    </div>
   {/if}
   {#if $loading && !$error}
     <div in:fade|local={{ duration: 130 }} class="grid-loading">
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 1eafb20756..e6251a5afa 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -94,12 +94,14 @@ export const createActions = context => {
   datasource.subscribe(async $datasource => {
     // Unsub from previous fetch if one exists
     unsubscribe?.()
+    unsubscribe = null
     fetch.set(null)
     instanceLoaded.set(false)
     loading.set(true)
 
     // Abandon if we don't have a valid datasource
     if (!datasource.actions.isDatasourceValid($datasource)) {
+      error.set("Datasource is invalid")
       return
     }
 

From 88c4d0cd203c0b1f8788b2898810c381c5c1af1f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 08:23:01 +0100
Subject: [PATCH 04/95] Update grids to work with all datasources

---
 .../src/components/grid/stores/config.js      |  2 +
 .../src/components/grid/stores/datasource.js  | 12 +++---
 .../{query.js => datasources/nonPlus.js}      | 38 ++++++++++++-------
 .../grid/stores/{ => datasources}/table.js    |  0
 .../grid/stores/{ => datasources}/viewV2.js   |  0
 .../src/components/grid/stores/index.js       |  8 ++--
 6 files changed, 38 insertions(+), 22 deletions(-)
 rename packages/frontend-core/src/components/grid/stores/{query.js => datasources/nonPlus.js} (64%)
 rename packages/frontend-core/src/components/grid/stores/{ => datasources}/table.js (100%)
 rename packages/frontend-core/src/components/grid/stores/{ => datasources}/viewV2.js (100%)

diff --git a/packages/frontend-core/src/components/grid/stores/config.js b/packages/frontend-core/src/components/grid/stores/config.js
index 0efdc2104e..6da6ebf11e 100644
--- a/packages/frontend-core/src/components/grid/stores/config.js
+++ b/packages/frontend-core/src/components/grid/stores/config.js
@@ -55,6 +55,8 @@ export const deriveStores = context => {
         config.canEditRows = false
         config.canDeleteRows = false
         config.canExpandRows = false
+        config.canSaveSchema = false
+        config.canEditColumns = false
       }
 
       return config
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 0d70424550..6a10cb8b9b 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -75,21 +75,23 @@ export const createActions = context => {
     dispatch,
     table,
     viewV2,
-    query,
+    nonPlus,
   } = context
 
   // Gets the appropriate API for the configured datasource type
   const getAPI = () => {
     const $datasource = get(datasource)
-    switch ($datasource?.type) {
+    const type = $datasource?.type
+    if (!type) {
+      return null
+    }
+    switch (type) {
       case "table":
         return table
       case "viewV2":
         return viewV2
-      case "query":
-        return query
       default:
-        return null
+        return nonPlus
     }
   }
 
diff --git a/packages/frontend-core/src/components/grid/stores/query.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
similarity index 64%
rename from packages/frontend-core/src/components/grid/stores/query.js
rename to packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
index 6ac0547d27..ea024643aa 100644
--- a/packages/frontend-core/src/components/grid/stores/query.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
@@ -1,26 +1,32 @@
 import { get } from "svelte/store"
 
 export const createActions = context => {
-  const { API, columns, stickyColumn } = context
+  const { columns, stickyColumn, table, viewV2 } = context
 
-  const saveDefinition = async newDefinition => {
-    await API.saveQuery(newDefinition)
+  const saveDefinition = async () => {
+    throw "This datasource does not support updating the definition"
   }
 
   const saveRow = async () => {
-    throw "Rows cannot be updated through queries"
+    throw "This datasource does not support saving rows"
   }
 
   const deleteRows = async () => {
-    throw "Rows cannot be deleted through queries"
+    throw "This datasource does not support deleting rows"
   }
 
   const getRow = () => {
-    throw "Queries don't support fetching individual rows"
+    throw "This datasource does not support fetching individual rows"
   }
 
   const isDatasourceValid = datasource => {
-    return datasource?.type === "query" && datasource?._id
+    // There are many different types and shapes of datasource, so we only
+    // check that we aren't null
+    return (
+      !table.actions.isDatasourceValid(datasource) &&
+      !viewV2.actions.isDatasourceValid(datasource) &&
+      datasource?.type != null
+    )
   }
 
   const canUseColumn = name => {
@@ -30,7 +36,7 @@ export const createActions = context => {
   }
 
   return {
-    query: {
+    nonPlus: {
       actions: {
         saveDefinition,
         addRow: saveRow,
@@ -44,18 +50,22 @@ export const createActions = context => {
   }
 }
 
+// Small util to compare datasource definitions
+const isSameDatasource = (a, b) => {
+  return JSON.stringify(a) === JSON.stringify(b)
+}
+
 export const initialise = context => {
   const {
     datasource,
     sort,
     filter,
-    query,
+    nonPlus,
     initialFilter,
     initialSortColumn,
     initialSortOrder,
     fetch,
   } = context
-
   // Keep a list of subscriptions so that we can clear them when the datasource
   // config changes
   let unsubscribers = []
@@ -65,7 +75,7 @@ export const initialise = context => {
     // Clear previous subscriptions
     unsubscribers?.forEach(unsubscribe => unsubscribe())
     unsubscribers = []
-    if (!query.actions.isDatasourceValid($datasource)) {
+    if (!nonPlus.actions.isDatasourceValid($datasource)) {
       return
     }
 
@@ -81,7 +91,8 @@ export const initialise = context => {
       filter.subscribe($filter => {
         // Ensure we're updating the correct fetch
         const $fetch = get(fetch)
-        if ($fetch?.options?.datasource?._id !== $datasource._id) {
+        if (!isSameDatasource($fetch?.options?.datasource, $datasource)) {
+          console.log("skip, different ds")
           return
         }
         $fetch.update({
@@ -95,7 +106,8 @@ export const initialise = context => {
       sort.subscribe($sort => {
         // Ensure we're updating the correct fetch
         const $fetch = get(fetch)
-        if ($fetch?.options?.datasource?._id !== $datasource._id) {
+        if (!isSameDatasource($fetch?.options?.datasource, $datasource)) {
+          console.log("skip, different ds")
           return
         }
         $fetch.update({
diff --git a/packages/frontend-core/src/components/grid/stores/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js
similarity index 100%
rename from packages/frontend-core/src/components/grid/stores/table.js
rename to packages/frontend-core/src/components/grid/stores/datasources/table.js
diff --git a/packages/frontend-core/src/components/grid/stores/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
similarity index 100%
rename from packages/frontend-core/src/components/grid/stores/viewV2.js
rename to packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
diff --git a/packages/frontend-core/src/components/grid/stores/index.js b/packages/frontend-core/src/components/grid/stores/index.js
index 5a46227b49..10fe932aab 100644
--- a/packages/frontend-core/src/components/grid/stores/index.js
+++ b/packages/frontend-core/src/components/grid/stores/index.js
@@ -15,10 +15,10 @@ import * as Config from "./config"
 import * as Sort from "./sort"
 import * as Filter from "./filter"
 import * as Notifications from "./notifications"
-import * as Table from "./table"
-import * as ViewV2 from "./viewV2"
-import * as Query from "./query"
 import * as Datasource from "./datasource"
+import * as Table from "./datasources/table"
+import * as ViewV2 from "./datasources/viewV2"
+import * as NonPlus from "./datasources/nonPlus"
 
 const DependencyOrderedStores = [
   Sort,
@@ -27,7 +27,7 @@ const DependencyOrderedStores = [
   Scroll,
   Table,
   ViewV2,
-  Query,
+  NonPlus,
   Datasource,
   Columns,
   Rows,

From b75c78dae54f74df60341615580449851297fe87 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 08:24:30 +0100
Subject: [PATCH 05/95] Remove log

---
 .../src/components/grid/stores/datasources/nonPlus.js           | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
index ea024643aa..e04d05ec59 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
@@ -92,7 +92,6 @@ export const initialise = context => {
         // Ensure we're updating the correct fetch
         const $fetch = get(fetch)
         if (!isSameDatasource($fetch?.options?.datasource, $datasource)) {
-          console.log("skip, different ds")
           return
         }
         $fetch.update({
@@ -107,7 +106,6 @@ export const initialise = context => {
         // Ensure we're updating the correct fetch
         const $fetch = get(fetch)
         if (!isSameDatasource($fetch?.options?.datasource, $datasource)) {
-          console.log("skip, different ds")
           return
         }
         $fetch.update({

From 43c30d877beb3b1e0b21b3da617d7ad205534206 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 14:42:26 +0100
Subject: [PATCH 06/95] Add new custom datasource type

---
 .../settings/controls/DataSourceSelect.svelte | 81 ++++++++++++----
 .../src/components/grid/lib/utils.js          |  2 +-
 .../src/components/grid/stores/rows.js        |  2 +-
 .../frontend-core/src/fetch/CustomFetch.js    | 97 +++++++++++++++++++
 packages/frontend-core/src/fetch/index.js     |  2 +
 5 files changed, 162 insertions(+), 22 deletions(-)
 create mode 100644 packages/frontend-core/src/fetch/CustomFetch.js

diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
index d70929469a..e4b3f972ba 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
@@ -9,6 +9,7 @@
     Heading,
     Drawer,
     DrawerContent,
+    Icon,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { store, currentAsset } from "builderStore"
@@ -22,6 +23,7 @@
   import BindingBuilder from "components/integration/QueryBindingBuilder.svelte"
   import IntegrationQueryEditor from "components/integration/index.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
 
   export let value = {}
   export let otherSources
@@ -31,9 +33,12 @@
 
   const dispatch = createEventDispatcher()
   const arrayTypes = ["attachment", "array"]
+
   let anchorRight, dropdownRight
   let drawer
   let tmpQueryParams
+  let tmpCustomData
+  let customDataValid = true
 
   $: text = value?.label ?? "Choose an option"
   $: tables = $tablesStore.list.map(m => ({
@@ -125,6 +130,10 @@
         value: `{{ literal ${runtimeBinding} }}`,
       }
     })
+  $: custom = {
+    type: "custom",
+    label: "Custom",
+  }
 
   const handleSelected = selected => {
     dispatch("change", selected)
@@ -151,6 +160,11 @@
     drawer.show()
   }
 
+  const openCustomDrawer = () => {
+    tmpCustomData = value.data || ""
+    drawer.show()
+  }
+
   const getQueryValue = queries => {
     return queries.find(q => q._id === value._id) || value
   }
@@ -162,6 +176,14 @@
     })
     drawer.hide()
   }
+
+  const saveCustomData = () => {
+    handleSelected({
+      ...value,
+      data: tmpCustomData,
+    })
+    drawer.hide()
+  }
 </script>
 
 <div class="container" bind:this={anchorRight}>
@@ -172,7 +194,9 @@
     on:click={dropdownRight.show}
   />
   {#if value?.type === "query"}
-    <i class="ri-settings-5-line" on:click={openQueryParamsDrawer} />
+    <div class="icon">
+      <Icon hoverable name="Settings" on:click={openQueryParamsDrawer} />
+    </div>
     <Drawer title={"Query Bindings"} bind:this={drawer}>
       <Button slot="buttons" cta on:click={saveQueryParams}>Save</Button>
       <DrawerContent slot="body">
@@ -198,6 +222,31 @@
       </DrawerContent>
     </Drawer>
   {/if}
+  {#if value?.type === "custom"}
+    <div class="icon">
+      <Icon hoverable name="Settings" on:click={openCustomDrawer} />
+    </div>
+    <Drawer title="Custom data" bind:this={drawer}>
+      <Button
+        slot="buttons"
+        cta
+        on:click={saveCustomData}
+        disabled={!customDataValid}>Save</Button
+      >
+      <div slot="description">
+        Provide a JavaScript or JSON array to use as data
+      </div>
+      <ClientBindingPanel
+        slot="body"
+        bind:valid={customDataValid}
+        value={tmpCustomData}
+        on:change={event => (tmpCustomData = event.detail)}
+        {bindings}
+        allowJS
+        allowHelpers
+      />
+    </Drawer>
+  {/if}
 </div>
 <Popover bind:this={dropdownRight} anchor={anchorRight}>
   <div class="dropdown">
@@ -285,17 +334,18 @@
         {/each}
       </ul>
     {/if}
-    {#if otherSources?.length}
-      <Divider />
-      <div class="title">
-        <Heading size="XS">Other</Heading>
-      </div>
-      <ul>
+    <Divider />
+    <div class="title">
+      <Heading size="XS">Other</Heading>
+    </div>
+    <ul>
+      <li on:click={() => handleSelected(custom)}>{custom.label}</li>
+      {#if otherSources?.length}
         {#each otherSources as source}
           <li on:click={() => handleSelected(source)}>{source.label}</li>
         {/each}
-      </ul>
-    {/if}
+      {/if}
+    </ul>
   </div>
 </Popover>
 
@@ -340,16 +390,7 @@
     background-color: var(--spectrum-global-color-gray-200);
   }
 
-  i {
-    margin-left: 5px;
-    display: flex;
-    align-items: center;
-    transition: all 0.2s;
-  }
-
-  i:hover {
-    transform: scale(1.1);
-    font-weight: 600;
-    cursor: pointer;
+  .icon {
+    margin-left: 8px;
   }
 </style>
diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js
index 9383f69f66..d3898a3b18 100644
--- a/packages/frontend-core/src/components/grid/lib/utils.js
+++ b/packages/frontend-core/src/components/grid/lib/utils.js
@@ -1,6 +1,6 @@
 export const getColor = (idx, opacity = 0.3) => {
   if (idx == null || idx === -1) {
-    return null
+    idx = 0
   }
   return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, ${opacity})`
 }
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index e6251a5afa..5b6f4fd7f9 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -420,7 +420,7 @@ export const createActions = context => {
       // Ensure we have a unique _id.
       // This means generating one for non DS+.
       if (!newRow._id) {
-        newRow._id = `fake-${Helpers.hashString(JSON.stringify(newRow))}`
+        newRow._id = `fake-${Math.random()}`
       }
 
       if (!rowCacheMap[newRow._id]) {
diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
new file mode 100644
index 0000000000..baa7ce3d4d
--- /dev/null
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -0,0 +1,97 @@
+import DataFetch from "./DataFetch.js"
+
+export default class CustomFetch extends DataFetch {
+  getType(value) {
+    if (value == null) {
+      return "string"
+    }
+    const type = typeof value
+    if (type === "object") {
+      if (Array.isArray(value)) {
+        return "array"
+      }
+      return "json"
+    } else {
+      return type
+    }
+  }
+
+  // Parses the custom data into an array format
+  parseCustomData(data) {
+    if (!data) {
+      return []
+    }
+
+    // Happy path - already an array
+    if (Array.isArray(data)) {
+      return data
+    }
+
+    // Handle string cases
+    if (typeof data === "string") {
+      // Try JSON parsing
+      try {
+        data = JSON.parse(data)
+        if (Array.isArray(data)) {
+          return data
+        }
+      } catch (error) {
+        // Ignore
+      }
+
+      // Try a simple CSV
+      return data.split(",").map(x => x.trim())
+    }
+
+    // Other cases we just assume it's a single object and wrap it
+    return [data]
+  }
+
+  // Enriches the custom data to ensure the structure and format is usable
+  enrichCustomData(data) {
+    if (!data?.length) {
+      return []
+    }
+
+    // Filter out any invalid values
+    data = data.filter(x => x != null && !Array.isArray(x))
+
+    // Ensure all values are packed into objects
+    return data.map(x => (typeof x === "object" ? x : { value: x }))
+  }
+
+  getCustomData(datasource) {
+    return this.enrichCustomData(this.parseCustomData(datasource?.data))
+  }
+
+  async getDefinition(datasource) {
+    // Try and work out the schema from the array provided
+    let schema = {}
+    const data = this.getCustomData(datasource)
+
+    // Go through every object and extract all valid keys
+    for (let datum of data) {
+      for (let key of Object.keys(datum)) {
+        if (key === "_id") {
+          continue
+        }
+        if (!schema[key]) {
+          schema[key] = { type: this.getType(datum[key]) }
+        }
+      }
+    }
+
+    return {
+      schema,
+    }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    return {
+      rows: this.getCustomData(datasource),
+      hasNextPage: false,
+      cursor: null,
+    }
+  }
+}
diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.js
index 0edd07762b..d133942bb7 100644
--- a/packages/frontend-core/src/fetch/index.js
+++ b/packages/frontend-core/src/fetch/index.js
@@ -8,6 +8,7 @@ import FieldFetch from "./FieldFetch.js"
 import JSONArrayFetch from "./JSONArrayFetch.js"
 import UserFetch from "./UserFetch.js"
 import GroupUserFetch from "./GroupUserFetch.js"
+import CustomFetch from "./CustomFetch.js"
 
 const DataFetchMap = {
   table: TableFetch,
@@ -17,6 +18,7 @@ const DataFetchMap = {
   link: RelationshipFetch,
   user: UserFetch,
   groupUser: GroupUserFetch,
+  custom: CustomFetch,
 
   // Client specific datasource types
   provider: NestedProviderFetch,

From 678033cc8b7276e5823536925cfebc577527f764 Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Thu, 5 Oct 2023 17:39:40 +0100
Subject: [PATCH 07/95] License Key - Activate & Manage Tests

There are two test files, license.activate.spec.ts and license.manage.spec.ts

These test files each contain a test:
- Creates, activates, and deletes an online license for a self hosted account - license.activate.spec.ts
- Retrieves plans, creates checkout session, and updates license - license.manage.spec.ts

Updated and created API files
- StripeAPI
- LicenseAPI
- internal-api LicenseAPI
- index & AccountInternalAPI also updated to reflect API file changes
---
 .../src/account-api/api/AccountInternalAPI.ts |   4 +-
 .../src/account-api/api/apis/LicenseAPI.ts    | 137 +++++++++++++-----
 qa-core/src/account-api/api/apis/StripeAPI.ts |  64 ++++++++
 qa-core/src/account-api/api/apis/index.ts     |   1 +
 .../tests/licensing/license.activate.spec.ts  |  74 ++++++++++
 .../tests/licensing/license.manage.spec.ts    |  57 ++++++++
 .../src/internal-api/api/apis/LicenseAPI.ts   |  40 +++--
 7 files changed, 329 insertions(+), 48 deletions(-)
 create mode 100644 qa-core/src/account-api/api/apis/StripeAPI.ts
 create mode 100644 qa-core/src/account-api/tests/licensing/license.activate.spec.ts
 create mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts

diff --git a/qa-core/src/account-api/api/AccountInternalAPI.ts b/qa-core/src/account-api/api/AccountInternalAPI.ts
index 3813ad2c9e..ef2c39d5a4 100644
--- a/qa-core/src/account-api/api/AccountInternalAPI.ts
+++ b/qa-core/src/account-api/api/AccountInternalAPI.ts
@@ -1,5 +1,5 @@
 import AccountInternalAPIClient from "./AccountInternalAPIClient"
-import { AccountAPI, LicenseAPI, AuthAPI } from "./apis"
+import { AccountAPI, LicenseAPI, AuthAPI, StripeAPI } from "./apis"
 import { State } from "../../types"
 
 export default class AccountInternalAPI {
@@ -8,11 +8,13 @@ export default class AccountInternalAPI {
   auth: AuthAPI
   accounts: AccountAPI
   licenses: LicenseAPI
+  stripe: StripeAPI
 
   constructor(state: State) {
     this.client = new AccountInternalAPIClient(state)
     this.auth = new AuthAPI(this.client)
     this.accounts = new AccountAPI(this.client)
     this.licenses = new LicenseAPI(this.client)
+    this.stripe = new StripeAPI((this.client))
   }
 }
diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts
index 44579f867b..9f06ec7198 100644
--- a/qa-core/src/account-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts
@@ -2,25 +2,23 @@ import AccountInternalAPIClient from "../AccountInternalAPIClient"
 import {
   Account,
   CreateOfflineLicenseRequest,
+  GetLicenseKeyResponse,
   GetOfflineLicenseResponse,
   UpdateLicenseRequest,
 } from "@budibase/types"
 import { Response } from "node-fetch"
 import BaseAPI from "./BaseAPI"
 import { APIRequestOpts } from "../../../types"
-
 export default class LicenseAPI extends BaseAPI {
   client: AccountInternalAPIClient
-
   constructor(client: AccountInternalAPIClient) {
     super()
     this.client = client
   }
-
   async updateLicense(
-    accountId: string,
-    body: UpdateLicenseRequest,
-    opts: APIRequestOpts = { status: 200 }
+      accountId: string,
+      body: UpdateLicenseRequest,
+      opts: APIRequestOpts = { status: 200 }
   ): Promise<[Response, Account]> {
     return this.doRequest(() => {
       return this.client.put(`/api/accounts/${accountId}/license`, {
@@ -29,44 +27,111 @@ export default class LicenseAPI extends BaseAPI {
       })
     }, opts)
   }
-
   // TODO: Better approach for setting tenant id header
-
   async createOfflineLicense(
-    accountId: string,
-    tenantId: string,
-    body: CreateOfflineLicenseRequest,
-    opts: { status?: number } = {}
+      accountId: string,
+      tenantId: string,
+      body: CreateOfflineLicenseRequest,
+      opts: { status?: number } = {}
   ): Promise<Response> {
     const [response, json] = await this.client.post(
-      `/api/internal/accounts/${accountId}/license/offline`,
-      {
-        body,
-        internal: true,
-        headers: {
-          "x-budibase-tenant-id": tenantId,
+        `/api/internal/accounts/${accountId}/license/offline`,
+        {
+          body,
+          internal: true,
+          headers: {
+            "x-budibase-tenant-id": tenantId,
+          },
+        }
+    )
+    expect(response.status).toBe(opts.status ? opts.status : 201)
+    return response
+  }
+  async getOfflineLicense(
+      accountId: string,
+      tenantId: string,
+      opts: { status?: number } = {}
+  ): Promise<[Response, GetOfflineLicenseResponse]> {
+    const [response, json] = await this.client.get(
+        `/api/internal/accounts/${accountId}/license/offline`,
+        {
+          internal: true,
+          headers: {
+            "x-budibase-tenant-id": tenantId,
+          },
+        }
+    )
+    expect(response.status).toBe(opts.status ? opts.status : 200)
+    return [response, json]
+  }
+  async getLicenseKey(
+      opts: { status?: number } = {}
+  ): Promise<[Response, GetLicenseKeyResponse]> {
+    const [response, json] = await this.client.get(`/api/license/key`)
+    expect(response.status).toBe(opts.status ? opts.status : 200)
+    return [response, json]
+  }
+  async activateLicense(
+      apiKey: string,
+      tenantId: string,
+      licenseKey: string,
+      opts: APIRequestOpts = { status: 200 }
+  ) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/license/activate`, {
+        body: {
+          apiKey: apiKey,
+          tenantId: tenantId,
+          licenseKey: licenseKey,
         },
-      }
+      })
+    }, opts)
+  }
+  async regenerateLicenseKey(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/license/key/regenerate`, {})
+    }, opts)
+  }
+
+  async getPlans(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/plans`)
+    }, opts)
+  }
+
+  async updatePlan(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.put(`/api/license/plan`)
+    }, opts)
+  }
+
+  async refreshAccountLicense(
+      accountId: string,
+      opts: { status?: number } = {}
+  ): Promise<Response> {
+    const [response, json] = await this.client.post(
+        `/api/accounts/${accountId}/license/refresh`,
+        {
+          internal: true,
+        }
     )
     expect(response.status).toBe(opts.status ? opts.status : 201)
     return response
   }
 
-  async getOfflineLicense(
-    accountId: string,
-    tenantId: string,
-    opts: { status?: number } = {}
-  ): Promise<[Response, GetOfflineLicenseResponse]> {
-    const [response, json] = await this.client.get(
-      `/api/internal/accounts/${accountId}/license/offline`,
-      {
-        internal: true,
-        headers: {
-          "x-budibase-tenant-id": tenantId,
-        },
-      }
-    )
-    expect(response.status).toBe(opts.status ? opts.status : 200)
-    return [response, json]
+  async getLicenseUsage(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/license/usage`)
+    }, opts)
   }
-}
+
+  async licenseUsageTriggered(
+      opts: { status?: number } = {}
+  ): Promise<Response> {
+    const [response, json] = await this.client.post(
+        `/api/license/usage/triggered`
+    )
+    expect(response.status).toBe(opts.status ? opts.status : 201)
+    return response
+  }
+}
\ No newline at end of file
diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts
new file mode 100644
index 0000000000..ffa96b3c2b
--- /dev/null
+++ b/qa-core/src/account-api/api/apis/StripeAPI.ts
@@ -0,0 +1,64 @@
+import AccountInternalAPIClient from "../AccountInternalAPIClient"
+import BaseAPI from "./BaseAPI"
+import { APIRequestOpts } from "../../../types"
+
+export default class StripeAPI extends BaseAPI {
+    client: AccountInternalAPIClient
+
+    constructor(client: AccountInternalAPIClient) {
+        super()
+        this.client = client
+    }
+
+    async createCheckoutSession(
+        priceId: string,
+        opts: APIRequestOpts = { status: 200 }
+    ) {
+        return this.doRequest(() => {
+            return this.client.post(`/api/stripe/checkout-session`, {
+                body: { priceId },
+            })
+        }, opts)
+    }
+
+    async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) {
+        return this.doRequest(() => {
+            return this.client.post(`/api/stripe/checkout-success`)
+        }, opts)
+    }
+
+    async createPortalSession(
+        stripeCustomerId: string,
+        opts: APIRequestOpts = { status: 200 }
+    ) {
+        return this.doRequest(() => {
+            return this.client.post(`/api/stripe/portal-session`, {
+                body: { stripeCustomerId },
+            })
+        }, opts)
+    }
+
+    async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) {
+        return this.doRequest(() => {
+            return this.client.post(`/api/stripe/link`)
+        }, opts)
+    }
+
+    async getInvoices(opts: APIRequestOpts = { status: 200 }) {
+        return this.doRequest(() => {
+            return this.client.get(`/api/stripe/invoices`)
+        }, opts)
+    }
+
+    async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) {
+        return this.doRequest(() => {
+            return this.client.get(`/api/stripe/upcoming-invoice`)
+        }, opts)
+    }
+
+    async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) {
+        return this.doRequest(() => {
+            return this.client.get(`/api/stripe/customers`)
+        }, opts)
+    }
+}
diff --git a/qa-core/src/account-api/api/apis/index.ts b/qa-core/src/account-api/api/apis/index.ts
index 1137ac3e36..5b0cf55110 100644
--- a/qa-core/src/account-api/api/apis/index.ts
+++ b/qa-core/src/account-api/api/apis/index.ts
@@ -1,3 +1,4 @@
 export { default as AuthAPI } from "./AuthAPI"
 export { default as AccountAPI } from "./AccountAPI"
 export { default as LicenseAPI } from "./LicenseAPI"
+export { default as StripeAPI } from "./StripeAPI"
diff --git a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
new file mode 100644
index 0000000000..709e2c33f0
--- /dev/null
+++ b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
@@ -0,0 +1,74 @@
+import TestConfiguration from "../../config/TestConfiguration"
+import * as fixures from "../../fixtures"
+import { Feature, Hosting } from "@budibase/types"
+
+describe("license activation", () => {
+    const config = new TestConfiguration()
+
+    beforeAll(async () => {
+        await config.beforeAll()
+    })
+
+    afterAll(async () => {
+        await config.afterAll()
+    })
+
+    it("creates, activates and deletes online license - self host", async () => {
+        // Remove existing license key
+        await config.internalApi.license.deleteLicenseKey()
+
+        // Verify license key not found
+        await config.internalApi.license.getLicenseKey({ status: 404 })
+
+        // Create self host account
+        const createAccountRequest = fixures.accounts.generateAccount({
+            hosting: Hosting.SELF,
+        })
+        const [createAccountRes, account] =
+            await config.accountsApi.accounts.create(createAccountRequest, { autoVerify: true })
+
+        let licenseKey: string = " "
+        await config.doInNewState(async () => {
+            await config.loginAsAccount(createAccountRequest)
+            // Retrieve license key
+            const [res, body] =
+                await config.accountsApi.licenses.getLicenseKey()
+            licenseKey = body.licenseKey
+        })
+
+        const accountId = account.accountId!
+
+        // Update license to have paid feature
+        const [res, acc] = await config.accountsApi.licenses.updateLicense(
+            accountId,
+            {
+                overrides: {
+                    features: [Feature.APP_BACKUPS],
+                },
+            }
+        )
+
+        // Activate license key
+        await config.internalApi.license.activateLicenseKey({licenseKey})
+
+        // Verify license updated with new feature
+        await config.doInNewState(async () => {
+            await config.loginAsAccount(createAccountRequest)
+            const [selfRes, body] = await config.api.accounts.self()
+            expect(body.license.features[0]).toBe("appBackups")
+        })
+
+        // Remove license key
+        await config.internalApi.license.deleteLicenseKey()
+
+        // Verify license key not found
+        await config.internalApi.license.getLicenseKey({ status: 404 })
+
+        // Verify user downgraded to free license
+        await config.doInNewState(async () => {
+            await config.loginAsAccount(createAccountRequest)
+            const [selfRes, body] = await config.api.accounts.self()
+            expect(body.license.plan.type).toBe("free")
+        })
+    })
+})
diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
new file mode 100644
index 0000000000..967252a0f9
--- /dev/null
+++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
@@ -0,0 +1,57 @@
+import TestConfiguration from "../../config/TestConfiguration"
+import * as fixtures from "../../fixtures"
+import { Hosting, PlanType } from "@budibase/types"
+
+describe("license management", () => {
+    const config = new TestConfiguration()
+
+    beforeAll(async () => {
+        await config.beforeAll()
+    })
+
+    afterAll(async () => {
+        await config.afterAll()
+    })
+
+    it("retrieves plans, creates checkout session, and updates license", async () => {
+        // Create cloud account
+        const createAccountRequest = fixtures.accounts.generateAccount({
+            hosting: Hosting.CLOUD,
+        })
+
+        // Self response has free license
+        const [selfRes, selfBody] = await config.api.accounts.self()
+        expect(selfBody.license.plan.type).toBe(PlanType.FREE)
+
+        // Retrieve plans
+        const [plansRes, planBody] = await config.api.licenses.getPlans()
+
+        // Select priceId from premium plan
+        let premiumPriceId = null
+        for (const plan of planBody) {
+            if (plan.type === PlanType.PREMIUM) {
+                premiumPriceId = plan.prices[0].priceId
+                break
+            }
+        }
+
+        // Create checkout session for price
+        const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
+            premiumPriceId
+        )
+        const checkoutSessionUrl = checkoutSessionRes[1].url
+        expect(checkoutSessionUrl).toContain("checkout.stripe.com")
+
+        // TODO: Mimic checkout success
+        // Create stripe customer
+        // Create subscription for premium plan
+        // Assert license updated from free to premium
+
+        // Create portal session
+        //await config.api.stripe.createPortalSession()
+
+        // Update from free to business license
+
+        // License updated
+    })
+})
diff --git a/qa-core/src/internal-api/api/apis/LicenseAPI.ts b/qa-core/src/internal-api/api/apis/LicenseAPI.ts
index 4c9d14c55e..268f8781c3 100644
--- a/qa-core/src/internal-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/internal-api/api/apis/LicenseAPI.ts
@@ -1,45 +1,63 @@
 import { Response } from "node-fetch"
 import {
+  ActivateLicenseKeyRequest,
   ActivateOfflineLicenseTokenRequest,
+  GetLicenseKeyResponse,
   GetOfflineIdentifierResponse,
   GetOfflineLicenseTokenResponse,
 } from "@budibase/types"
 import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
 import BaseAPI from "./BaseAPI"
+import { APIRequestOpts } from "../../../types"
 
 export default class LicenseAPI extends BaseAPI {
   constructor(client: BudibaseInternalAPIClient) {
     super(client)
   }
-
   async getOfflineLicenseToken(
-    opts: { status?: number } = {}
+      opts: { status?: number } = {}
   ): Promise<[Response, GetOfflineLicenseTokenResponse]> {
     const [response, body] = await this.get(
-      `/global/license/offline`,
-      opts.status
+        `/global/license/offline`,
+        opts.status
     )
     return [response, body]
   }
-
   async deleteOfflineLicenseToken(): Promise<[Response]> {
     const [response] = await this.del(`/global/license/offline`, 204)
     return [response]
   }
-
   async activateOfflineLicenseToken(
-    body: ActivateOfflineLicenseTokenRequest
+      body: ActivateOfflineLicenseTokenRequest
   ): Promise<[Response]> {
     const [response] = await this.post(`/global/license/offline`, body)
     return [response]
   }
-
   async getOfflineIdentifier(): Promise<
-    [Response, GetOfflineIdentifierResponse]
+      [Response, GetOfflineIdentifierResponse]
   > {
     const [response, body] = await this.get(
-      `/global/license/offline/identifier`
+        `/global/license/offline/identifier`
     )
     return [response, body]
   }
-}
+
+  async getLicenseKey(
+      opts: { status?: number } = {}
+  ): Promise<[Response, GetLicenseKeyResponse]> {
+    const [response, body] = await this.get(`/global/license/key`, opts.status)
+    return [response, body]
+  }
+
+  async activateLicenseKey(
+      body: ActivateLicenseKeyRequest
+  ): Promise<[Response]> {
+    const [response] = await this.post(`/global/license/key`, body)
+    return [response]
+  }
+
+  async deleteLicenseKey(): Promise<[Response]> {
+    const [response] = await this.del(`/global/license/key`, 204)
+    return [response]
+  }
+}
\ No newline at end of file

From 5e16d0451936ed2b1c2c0c6d1928c146b45c65c1 Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Thu, 5 Oct 2023 17:43:25 +0100
Subject: [PATCH 08/95] lint

---
 .../src/account-api/api/AccountInternalAPI.ts |   2 +-
 .../src/account-api/api/apis/LicenseAPI.ts    |  78 +++++------
 qa-core/src/account-api/api/apis/StripeAPI.ts | 100 +++++++--------
 .../tests/licensing/license.activate.spec.ts  | 121 +++++++++---------
 .../tests/licensing/license.manage.spec.ts    |  82 ++++++------
 .../src/internal-api/api/apis/LicenseAPI.ts   |  18 +--
 6 files changed, 201 insertions(+), 200 deletions(-)

diff --git a/qa-core/src/account-api/api/AccountInternalAPI.ts b/qa-core/src/account-api/api/AccountInternalAPI.ts
index ef2c39d5a4..f89bf556f2 100644
--- a/qa-core/src/account-api/api/AccountInternalAPI.ts
+++ b/qa-core/src/account-api/api/AccountInternalAPI.ts
@@ -15,6 +15,6 @@ export default class AccountInternalAPI {
     this.auth = new AuthAPI(this.client)
     this.accounts = new AccountAPI(this.client)
     this.licenses = new LicenseAPI(this.client)
-    this.stripe = new StripeAPI((this.client))
+    this.stripe = new StripeAPI(this.client)
   }
 }
diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts
index 9f06ec7198..dba1a661d4 100644
--- a/qa-core/src/account-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts
@@ -16,9 +16,9 @@ export default class LicenseAPI extends BaseAPI {
     this.client = client
   }
   async updateLicense(
-      accountId: string,
-      body: UpdateLicenseRequest,
-      opts: APIRequestOpts = { status: 200 }
+    accountId: string,
+    body: UpdateLicenseRequest,
+    opts: APIRequestOpts = { status: 200 }
   ): Promise<[Response, Account]> {
     return this.doRequest(() => {
       return this.client.put(`/api/accounts/${accountId}/license`, {
@@ -29,53 +29,53 @@ export default class LicenseAPI extends BaseAPI {
   }
   // TODO: Better approach for setting tenant id header
   async createOfflineLicense(
-      accountId: string,
-      tenantId: string,
-      body: CreateOfflineLicenseRequest,
-      opts: { status?: number } = {}
+    accountId: string,
+    tenantId: string,
+    body: CreateOfflineLicenseRequest,
+    opts: { status?: number } = {}
   ): Promise<Response> {
     const [response, json] = await this.client.post(
-        `/api/internal/accounts/${accountId}/license/offline`,
-        {
-          body,
-          internal: true,
-          headers: {
-            "x-budibase-tenant-id": tenantId,
-          },
-        }
+      `/api/internal/accounts/${accountId}/license/offline`,
+      {
+        body,
+        internal: true,
+        headers: {
+          "x-budibase-tenant-id": tenantId,
+        },
+      }
     )
     expect(response.status).toBe(opts.status ? opts.status : 201)
     return response
   }
   async getOfflineLicense(
-      accountId: string,
-      tenantId: string,
-      opts: { status?: number } = {}
+    accountId: string,
+    tenantId: string,
+    opts: { status?: number } = {}
   ): Promise<[Response, GetOfflineLicenseResponse]> {
     const [response, json] = await this.client.get(
-        `/api/internal/accounts/${accountId}/license/offline`,
-        {
-          internal: true,
-          headers: {
-            "x-budibase-tenant-id": tenantId,
-          },
-        }
+      `/api/internal/accounts/${accountId}/license/offline`,
+      {
+        internal: true,
+        headers: {
+          "x-budibase-tenant-id": tenantId,
+        },
+      }
     )
     expect(response.status).toBe(opts.status ? opts.status : 200)
     return [response, json]
   }
   async getLicenseKey(
-      opts: { status?: number } = {}
+    opts: { status?: number } = {}
   ): Promise<[Response, GetLicenseKeyResponse]> {
     const [response, json] = await this.client.get(`/api/license/key`)
     expect(response.status).toBe(opts.status ? opts.status : 200)
     return [response, json]
   }
   async activateLicense(
-      apiKey: string,
-      tenantId: string,
-      licenseKey: string,
-      opts: APIRequestOpts = { status: 200 }
+    apiKey: string,
+    tenantId: string,
+    licenseKey: string,
+    opts: APIRequestOpts = { status: 200 }
   ) {
     return this.doRequest(() => {
       return this.client.post(`/api/license/activate`, {
@@ -106,14 +106,14 @@ export default class LicenseAPI extends BaseAPI {
   }
 
   async refreshAccountLicense(
-      accountId: string,
-      opts: { status?: number } = {}
+    accountId: string,
+    opts: { status?: number } = {}
   ): Promise<Response> {
     const [response, json] = await this.client.post(
-        `/api/accounts/${accountId}/license/refresh`,
-        {
-          internal: true,
-        }
+      `/api/accounts/${accountId}/license/refresh`,
+      {
+        internal: true,
+      }
     )
     expect(response.status).toBe(opts.status ? opts.status : 201)
     return response
@@ -126,12 +126,12 @@ export default class LicenseAPI extends BaseAPI {
   }
 
   async licenseUsageTriggered(
-      opts: { status?: number } = {}
+    opts: { status?: number } = {}
   ): Promise<Response> {
     const [response, json] = await this.client.post(
-        `/api/license/usage/triggered`
+      `/api/license/usage/triggered`
     )
     expect(response.status).toBe(opts.status ? opts.status : 201)
     return response
   }
-}
\ No newline at end of file
+}
diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts
index ffa96b3c2b..c9c776e89b 100644
--- a/qa-core/src/account-api/api/apis/StripeAPI.ts
+++ b/qa-core/src/account-api/api/apis/StripeAPI.ts
@@ -3,62 +3,62 @@ import BaseAPI from "./BaseAPI"
 import { APIRequestOpts } from "../../../types"
 
 export default class StripeAPI extends BaseAPI {
-    client: AccountInternalAPIClient
+  client: AccountInternalAPIClient
 
-    constructor(client: AccountInternalAPIClient) {
-        super()
-        this.client = client
-    }
+  constructor(client: AccountInternalAPIClient) {
+    super()
+    this.client = client
+  }
 
-    async createCheckoutSession(
-        priceId: string,
-        opts: APIRequestOpts = { status: 200 }
-    ) {
-        return this.doRequest(() => {
-            return this.client.post(`/api/stripe/checkout-session`, {
-                body: { priceId },
-            })
-        }, opts)
-    }
+  async createCheckoutSession(
+    priceId: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/checkout-session`, {
+        body: { priceId },
+      })
+    }, opts)
+  }
 
-    async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) {
-        return this.doRequest(() => {
-            return this.client.post(`/api/stripe/checkout-success`)
-        }, opts)
-    }
+  async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/checkout-success`)
+    }, opts)
+  }
 
-    async createPortalSession(
-        stripeCustomerId: string,
-        opts: APIRequestOpts = { status: 200 }
-    ) {
-        return this.doRequest(() => {
-            return this.client.post(`/api/stripe/portal-session`, {
-                body: { stripeCustomerId },
-            })
-        }, opts)
-    }
+  async createPortalSession(
+    stripeCustomerId: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/portal-session`, {
+        body: { stripeCustomerId },
+      })
+    }, opts)
+  }
 
-    async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) {
-        return this.doRequest(() => {
-            return this.client.post(`/api/stripe/link`)
-        }, opts)
-    }
+  async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/link`)
+    }, opts)
+  }
 
-    async getInvoices(opts: APIRequestOpts = { status: 200 }) {
-        return this.doRequest(() => {
-            return this.client.get(`/api/stripe/invoices`)
-        }, opts)
-    }
+  async getInvoices(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/stripe/invoices`)
+    }, opts)
+  }
 
-    async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) {
-        return this.doRequest(() => {
-            return this.client.get(`/api/stripe/upcoming-invoice`)
-        }, opts)
-    }
+  async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/stripe/upcoming-invoice`)
+    }, opts)
+  }
 
-    async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) {
-        return this.doRequest(() => {
-            return this.client.get(`/api/stripe/customers`)
-        }, opts)
-    }
+  async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/stripe/customers`)
+    }, opts)
+  }
 }
diff --git a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
index 709e2c33f0..a494ceb354 100644
--- a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
+++ b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
@@ -3,72 +3,73 @@ import * as fixures from "../../fixtures"
 import { Feature, Hosting } from "@budibase/types"
 
 describe("license activation", () => {
-    const config = new TestConfiguration()
+  const config = new TestConfiguration()
 
-    beforeAll(async () => {
-        await config.beforeAll()
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  it("creates, activates and deletes online license - self host", async () => {
+    // Remove existing license key
+    await config.internalApi.license.deleteLicenseKey()
+
+    // Verify license key not found
+    await config.internalApi.license.getLicenseKey({ status: 404 })
+
+    // Create self host account
+    const createAccountRequest = fixures.accounts.generateAccount({
+      hosting: Hosting.SELF,
+    })
+    const [createAccountRes, account] =
+      await config.accountsApi.accounts.create(createAccountRequest, {
+        autoVerify: true,
+      })
+
+    let licenseKey: string = " "
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      // Retrieve license key
+      const [res, body] = await config.accountsApi.licenses.getLicenseKey()
+      licenseKey = body.licenseKey
     })
 
-    afterAll(async () => {
-        await config.afterAll()
+    const accountId = account.accountId!
+
+    // Update license to have paid feature
+    const [res, acc] = await config.accountsApi.licenses.updateLicense(
+      accountId,
+      {
+        overrides: {
+          features: [Feature.APP_BACKUPS],
+        },
+      }
+    )
+
+    // Activate license key
+    await config.internalApi.license.activateLicenseKey({ licenseKey })
+
+    // Verify license updated with new feature
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      const [selfRes, body] = await config.api.accounts.self()
+      expect(body.license.features[0]).toBe("appBackups")
     })
 
-    it("creates, activates and deletes online license - self host", async () => {
-        // Remove existing license key
-        await config.internalApi.license.deleteLicenseKey()
+    // Remove license key
+    await config.internalApi.license.deleteLicenseKey()
 
-        // Verify license key not found
-        await config.internalApi.license.getLicenseKey({ status: 404 })
+    // Verify license key not found
+    await config.internalApi.license.getLicenseKey({ status: 404 })
 
-        // Create self host account
-        const createAccountRequest = fixures.accounts.generateAccount({
-            hosting: Hosting.SELF,
-        })
-        const [createAccountRes, account] =
-            await config.accountsApi.accounts.create(createAccountRequest, { autoVerify: true })
-
-        let licenseKey: string = " "
-        await config.doInNewState(async () => {
-            await config.loginAsAccount(createAccountRequest)
-            // Retrieve license key
-            const [res, body] =
-                await config.accountsApi.licenses.getLicenseKey()
-            licenseKey = body.licenseKey
-        })
-
-        const accountId = account.accountId!
-
-        // Update license to have paid feature
-        const [res, acc] = await config.accountsApi.licenses.updateLicense(
-            accountId,
-            {
-                overrides: {
-                    features: [Feature.APP_BACKUPS],
-                },
-            }
-        )
-
-        // Activate license key
-        await config.internalApi.license.activateLicenseKey({licenseKey})
-
-        // Verify license updated with new feature
-        await config.doInNewState(async () => {
-            await config.loginAsAccount(createAccountRequest)
-            const [selfRes, body] = await config.api.accounts.self()
-            expect(body.license.features[0]).toBe("appBackups")
-        })
-
-        // Remove license key
-        await config.internalApi.license.deleteLicenseKey()
-
-        // Verify license key not found
-        await config.internalApi.license.getLicenseKey({ status: 404 })
-
-        // Verify user downgraded to free license
-        await config.doInNewState(async () => {
-            await config.loginAsAccount(createAccountRequest)
-            const [selfRes, body] = await config.api.accounts.self()
-            expect(body.license.plan.type).toBe("free")
-        })
+    // Verify user downgraded to free license
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      const [selfRes, body] = await config.api.accounts.self()
+      expect(body.license.plan.type).toBe("free")
     })
+  })
 })
diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
index 967252a0f9..3f87838ee4 100644
--- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
+++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
@@ -3,55 +3,55 @@ import * as fixtures from "../../fixtures"
 import { Hosting, PlanType } from "@budibase/types"
 
 describe("license management", () => {
-    const config = new TestConfiguration()
+  const config = new TestConfiguration()
 
-    beforeAll(async () => {
-        await config.beforeAll()
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  it("retrieves plans, creates checkout session, and updates license", async () => {
+    // Create cloud account
+    const createAccountRequest = fixtures.accounts.generateAccount({
+      hosting: Hosting.CLOUD,
     })
 
-    afterAll(async () => {
-        await config.afterAll()
-    })
+    // Self response has free license
+    const [selfRes, selfBody] = await config.api.accounts.self()
+    expect(selfBody.license.plan.type).toBe(PlanType.FREE)
 
-    it("retrieves plans, creates checkout session, and updates license", async () => {
-        // Create cloud account
-        const createAccountRequest = fixtures.accounts.generateAccount({
-            hosting: Hosting.CLOUD,
-        })
+    // Retrieve plans
+    const [plansRes, planBody] = await config.api.licenses.getPlans()
 
-        // Self response has free license
-        const [selfRes, selfBody] = await config.api.accounts.self()
-        expect(selfBody.license.plan.type).toBe(PlanType.FREE)
+    // Select priceId from premium plan
+    let premiumPriceId = null
+    for (const plan of planBody) {
+      if (plan.type === PlanType.PREMIUM) {
+        premiumPriceId = plan.prices[0].priceId
+        break
+      }
+    }
 
-        // Retrieve plans
-        const [plansRes, planBody] = await config.api.licenses.getPlans()
+    // Create checkout session for price
+    const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
+      premiumPriceId
+    )
+    const checkoutSessionUrl = checkoutSessionRes[1].url
+    expect(checkoutSessionUrl).toContain("checkout.stripe.com")
 
-        // Select priceId from premium plan
-        let premiumPriceId = null
-        for (const plan of planBody) {
-            if (plan.type === PlanType.PREMIUM) {
-                premiumPriceId = plan.prices[0].priceId
-                break
-            }
-        }
+    // TODO: Mimic checkout success
+    // Create stripe customer
+    // Create subscription for premium plan
+    // Assert license updated from free to premium
 
-        // Create checkout session for price
-        const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
-            premiumPriceId
-        )
-        const checkoutSessionUrl = checkoutSessionRes[1].url
-        expect(checkoutSessionUrl).toContain("checkout.stripe.com")
+    // Create portal session
+    //await config.api.stripe.createPortalSession()
 
-        // TODO: Mimic checkout success
-        // Create stripe customer
-        // Create subscription for premium plan
-        // Assert license updated from free to premium
+    // Update from free to business license
 
-        // Create portal session
-        //await config.api.stripe.createPortalSession()
-
-        // Update from free to business license
-
-        // License updated
-    })
+    // License updated
+  })
 })
diff --git a/qa-core/src/internal-api/api/apis/LicenseAPI.ts b/qa-core/src/internal-api/api/apis/LicenseAPI.ts
index 268f8781c3..ef322e069a 100644
--- a/qa-core/src/internal-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/internal-api/api/apis/LicenseAPI.ts
@@ -15,11 +15,11 @@ export default class LicenseAPI extends BaseAPI {
     super(client)
   }
   async getOfflineLicenseToken(
-      opts: { status?: number } = {}
+    opts: { status?: number } = {}
   ): Promise<[Response, GetOfflineLicenseTokenResponse]> {
     const [response, body] = await this.get(
-        `/global/license/offline`,
-        opts.status
+      `/global/license/offline`,
+      opts.status
     )
     return [response, body]
   }
@@ -28,29 +28,29 @@ export default class LicenseAPI extends BaseAPI {
     return [response]
   }
   async activateOfflineLicenseToken(
-      body: ActivateOfflineLicenseTokenRequest
+    body: ActivateOfflineLicenseTokenRequest
   ): Promise<[Response]> {
     const [response] = await this.post(`/global/license/offline`, body)
     return [response]
   }
   async getOfflineIdentifier(): Promise<
-      [Response, GetOfflineIdentifierResponse]
+    [Response, GetOfflineIdentifierResponse]
   > {
     const [response, body] = await this.get(
-        `/global/license/offline/identifier`
+      `/global/license/offline/identifier`
     )
     return [response, body]
   }
 
   async getLicenseKey(
-      opts: { status?: number } = {}
+    opts: { status?: number } = {}
   ): Promise<[Response, GetLicenseKeyResponse]> {
     const [response, body] = await this.get(`/global/license/key`, opts.status)
     return [response, body]
   }
 
   async activateLicenseKey(
-      body: ActivateLicenseKeyRequest
+    body: ActivateLicenseKeyRequest
   ): Promise<[Response]> {
     const [response] = await this.post(`/global/license/key`, body)
     return [response]
@@ -60,4 +60,4 @@ export default class LicenseAPI extends BaseAPI {
     const [response] = await this.del(`/global/license/key`, 204)
     return [response]
   }
-}
\ No newline at end of file
+}

From 9667c954ef79288a42838c0a72c8bea6490d9a63 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 17:55:57 +0100
Subject: [PATCH 09/95] Enable uploading a CSV file as a datasource and fix
 edge cases

---
 .../settings/controls/DataSourceSelect.svelte | 44 ++++++++++++++++---
 .../src/components/grid/stores/datasource.js  |  6 +++
 .../grid/stores/datasources/nonPlus.js        |  9 ++++
 .../grid/stores/datasources/table.js          |  8 ++++
 .../grid/stores/datasources/viewV2.js         |  8 ++++
 .../src/components/grid/stores/rows.js        |  7 ++-
 .../frontend-core/src/fetch/CustomFetch.js    |  6 +--
 7 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
index e4b3f972ba..de9a37dfae 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
@@ -10,6 +10,10 @@
     Drawer,
     DrawerContent,
     Icon,
+    Modal,
+    ModalContent,
+    CoreDropzone,
+    notifications,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { store, currentAsset } from "builderStore"
@@ -24,6 +28,7 @@
   import IntegrationQueryEditor from "components/integration/index.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { API } from "api"
 
   export let value = {}
   export let otherSources
@@ -39,6 +44,7 @@
   let tmpQueryParams
   let tmpCustomData
   let customDataValid = true
+  let modal
 
   $: text = value?.label ?? "Choose an option"
   $: tables = $tablesStore.list.map(m => ({
@@ -184,6 +190,26 @@
     })
     drawer.hide()
   }
+
+  const promptForCSV = () => {
+    drawer.hide()
+    modal.show()
+  }
+
+  const handleCSV = async e => {
+    try {
+      const csv = await e.detail[0]?.text()
+      if (csv?.length) {
+        const js = await API.csvToJson(csv)
+        tmpCustomData = JSON.stringify(js)
+      }
+    } catch (error) {
+      console.log(error)
+      notifications.error("Failed to parse CSV")
+    }
+    modal.hide()
+    drawer.show()
+  }
 </script>
 
 <div class="container" bind:this={anchorRight}>
@@ -227,12 +253,12 @@
       <Icon hoverable name="Settings" on:click={openCustomDrawer} />
     </div>
     <Drawer title="Custom data" bind:this={drawer}>
-      <Button
-        slot="buttons"
-        cta
-        on:click={saveCustomData}
-        disabled={!customDataValid}>Save</Button
-      >
+      <div slot="buttons" style="display:contents">
+        <Button primary on:click={promptForCSV}>Load CSV</Button>
+        <Button cta on:click={saveCustomData} disabled={!customDataValid}>
+          Save
+        </Button>
+      </div>
       <div slot="description">
         Provide a JavaScript or JSON array to use as data
       </div>
@@ -349,6 +375,12 @@
   </div>
 </Popover>
 
+<Modal bind:this={modal}>
+  <ModalContent title="Choose a CSV">
+    <CoreDropzone compact extensions=".csv" on:change={handleCSV} />
+  </ModalContent>
+</Modal>
+
 <style>
   .container {
     display: flex;
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 6a10cb8b9b..7f69d0031d 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -146,6 +146,11 @@ export const createActions = context => {
     return getAPI()?.actions.canUseColumn(name)
   }
 
+  // Gets the default number of rows for a single page
+  const getFeatures = () => {
+    return getAPI()?.actions.getFeatures()
+  }
+
   return {
     datasource: {
       ...datasource,
@@ -158,6 +163,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
index e04d05ec59..b89edb348c 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
@@ -1,4 +1,7 @@
 import { get } from "svelte/store"
+import QueryFetch from "../../../../fetch/QueryFetch"
+
+export const RowPageSize = 100000
 
 export const createActions = context => {
   const { columns, stickyColumn, table, viewV2 } = context
@@ -35,6 +38,11 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
+  const getFeatures = () => {
+    // We don't support any features
+    return {}
+  }
+
   return {
     nonPlus: {
       actions: {
@@ -45,6 +53,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js
index 9a4058ac8f..5409f95092 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/table.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/table.js
@@ -1,7 +1,10 @@
 import { get } from "svelte/store"
+import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
+export const RowPageSize = 100
+
 export const createActions = context => {
   const { API, datasource, columns, stickyColumn } = context
 
@@ -45,6 +48,10 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
+  const getFeatures = () => {
+    return new TableFetch(null).determineFeatureFlags()
+  }
+
   return {
     table: {
       actions: {
@@ -55,6 +62,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 000727b262..bd8e023149 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -1,7 +1,10 @@
 import { get } from "svelte/store"
+import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
+export const RowPageSize = 100
+
 export const createActions = context => {
   const { API, datasource, columns, stickyColumn } = context
 
@@ -45,6 +48,10 @@ export const createActions = context => {
     )
   }
 
+  const getFeatures = () => {
+    return new ViewV2Fetch(null).determineFeatureFlags()
+  }
+
   return {
     viewV2: {
       actions: {
@@ -55,6 +62,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 5b6f4fd7f9..3acd0d5e4d 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -2,7 +2,6 @@ import { writable, derived, get } from "svelte/store"
 import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import { tick } from "svelte"
-import { Helpers } from "@budibase/bbui"
 
 export const createStores = () => {
   const rows = writable([])
@@ -111,6 +110,10 @@ export const createActions = context => {
     const $filter = get(filter)
     const $sort = get(sort)
 
+    // Determine how many rows to fetch per page
+    const features = datasource.actions.getFeatures()
+    const limit = features?.supportsPagination ? RowPageSize : 100000
+
     // Create new fetch model
     const newFetch = fetchData({
       API,
@@ -119,7 +122,7 @@ export const createActions = context => {
         filter: $filter,
         sortColumn: $sort.column,
         sortOrder: $sort.order,
-        limit: RowPageSize,
+        limit,
         paginate: true,
       },
     })
diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index baa7ce3d4d..02756fa137 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -31,9 +31,9 @@ export default class CustomFetch extends DataFetch {
     if (typeof data === "string") {
       // Try JSON parsing
       try {
-        data = JSON.parse(data)
-        if (Array.isArray(data)) {
-          return data
+        const js = JSON.parse(data)
+        if (Array.isArray(js)) {
+          return js
         }
       } catch (error) {
         // Ignore

From 5990f43f3f5e1c4d80c41ee7e4380cdac26548e6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 18:30:40 +0100
Subject: [PATCH 10/95] Fix issues with fetching features and bindings in
 custom datasources

---
 .../settings/controls/DataSourceSelect.svelte       | 13 ++++++++-----
 .../src/components/grid/stores/datasources/table.js |  2 +-
 .../components/grid/stores/datasources/viewV2.js    |  2 +-
 .../src/components/grid/stores/rows.js              |  2 +-
 4 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
index de9a37dfae..887ce1090d 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
@@ -1,5 +1,9 @@
 <script>
-  import { getContextProviderComponents } from "builderStore/dataBinding"
+  import {
+    getContextProviderComponents,
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
   import {
     Button,
     Popover,
@@ -167,7 +171,7 @@
   }
 
   const openCustomDrawer = () => {
-    tmpCustomData = value.data || ""
+    tmpCustomData = runtimeToReadableBinding(bindings, value.data || "")
     drawer.show()
   }
 
@@ -186,7 +190,7 @@
   const saveCustomData = () => {
     handleSelected({
       ...value,
-      data: tmpCustomData,
+      data: readableToRuntimeBinding(bindings, tmpCustomData),
     })
     drawer.hide()
   }
@@ -204,7 +208,6 @@
         tmpCustomData = JSON.stringify(js)
       }
     } catch (error) {
-      console.log(error)
       notifications.error("Failed to parse CSV")
     }
     modal.hide()
@@ -376,7 +379,7 @@
 </Popover>
 
 <Modal bind:this={modal}>
-  <ModalContent title="Choose a CSV">
+  <ModalContent title="Load CSV" showConfirmButton={false}>
     <CoreDropzone compact extensions=".csv" on:change={handleCSV} />
   </ModalContent>
 </Modal>
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js
index 5409f95092..beafd1dcfe 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/table.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/table.js
@@ -49,7 +49,7 @@ export const createActions = context => {
   }
 
   const getFeatures = () => {
-    return new TableFetch(null).determineFeatureFlags()
+    return new TableFetch({ API }).determineFeatureFlags()
   }
 
   return {
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index bd8e023149..78e8aa8f12 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -49,7 +49,7 @@ export const createActions = context => {
   }
 
   const getFeatures = () => {
-    return new ViewV2Fetch(null).determineFeatureFlags()
+    return new ViewV2Fetch({ API }).determineFeatureFlags()
   }
 
   return {
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 3acd0d5e4d..667f0d323e 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -112,7 +112,7 @@ export const createActions = context => {
 
     // Determine how many rows to fetch per page
     const features = datasource.actions.getFeatures()
-    const limit = features?.supportsPagination ? RowPageSize : 100000
+    const limit = features?.supportsPagination ? RowPageSize : null
 
     // Create new fetch model
     const newFetch = fetchData({

From 0e394f0259daa1894075a15d5c1cee0b0a9566e8 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 18:31:43 +0100
Subject: [PATCH 11/95] Lint

---
 .../src/components/grid/stores/datasources/nonPlus.js          | 3 ---
 .../src/components/grid/stores/datasources/table.js            | 2 --
 .../src/components/grid/stores/datasources/viewV2.js           | 2 --
 3 files changed, 7 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
index b89edb348c..a05e1f7d37 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
@@ -1,7 +1,4 @@
 import { get } from "svelte/store"
-import QueryFetch from "../../../../fetch/QueryFetch"
-
-export const RowPageSize = 100000
 
 export const createActions = context => {
   const { columns, stickyColumn, table, viewV2 } = context
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js
index beafd1dcfe..9ced1530ba 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/table.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/table.js
@@ -3,8 +3,6 @@ import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
-export const RowPageSize = 100
-
 export const createActions = context => {
   const { API, datasource, columns, stickyColumn } = context
 
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 78e8aa8f12..f0572003c2 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -3,8 +3,6 @@ import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
-export const RowPageSize = 100
-
 export const createActions = context => {
   const { API, datasource, columns, stickyColumn } = context
 

From ecd2ed3d07805f92d552e403150bfc304fbd3aa8 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 19:00:11 +0100
Subject: [PATCH 12/95] Improve custom datasource schema generation and error
 resistance

---
 .../frontend-core/src/fetch/CustomFetch.js    | 46 ++++++++++++++-----
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index 02756fa137..5a92de635a 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -1,6 +1,7 @@
 import DataFetch from "./DataFetch.js"
 
 export default class CustomFetch extends DataFetch {
+  // Gets the correct Budibase type for a JS value
   getType(value) {
     if (value == null) {
       return "string"
@@ -8,11 +9,15 @@ export default class CustomFetch extends DataFetch {
     const type = typeof value
     if (type === "object") {
       if (Array.isArray(value)) {
+        // Use our custom array type to render badges
         return "array"
       }
+      // Use JSON for objects to ensure they are stringified
       return "json"
+    } else if (!isNaN(value)) {
+      return "number"
     } else {
-      return type
+      return "string"
     }
   }
 
@@ -27,9 +32,8 @@ export default class CustomFetch extends DataFetch {
       return data
     }
 
-    // Handle string cases
+    // For strings, try JSON then fall back to attempting a CSV
     if (typeof data === "string") {
-      // Try JSON parsing
       try {
         const js = JSON.parse(data)
         if (Array.isArray(js)) {
@@ -38,8 +42,6 @@ export default class CustomFetch extends DataFetch {
       } catch (error) {
         // Ignore
       }
-
-      // Try a simple CSV
       return data.split(",").map(x => x.trim())
     }
 
@@ -54,12 +56,13 @@ export default class CustomFetch extends DataFetch {
     }
 
     // Filter out any invalid values
-    data = data.filter(x => x != null && !Array.isArray(x))
+    data = data.filter(x => x != null && x !== "" && !Array.isArray(x))
 
     // Ensure all values are packed into objects
     return data.map(x => (typeof x === "object" ? x : { value: x }))
   }
 
+  // Extracts and parses the custom data from the datasource definition
   getCustomData(datasource) {
     return this.enrichCustomData(this.parseCustomData(datasource?.data))
   }
@@ -68,6 +71,9 @@ export default class CustomFetch extends DataFetch {
     // Try and work out the schema from the array provided
     let schema = {}
     const data = this.getCustomData(datasource)
+    if (!data?.length) {
+      return { schema }
+    }
 
     // Go through every object and extract all valid keys
     for (let datum of data) {
@@ -76,14 +82,32 @@ export default class CustomFetch extends DataFetch {
           continue
         }
         if (!schema[key]) {
-          schema[key] = { type: this.getType(datum[key]) }
+          let type = this.getType(datum[key])
+          let constraints = {}
+
+          // Determine whether we should render text columns as options instead
+          if (type === "string") {
+            const uniqueValues = [...new Set(data.map(x => x[key]))]
+            const uniqueness = uniqueValues.length / data.length
+            if (uniqueness <= 0.8) {
+              type = "options"
+              constraints.inclusion = uniqueValues
+            }
+          }
+
+          // Generate options for array columns
+          if (type === "array") {
+            constraints.inclusion = [...new Set(data.map(x => x[key]).flat())]
+          }
+
+          schema[key] = {
+            type,
+            constraints,
+          }
         }
       }
     }
-
-    return {
-      schema,
-    }
+    return { schema }
   }
 
   async getData() {

From 549ea442458d64dc709c1288754cb105a8237c82 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 19:17:13 +0100
Subject: [PATCH 13/95] Improve handling of file uploads

---
 .../design/settings/controls/DataSourceSelect.svelte   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
index 887ce1090d..2dd77870a1 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
@@ -207,11 +207,13 @@
         const js = await API.csvToJson(csv)
         tmpCustomData = JSON.stringify(js)
       }
+      modal.hide()
+      saveCustomData()
     } catch (error) {
       notifications.error("Failed to parse CSV")
+      modal.hide()
+      drawer.show()
     }
-    modal.hide()
-    drawer.show()
   }
 </script>
 
@@ -262,9 +264,7 @@
           Save
         </Button>
       </div>
-      <div slot="description">
-        Provide a JavaScript or JSON array to use as data
-      </div>
+      <div slot="description">Provide a JSON array to use as data</div>
       <ClientBindingPanel
         slot="body"
         bind:valid={customDataValid}

From 3564fd56211d55b6f6b530cd5cf98d691acb9ed5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 5 Oct 2023 19:30:00 +0100
Subject: [PATCH 14/95] Improve handling of JSON objects in custom datasources

---
 packages/frontend-core/src/fetch/CustomFetch.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index 5a92de635a..5b367ec158 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -36,9 +36,7 @@ export default class CustomFetch extends DataFetch {
     if (typeof data === "string") {
       try {
         const js = JSON.parse(data)
-        if (Array.isArray(js)) {
-          return js
-        }
+        return Array.isArray(js) ? js : [js]
       } catch (error) {
         // Ignore
       }

From c2acac36a8827709ee9b449a33a44f690ed8cc18 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 6 Oct 2023 12:02:30 +0100
Subject: [PATCH 15/95] Improve retention of grid metadata when updating
 datasources to avoid resetting state

---
 .../src/components/grid/stores/columns.js     | 44 ++++++++++++-------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index c36afea7a1..31638e7c79 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -152,14 +152,20 @@ export const initialise = context => {
       return
     }
     const $definition = get(definition)
+    const $columns = get(columns)
+    const $stickyColumn = get(stickyColumn)
+
+    // Generate array of all columns to easily find pre-existing columns
+    let allColumns = $columns || []
+    if ($stickyColumn) {
+      allColumns.push($stickyColumn)
+    }
 
     // Find primary display
     let primaryDisplay
-    if (
-      $definition.primaryDisplay &&
-      $enrichedSchema[$definition.primaryDisplay]
-    ) {
-      primaryDisplay = $definition.primaryDisplay
+    const candidatePD = $definition.primaryDisplay || $stickyColumn?.name
+    if (candidatePD && $enrichedSchema[candidatePD]) {
+      primaryDisplay = candidatePD
     }
 
     // Get field list
@@ -173,14 +179,18 @@ export const initialise = context => {
     // Update columns, removing extraneous columns and adding missing ones
     columns.set(
       fields
-        .map(field => ({
-          name: field,
-          label: $enrichedSchema[field].displayName || field,
-          schema: $enrichedSchema[field],
-          width: $enrichedSchema[field].width || DefaultColumnWidth,
-          visible: $enrichedSchema[field].visible ?? true,
-          order: $enrichedSchema[field].order,
-        }))
+        .map(field => {
+          const fieldSchema = $enrichedSchema[field]
+          const oldColumn = allColumns?.find(x => x.name === field)
+          return {
+            name: field,
+            label: fieldSchema.displayName || field,
+            schema: fieldSchema,
+            width: fieldSchema.width || oldColumn?.width || DefaultColumnWidth,
+            visible: fieldSchema.visible ?? true,
+            order: fieldSchema.order ?? oldColumn?.order,
+          }
+        })
         .sort((a, b) => {
           // Sort by order first
           const orderA = a.order
@@ -208,11 +218,13 @@ export const initialise = context => {
       stickyColumn.set(null)
       return
     }
+    const stickySchema = $enrichedSchema[primaryDisplay]
+    const oldStickyColumn = allColumns?.find(x => x.name === primaryDisplay)
     stickyColumn.set({
       name: primaryDisplay,
-      label: $enrichedSchema[primaryDisplay].displayName || primaryDisplay,
-      schema: $enrichedSchema[primaryDisplay],
-      width: $enrichedSchema[primaryDisplay].width || DefaultColumnWidth,
+      label: stickySchema.displayName || primaryDisplay,
+      schema: stickySchema,
+      width: stickySchema.width || oldStickyColumn?.width || DefaultColumnWidth,
       visible: true,
       order: 0,
       left: GutterWidth,

From f3234f6bd63e0bf5307ffc752c9992bfc4813b0d Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Fri, 6 Oct 2023 16:22:56 +0100
Subject: [PATCH 16/95] Update license activate test

I have removed the end of the test which was to 'Verify user downgraded to free license'
- This is not needed

I have also updated getLicenseKey - specifically how it handles the expected 200 response
---
 qa-core/src/account-api/api/apis/LicenseAPI.ts             | 2 +-
 .../account-api/tests/licensing/license.activate.spec.ts   | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts
index dba1a661d4..b371f00f05 100644
--- a/qa-core/src/account-api/api/apis/LicenseAPI.ts
+++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts
@@ -68,7 +68,7 @@ export default class LicenseAPI extends BaseAPI {
     opts: { status?: number } = {}
   ): Promise<[Response, GetLicenseKeyResponse]> {
     const [response, json] = await this.client.get(`/api/license/key`)
-    expect(response.status).toBe(opts.status ? opts.status : 200)
+    expect(response.status).toBe(opts.status || 200)
     return [response, json]
   }
   async activateLicense(
diff --git a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
index a494ceb354..96c6eaea2a 100644
--- a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
+++ b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts
@@ -64,12 +64,5 @@ describe("license activation", () => {
 
     // Verify license key not found
     await config.internalApi.license.getLicenseKey({ status: 404 })
-
-    // Verify user downgraded to free license
-    await config.doInNewState(async () => {
-      await config.loginAsAccount(createAccountRequest)
-      const [selfRes, body] = await config.api.accounts.self()
-      expect(body.license.plan.type).toBe("free")
-    })
   })
 })

From c48eca4a0f4c1d9b4b5279353f27d771f4a7eb37 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Oct 2023 08:33:28 +0100
Subject: [PATCH 17/95] Update if logic to avoid redundant check

---
 packages/frontend-core/src/fetch/CustomFetch.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index 5b367ec158..09504b08f8 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -94,7 +94,7 @@ export default class CustomFetch extends DataFetch {
           }
 
           // Generate options for array columns
-          if (type === "array") {
+          else if (type === "array") {
             constraints.inclusion = [...new Set(data.map(x => x[key]).flat())]
           }
 

From a781860b1cab22bb55d55e34a89d470463125b0a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Oct 2023 09:36:01 +0100
Subject: [PATCH 18/95] Improve grid handling of non datasource plus and fix
 focused cell issue with row click actions

---
 .../src/components/grid/layout/GridRow.svelte |  3 +-
 .../grid/layout/StickyColumn.svelte           |  2 +-
 .../grid/overlays/MenuOverlay.svelte          |  5 +--
 .../src/components/grid/stores/datasource.js  |  7 ++-
 .../src/components/grid/stores/rows.js        | 45 +++++++++++++------
 5 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/layout/GridRow.svelte b/packages/frontend-core/src/components/grid/layout/GridRow.svelte
index bd65d34498..4754d493bf 100644
--- a/packages/frontend-core/src/components/grid/layout/GridRow.svelte
+++ b/packages/frontend-core/src/components/grid/layout/GridRow.svelte
@@ -18,6 +18,7 @@
     contentLines,
     isDragging,
     dispatch,
+    rows,
   } = getContext("grid")
 
   $: rowSelected = !!$selectedRows[row._id]
@@ -31,7 +32,7 @@
   on:focus
   on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
   on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-  on:click={() => dispatch("rowclick", row)}
+  on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))}
 >
   {#each $renderedColumns as column, columnIdx (column.name)}
     {@const cellId = `${row._id}-${column.name}`}
diff --git a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte
index 82b29f1535..70da496042 100644
--- a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte
+++ b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte
@@ -74,7 +74,7 @@
           class="row"
           on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
           on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-          on:click={() => dispatch("rowclick", row)}
+          on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))}
         >
           <GutterCell {row} {rowFocused} {rowHovered} {rowSelected} />
           {#if $stickyColumn}
diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
index 7fb2bb138d..55308d4c6e 100644
--- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
@@ -17,6 +17,7 @@
     focusedCellAPI,
     focusedRowId,
     notifications,
+    isDatasourcePlus,
   } = getContext("grid")
 
   $: style = makeStyle($menu)
@@ -75,9 +76,7 @@
       </MenuItem>
       <MenuItem
         icon="Copy"
-        disabled={isNewRow ||
-          !$focusedRow?._id ||
-          $focusedRow?._id?.startsWith("fake-")}
+        disabled={isNewRow || !$focusedRow?._id || !$isDatasourcePlus}
         on:click={() => copyToClipboard($focusedRow?._id)}
         on:click={menu.actions.close}
       >
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 7f69d0031d..fd553a049b 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -10,7 +10,7 @@ export const createStores = () => {
 }
 
 export const deriveStores = context => {
-  const { definition, schemaOverrides, columnWhitelist } = context
+  const { definition, schemaOverrides, columnWhitelist, datasource } = context
 
   const schema = derived(definition, $definition => {
     let schema = $definition?.schema
@@ -60,9 +60,14 @@ export const deriveStores = context => {
     }
   )
 
+  const isDatasourcePlus = derived(datasource, $datasource => {
+    return ["table", "viewV2"].includes($datasource?.type)
+  })
+
   return {
     schema,
     enrichedSchema,
+    isDatasourcePlus,
   }
 }
 
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 667f0d323e..49adb62936 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store"
 import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import { tick } from "svelte"
+import { Helpers } from "@budibase/bbui"
 
 export const createStores = () => {
   const rows = writable([])
@@ -76,11 +77,11 @@ export const createActions = context => {
     columns,
     rowChangeCache,
     inProgressChanges,
-    previousFocusedRowId,
     hasNextPage,
     error,
     notifications,
     fetch,
+    isDatasourcePlus,
   } = context
   const instanceLoaded = writable(false)
 
@@ -361,7 +362,7 @@ export const createActions = context => {
 
       // Update row
       const saved = await datasource.actions.updateRow({
-        ...row,
+        ...cleanRow(row),
         ...get(rowChangeCache)[rowId],
       })
 
@@ -417,13 +418,15 @@ export const createActions = context => {
     }
     let rowsToAppend = []
     let newRow
+    const $isDatasourcePlus = get(isDatasourcePlus)
     for (let i = 0; i < newRows.length; i++) {
       newRow = newRows[i]
 
       // Ensure we have a unique _id.
-      // This means generating one for non DS+.
-      if (!newRow._id) {
-        newRow._id = `fake-${Math.random()}`
+      // This means generating one for non DS+, overriting any that may already
+      // exist as we cannot allow duplicates.
+      if (!$isDatasourcePlus) {
+        newRow._id = Helpers.uuid()
       }
 
       if (!rowCacheMap[newRow._id]) {
@@ -462,15 +465,16 @@ export const createActions = context => {
     return get(rowLookupMap)[id] != null
   }
 
-  // Wipe the row change cache when changing row
-  previousFocusedRowId.subscribe(id => {
-    if (id && !get(inProgressChanges)[id]) {
-      rowChangeCache.update(state => {
-        delete state[id]
-        return state
-      })
+  // Cleans a row by removing any internal grid metadata from it.
+  // Call this before passing a row to any sort of external flow.
+  const cleanRow = row => {
+    let clone = { ...row }
+    delete clone.__idx
+    if (!get(isDatasourcePlus)) {
+      delete clone._id
     }
-  })
+    return clone
+  }
 
   return {
     rows: {
@@ -487,7 +491,22 @@ export const createActions = context => {
         refreshRow,
         replaceRow,
         refreshData,
+        cleanRow,
       },
     },
   }
 }
+
+export const initialise = context => {
+  const { rowChangeCache, inProgressChanges, previousFocusedRowId } = context
+
+  // Wipe the row change cache when changing row
+  previousFocusedRowId.subscribe(id => {
+    if (id && !get(inProgressChanges)[id]) {
+      rowChangeCache.update(state => {
+        delete state[id]
+        return state
+      })
+    }
+  })
+}

From 513aaf1174d1301cc512be2244705db35c68265a Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 9 Oct 2023 16:03:12 +0100
Subject: [PATCH 19/95] Update custom datasource label to 'JSON / CSV'

---
 .../components/design/settings/controls/DataSourceSelect.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
index 2dd77870a1..29f32aa345 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
@@ -142,7 +142,7 @@
     })
   $: custom = {
     type: "custom",
-    label: "Custom",
+    label: "JSON / CSV",
   }
 
   const handleSelected = selected => {

From 08331c3632276250bb51eb9a71b4428861875343 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Oct 2023 11:59:24 +0100
Subject: [PATCH 20/95] Fix bug that was causing an empty grid when adding a
 new column

---
 .../frontend-core/src/components/grid/stores/datasource.js   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index fd553a049b..0b62194f73 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -102,7 +102,10 @@ export const createActions = context => {
 
   // Refreshes the datasource definition
   const refreshDefinition = async () => {
-    const def = await getDatasourceDefinition({ API, datasource })
+    const def = await getDatasourceDefinition({
+      API,
+      datasource: get(datasource),
+    })
     definition.set(def)
   }
 

From a31186ee56639340a7a7a8c9f55920e02cbbdf63 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 10 Oct 2023 15:42:09 +0100
Subject: [PATCH 21/95] Improve custom data fetch parsing to better support
 csv-like strings

---
 .../frontend-core/src/fetch/CustomFetch.js    | 30 +++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js
index 09504b08f8..fc62d790e2 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.js
+++ b/packages/frontend-core/src/fetch/CustomFetch.js
@@ -40,6 +40,13 @@ export default class CustomFetch extends DataFetch {
       } catch (error) {
         // Ignore
       }
+
+      // Try splitting by newlines first
+      if (data.includes("\n")) {
+        return data.split("\n").map(x => x.trim())
+      }
+
+      // Split by commas next
       return data.split(",").map(x => x.trim())
     }
 
@@ -57,7 +64,26 @@ export default class CustomFetch extends DataFetch {
     data = data.filter(x => x != null && x !== "" && !Array.isArray(x))
 
     // Ensure all values are packed into objects
-    return data.map(x => (typeof x === "object" ? x : { value: x }))
+    return data.map(value => {
+      if (typeof value === "object") {
+        return value
+      }
+
+      // Try parsing strings
+      if (typeof value === "string") {
+        const split = value.split(",").map(x => x.trim())
+        let obj = {}
+        for (let i = 0; i < split.length; i++) {
+          const suffix = i === 0 ? "" : ` ${i + 1}`
+          const key = `Value${suffix}`
+          obj[key] = split[i]
+        }
+        return obj
+      }
+
+      // For anything else, wrap in an object
+      return { Value: value }
+    })
   }
 
   // Extracts and parses the custom data from the datasource definition
@@ -87,7 +113,7 @@ export default class CustomFetch extends DataFetch {
           if (type === "string") {
             const uniqueValues = [...new Set(data.map(x => x[key]))]
             const uniqueness = uniqueValues.length / data.length
-            if (uniqueness <= 0.8) {
+            if (uniqueness <= 0.8 && uniqueValues.length > 1) {
               type = "options"
               constraints.inclusion = uniqueValues
             }

From aa0dc7dd37a86a47bcacab8e4379f03411f35b6d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 11 Oct 2023 10:24:49 +0100
Subject: [PATCH 22/95] Bump

---
 .../frontend-core/src/components/grid/cells/HeaderCell.svelte    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 5ac70c93c8..953d925123 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -74,6 +74,7 @@
     }
   }
 
+  // change
   const onContextMenu = e => {
     e.preventDefault()
     ui.actions.blur()

From b51950a43a8a979441f0615ec60846a8664a9554 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 12 Oct 2023 08:31:46 +0100
Subject: [PATCH 23/95] Undo bump comment change

---
 .../frontend-core/src/components/grid/cells/HeaderCell.svelte    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 953d925123..5ac70c93c8 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -74,7 +74,6 @@
     }
   }
 
-  // change
   const onContextMenu = e => {
     e.preventDefault()
     ui.actions.blur()

From 5c7bde95ab995b2645a9c13870699075ad11967f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 16 Oct 2023 16:46:49 +0100
Subject: [PATCH 24/95] Update conditional setting rendering to support nested
 paths and multiple values, and hide grid block CRUD when using non DS+

---
 .../Component/ComponentSettingsSection.svelte | 36 +++++++++----------
 packages/client/manifest.json                 | 24 ++++++++-----
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index b4ed8995a0..8eb2ebe10b 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { isEmpty } from "lodash/fp"
+  import { helpers } from "@budibase/shared-core"
   import { Input, DetailSummary, notifications } from "@budibase/bbui"
   import { store } from "builderStore"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
@@ -73,35 +73,31 @@
     // Parse dependant settings
     if (setting.dependsOn) {
       let dependantSetting = setting.dependsOn
-      let dependantValue = null
+      let dependantValues = null
       let invert = !!setting.dependsOn.invert
       if (typeof setting.dependsOn === "object") {
         dependantSetting = setting.dependsOn.setting
-        dependantValue = setting.dependsOn.value
+        dependantValues = setting.dependsOn.value
       }
       if (!dependantSetting) {
         return false
       }
 
-      // If no specific value is depended upon, check if a value exists at all
-      // for the dependent setting
-      if (dependantValue == null) {
-        const currentValue = instance[dependantSetting]
-        if (currentValue === false) {
-          return false
-        }
-        if (currentValue === true) {
-          return true
-        }
-        return !isEmpty(currentValue)
+      // Ensure values is an array
+      if (!Array.isArray(dependantValues)) {
+        dependantValues = [dependantValues]
       }
 
-      // Otherwise check the value matches
-      if (invert) {
-        return instance[dependantSetting] !== dependantValue
-      } else {
-        return instance[dependantSetting] === dependantValue
-      }
+      // If inverting, we want to ensure that we don't have any matches.
+      // If not inverting, we want to ensure that we do have any matches.
+      const currentVal = helpers.deepGet(instance, dependantSetting)
+      const anyMatches = dependantValues.some(dependantVal => {
+        if (dependantVal == null) {
+          return currentVal == null || currentVal === false || currentVal === ""
+        }
+        return dependantVal === currentVal
+      })
+      return anyMatches !== invert
     }
 
     return typeof setting.visible == "boolean" ? setting.visible : true
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 07e4997f00..fbf7ffb0b3 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5617,29 +5617,37 @@
             "label": "Clicked row",
             "key": "row"
           }
-        ],
-        "dependsOn": {
-          "setting": "allowEditRows",
-          "value": false
-        }
+        ]
       },
       {
         "type": "boolean",
         "label": "Add rows",
         "key": "allowAddRows",
-        "defaultValue": true
+        "defaultValue": true,
+        "dependsOn": {
+          "setting": "datasource.type",
+          "value": ["table", "viewV2"]
+        }
       },
       {
         "type": "boolean",
         "label": "Edit rows",
         "key": "allowEditRows",
-        "defaultValue": true
+        "defaultValue": true,
+        "dependsOn": {
+          "setting": "datasource.type",
+          "value": ["table", "viewV2"]
+        }
       },
       {
         "type": "boolean",
         "label": "Delete rows",
         "key": "allowDeleteRows",
-        "defaultValue": true
+        "defaultValue": true,
+        "dependsOn": {
+          "setting": "datasource.type",
+          "value": ["table", "viewV2"]
+        }
       },
       {
         "type": "boolean",

From a06451222400312d9928e67618e870cc28f2dbf7 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Wed, 18 Oct 2023 11:14:16 +0100
Subject: [PATCH 25/95] Added description field to the formblock

---
 packages/client/manifest.json                 |   6 ++
 .../app/blocks/form/FormBlock.svelte          |   2 +
 .../app/blocks/form/InnerFormBlock.svelte     | 101 ++++++++++--------
 3 files changed, 67 insertions(+), 42 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 8d0a4e456f..ed69b8868f 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5305,6 +5305,12 @@
         "key": "title",
         "nested": true
       },
+      {
+        "type": "text",
+        "label": "Description",
+        "key": "description",
+        "nested": true
+      },
       {
         "section": true,
         "dependsOn": {
diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte
index 5d57d10ab6..f905227af9 100644
--- a/packages/client/src/components/app/blocks/form/FormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte
@@ -12,6 +12,7 @@
   export let fields
   export let labelPosition
   export let title
+  export let description
   export let showDeleteButton
   export let showSaveButton
   export let saveButtonLabel
@@ -98,6 +99,7 @@
     fields: fieldsOrDefault,
     labelPosition,
     title,
+    description,
     saveButtonLabel: saveLabel,
     deleteButtonLabel: deleteLabel,
     schema,
diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
index ec5daa21b1..e65d2cf90b 100644
--- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
+++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte
@@ -11,6 +11,7 @@
   export let fields
   export let labelPosition
   export let title
+  export let description
   export let saveButtonLabel
   export let deleteButtonLabel
   export let schema
@@ -160,55 +161,71 @@
         <BlockComponent
           type="container"
           props={{
-            direction: "row",
-            hAlign: "stretch",
-            vAlign: "center",
-            gap: "M",
-            wrap: true,
+            direction: "column",
+            gap: "S",
           }}
           order={0}
         >
           <BlockComponent
-            type="heading"
-            props={{ text: title || "" }}
+            type="container"
+            props={{
+              direction: "row",
+              hAlign: "stretch",
+              vAlign: "center",
+              gap: "M",
+              wrap: true,
+            }}
             order={0}
-          />
-          {#if renderButtons}
+          >
             <BlockComponent
-              type="container"
-              props={{
-                direction: "row",
-                hAlign: "stretch",
-                vAlign: "center",
-                gap: "M",
-                wrap: true,
-              }}
+              type="heading"
+              props={{ text: title || "" }}
+              order={0}
+            />
+            {#if renderButtons}
+              <BlockComponent
+                type="container"
+                props={{
+                  direction: "row",
+                  hAlign: "stretch",
+                  vAlign: "center",
+                  gap: "M",
+                  wrap: true,
+                }}
+                order={1}
+              >
+                {#if renderDeleteButton}
+                  <BlockComponent
+                    type="button"
+                    props={{
+                      text: deleteButtonLabel,
+                      onClick: onDelete,
+                      quiet: true,
+                      type: "secondary",
+                    }}
+                    order={0}
+                  />
+                {/if}
+                {#if renderSaveButton}
+                  <BlockComponent
+                    type="button"
+                    props={{
+                      text: saveButtonLabel,
+                      onClick: onSave,
+                      type: "cta",
+                    }}
+                    order={1}
+                  />
+                {/if}
+              </BlockComponent>
+            {/if}
+          </BlockComponent>
+          {#if description}
+            <BlockComponent
+              type="text"
+              props={{ text: description }}
               order={1}
-            >
-              {#if renderDeleteButton}
-                <BlockComponent
-                  type="button"
-                  props={{
-                    text: deleteButtonLabel,
-                    onClick: onDelete,
-                    quiet: true,
-                    type: "secondary",
-                  }}
-                  order={0}
-                />
-              {/if}
-              {#if renderSaveButton}
-                <BlockComponent
-                  type="button"
-                  props={{
-                    text: saveButtonLabel,
-                    onClick: onSave,
-                    type: "cta",
-                  }}
-                  order={1}
-                />
-              {/if}
-            </BlockComponent>
+            />
           {/if}
         </BlockComponent>
       {/if}

From faa70b79b639d8c950d357f297d32b37a4a3ca02 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Oct 2023 14:21:09 +0100
Subject: [PATCH 26/95] Fix on click actions not working when using non DS+
 datasources and edit row setting is hidden but enabled

---
 packages/client/src/components/app/GridBlock.svelte | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index c1c71a1539..b3f7e826cd 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -24,7 +24,6 @@
 
   $: columnWhitelist = columns?.map(col => col.name)
   $: schemaOverrides = getSchemaOverrides(columns)
-  $: handleRowClick = allowEditRows ? undefined : onRowClick
 
   const getSchemaOverrides = columns => {
     let overrides = {}
@@ -61,7 +60,7 @@
     showControls={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
-    on:rowclick={e => handleRowClick?.({ row: e.detail })}
+    on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>
 

From 93d717d2e97a68c74b2777dfe44049f44ab46464 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Oct 2023 14:25:59 +0100
Subject: [PATCH 27/95] Fix add new row setting not being enough to edit new
 rows with also enabling edit rows flag

---
 .../frontend-core/src/components/grid/layout/NewRow.svelte    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte
index aa9b6fa051..88fe004455 100644
--- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte
+++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte
@@ -33,7 +33,7 @@
 
   let visible = false
   let isAdding = false
-  let newRow = { _isNewRow: true }
+  let newRow
   let offset = 0
 
   $: firstColumn = $stickyColumn || $renderedColumns[0]
@@ -111,7 +111,7 @@
     }
 
     // Update state and select initial cell
-    newRow = {}
+    newRow = { _isNewRow: true }
     visible = true
     $hoveredRowId = NewRowID
     if (firstColumn) {

From bdc2bcd97d4ab6e74b1b95a6473235a40e12e3c7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Oct 2023 14:41:42 +0100
Subject: [PATCH 28/95] Add support for multiple dependsOn constraints in the
 manifest and fix issue with nullish dependsOn contraints

---
 .../Component/ComponentSettingsSection.svelte | 28 +++++++++++--------
 packages/client/manifest.json                 | 23 +++++++++++++--
 2 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
index 8eb2ebe10b..639cef332e 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
@@ -70,14 +70,22 @@
   }
 
   const shouldDisplay = (instance, setting) => {
-    // Parse dependant settings
-    if (setting.dependsOn) {
-      let dependantSetting = setting.dependsOn
+    let dependsOn = setting.dependsOn
+    if (dependsOn && !Array.isArray(dependsOn)) {
+      dependsOn = [dependsOn]
+    }
+    if (!dependsOn?.length) {
+      return true
+    }
+
+    // Ensure all conditions are met
+    return dependsOn.every(condition => {
+      let dependantSetting = condition
       let dependantValues = null
-      let invert = !!setting.dependsOn.invert
-      if (typeof setting.dependsOn === "object") {
-        dependantSetting = setting.dependsOn.setting
-        dependantValues = setting.dependsOn.value
+      let invert = !!condition.invert
+      if (typeof condition === "object") {
+        dependantSetting = condition.setting
+        dependantValues = condition.value
       }
       if (!dependantSetting) {
         return false
@@ -93,14 +101,12 @@
       const currentVal = helpers.deepGet(instance, dependantSetting)
       const anyMatches = dependantValues.some(dependantVal => {
         if (dependantVal == null) {
-          return currentVal == null || currentVal === false || currentVal === ""
+          return currentVal != null && currentVal !== false && currentVal !== ""
         }
         return dependantVal === currentVal
       })
       return anyMatches !== invert
-    }
-
-    return typeof setting.visible == "boolean" ? setting.visible : true
+    })
   }
 
   const canRenderControl = (instance, setting, isScreen) => {
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index fbf7ffb0b3..ff0fbfac93 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5567,18 +5567,35 @@
         "type": "columns/grid",
         "label": "Columns",
         "key": "columns",
-        "dependsOn": "table"
+        "dependsOn": [
+          "datasource",
+          {
+            "setting": "datasource.type",
+            "value": "custom",
+            "invert": true
+          }
+        ]
       },
       {
         "type": "filter",
         "label": "Filtering",
-        "key": "initialFilter"
+        "key": "initialFilter",
+        "dependsOn": {
+          "setting": "datasource.type",
+          "value": "custom",
+          "invert": true
+        }
       },
       {
         "type": "field/sortable",
         "label": "Sort column",
         "key": "initialSortColumn",
-        "placeholder": "Default"
+        "placeholder": "Default",
+        "dependsOn": {
+          "setting": "datasource.type",
+          "value": "custom",
+          "invert": true
+        }
       },
       {
         "type": "select",

From 705c948de73bab6766ba5a9d066ccbea4b39c0d4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 18 Oct 2023 14:51:49 +0100
Subject: [PATCH 29/95] Removing a sudo that was causing the docker image build
 to fail with buildx - we don't need sudo commands as the Dockerfile is always
 running as the root.

---
 hosting/couchdb/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile
index ce77002052..792856cac7 100644
--- a/hosting/couchdb/Dockerfile
+++ b/hosting/couchdb/Dockerfile
@@ -5,7 +5,7 @@ ENV COUCHDB_PASSWORD admin
 EXPOSE 5984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
-    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo apt-key add - && \
+    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
     apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
     apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
     apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \

From fb69c3a0da0805e8ecbe310b5aa75f7f6e12d0c4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Oct 2023 15:38:37 +0100
Subject: [PATCH 30/95] Add new shared core util for determining whether
 columns can be sort columns or not

---
 .../components/backend/DataTable/Table.svelte | 17 +++++--------
 .../controls/SortableFieldSelect.svelte       |  4 ++--
 packages/builder/src/constants/index.js       |  2 --
 .../src/components/app/table/Table.svelte     |  4 ++--
 packages/client/src/constants.js              |  8 -------
 .../components/grid/cells/HeaderCell.svelte   |  8 ++++---
 packages/shared-core/src/table.ts             | 24 +++++++++++++++++++
 7 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte
index 4569586762..f8087d8a39 100644
--- a/packages/builder/src/components/backend/DataTable/Table.svelte
+++ b/packages/builder/src/components/backend/DataTable/Table.svelte
@@ -3,13 +3,10 @@
   import { goto, params } from "@roxi/routify"
   import { Table, Heading, Layout } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
-  import {
-    TableNames,
-    UNEDITABLE_USER_FIELDS,
-    UNSORTABLE_TYPES,
-  } from "constants"
+  import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import RoleCell from "./cells/RoleCell.svelte"
   import { createEventDispatcher } from "svelte"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   export let schema = {}
   export let data = []
@@ -32,12 +29,10 @@
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
   $: {
-    UNSORTABLE_TYPES.forEach(type => {
-      Object.values(schema || {}).forEach(col => {
-        if (col.type === type) {
-          col.sortable = false
-        }
-      })
+    Object.values(schema || {}).forEach(col => {
+      if (!canBeSortColumn(col.type)) {
+        col.sortable = false
+      }
     })
   }
   $: {
diff --git a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
index 21ed68ce68..350f59f456 100644
--- a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
@@ -6,7 +6,7 @@
   } from "builderStore/dataBinding"
   import { currentAsset } from "builderStore"
   import { createEventDispatcher } from "svelte"
-  import { UNSORTABLE_TYPES } from "constants"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   export let componentInstance = {}
   export let value = ""
@@ -20,7 +20,7 @@
 
   const getSortableFields = schema => {
     return Object.entries(schema || {})
-      .filter(entry => !UNSORTABLE_TYPES.includes(entry[1].type))
+      .filter(entry => canBeSortColumn(entry[1].type))
       .map(entry => entry[0])
   }
 
diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.js
index 66fd926a77..f556ee4b05 100644
--- a/packages/builder/src/constants/index.js
+++ b/packages/builder/src/constants/index.js
@@ -34,8 +34,6 @@ export const UNEDITABLE_USER_FIELDS = [
   "lastName",
 ]
 
-export const UNSORTABLE_TYPES = ["formula", "attachment", "array", "link"]
-
 export const LAYOUT_NAMES = {
   MASTER: {
     PRIVATE: "layout_private_master",
diff --git a/packages/client/src/components/app/table/Table.svelte b/packages/client/src/components/app/table/Table.svelte
index 0ed76317db..8ef25846f8 100644
--- a/packages/client/src/components/app/table/Table.svelte
+++ b/packages/client/src/components/app/table/Table.svelte
@@ -2,8 +2,8 @@
   import { getContext } from "svelte"
   import { Table } from "@budibase/bbui"
   import SlotRenderer from "./SlotRenderer.svelte"
-  import { UnsortableTypes } from "../../../constants"
   import { onDestroy } from "svelte"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   export let dataProvider
   export let columns
@@ -102,7 +102,7 @@
         return
       }
       newSchema[columnName] = schema[columnName]
-      if (UnsortableTypes.includes(schema[columnName].type)) {
+      if (!canBeSortColumn(schema[columnName].type)) {
         newSchema[columnName].sortable = false
       }
 
diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js
index f0a89b9cfd..a4d7411bbd 100644
--- a/packages/client/src/constants.js
+++ b/packages/client/src/constants.js
@@ -1,13 +1,5 @@
-import { FieldType as FieldTypes } from "@budibase/types"
 export { FieldType as FieldTypes } from "@budibase/types"
 
-export const UnsortableTypes = [
-  FieldTypes.FORMULA,
-  FieldTypes.ATTACHMENT,
-  FieldTypes.ARRAY,
-  FieldTypes.LINK,
-]
-
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   UpdateFieldValue: "UpdateFieldValue",
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 8600656520..d6cbcb582d 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -1,6 +1,6 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
-  import { canBeDisplayColumn } from "@budibase/shared-core"
+  import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
   import { Icon, Popover, Menu, MenuItem, clickOutside } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
@@ -232,14 +232,16 @@
       <MenuItem
         icon="SortOrderUp"
         on:click={sortAscending}
-        disabled={column.name === $sort.column && $sort.order === "ascending"}
+        disabled={!canBeSortColumn(column.schema.type) ||
+          (column.name === $sort.column && $sort.order === "ascending")}
       >
         Sort {ascendingLabel}
       </MenuItem>
       <MenuItem
         icon="SortOrderDown"
         on:click={sortDescending}
-        disabled={column.name === $sort.column && $sort.order === "descending"}
+        disabled={!canBeSortColumn(column.schema.type) ||
+          (column.name === $sort.column && $sort.order === "descending")}
       >
         Sort {descendingLabel}
       </MenuItem>
diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts
index ffdd9bdd11..5eab2fc340 100644
--- a/packages/shared-core/src/table.ts
+++ b/packages/shared-core/src/table.ts
@@ -20,6 +20,30 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.BB_REFERENCE]: false,
 }
 
+const allowSortColumnByType: Record<FieldType, boolean> = {
+  [FieldType.STRING]: true,
+  [FieldType.LONGFORM]: true,
+  [FieldType.OPTIONS]: true,
+  [FieldType.NUMBER]: true,
+  [FieldType.DATETIME]: true,
+  [FieldType.AUTO]: true,
+  [FieldType.INTERNAL]: true,
+  [FieldType.BARCODEQR]: true,
+  [FieldType.BIGINT]: true,
+  [FieldType.BOOLEAN]: true,
+  [FieldType.JSON]: true,
+
+  [FieldType.FORMULA]: false,
+  [FieldType.ATTACHMENT]: false,
+  [FieldType.ARRAY]: false,
+  [FieldType.LINK]: false,
+  [FieldType.BB_REFERENCE]: false,
+}
+
 export function canBeDisplayColumn(type: FieldType): boolean {
   return !!allowDisplayColumnByType[type]
 }
+
+export function canBeSortColumn(type: FieldType): boolean {
+  return !!allowSortColumnByType[type]
+}

From 05133d7e69df5aad2ac85f413dff8cd8f43e3e9f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Oct 2023 15:44:27 +0100
Subject: [PATCH 31/95] Ensure data section grid sort column options are
 properly filtered down

---
 .../src/components/grid/controls/SortButton.svelte            | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/controls/SortButton.svelte b/packages/frontend-core/src/components/grid/controls/SortButton.svelte
index 60e3c8e514..339ed32293 100644
--- a/packages/frontend-core/src/components/grid/controls/SortButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/SortButton.svelte
@@ -1,6 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { ActionButton, Popover, Select } from "@budibase/bbui"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   const { sort, columns, stickyColumn } = getContext("grid")
 
@@ -19,7 +20,7 @@
         type: stickyColumn.schema?.type,
       })
     }
-    return [
+    options = [
       ...options,
       ...columns.map(col => ({
         label: col.label || col.name,
@@ -27,6 +28,7 @@
         type: col.schema?.type,
       })),
     ]
+    return options.filter(col => canBeSortColumn(col.type))
   }
 
   const getOrderOptions = (column, columnOptions) => {

From d0b71ada08ec6695559f3687647d342d480e7a2c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 09:08:45 +0100
Subject: [PATCH 32/95] Ensure BBUI table component ignores numeric widths that
 are invalid

---
 packages/bbui/src/Table/Table.svelte | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte
index 529d1144ee..885dd75671 100644
--- a/packages/bbui/src/Table/Table.svelte
+++ b/packages/bbui/src/Table/Table.svelte
@@ -106,6 +106,13 @@
           name: fieldName,
         }
       }
+
+      // Delete numeric only widths as these are grid widths and should be
+      // ignored
+      let width = fixedSchema[fieldName].width
+      if (width != null && `${width}`.trim().match(/^[0-9]+$/)) {
+        delete fixedSchema[fieldName].width
+      }
     })
     return fixedSchema
   }

From ff257abab3dd1d747bc309075e13d2d993fc1481 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 09:09:32 +0100
Subject: [PATCH 33/95] Update copy for table column widths to inform user that
 a unit is required

---
 .../design/settings/controls/ColumnEditor/CellDrawer.svelte | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/ColumnEditor/CellDrawer.svelte b/packages/builder/src/components/design/settings/controls/ColumnEditor/CellDrawer.svelte
index 8e3079101a..9e53f7f1cf 100644
--- a/packages/builder/src/components/design/settings/controls/ColumnEditor/CellDrawer.svelte
+++ b/packages/builder/src/components/design/settings/controls/ColumnEditor/CellDrawer.svelte
@@ -16,7 +16,11 @@
 <DrawerContent>
   <div class="container">
     <Layout noPadding gap="S">
-      <Input bind:value={column.width} label="Width" placeholder="Auto" />
+      <Input
+        bind:value={column.width}
+        label="Width (must include a unit like px or %)"
+        placeholder="Auto"
+      />
       <Select
         label="Alignment"
         bind:value={column.align}

From 880c509f649dec592a53a8b2a5b0194f04d105b3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 09:13:42 +0100
Subject: [PATCH 34/95] Always use custom table columns if provided to ensure
 we allow showing hidden columns if requested. Use grid visibility and order
 metadata by default in tables

---
 packages/client/src/components/app/DataProvider.svelte | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte
index 2639c77edd..342194d5e2 100644
--- a/packages/client/src/components/app/DataProvider.svelte
+++ b/packages/client/src/components/app/DataProvider.svelte
@@ -81,6 +81,7 @@
       sortOrder: $fetch.sortOrder,
     },
     limit,
+    primaryDisplay: $fetch.definition?.primaryDisplay,
   }
 
   const createFetch = datasource => {

From 357f495c0d129344b5e1aaeb4190d65f229e35d7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 09:14:15 +0100
Subject: [PATCH 35/95] Respect grid visibility and ordering by default when
 rendering tables

---
 .../src/components/app/table/Table.svelte     | 49 +++++++++++++------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/packages/client/src/components/app/table/Table.svelte b/packages/client/src/components/app/table/Table.svelte
index 0ed76317db..34fa8c204d 100644
--- a/packages/client/src/components/app/table/Table.svelte
+++ b/packages/client/src/components/app/table/Table.svelte
@@ -32,7 +32,8 @@
   $: loading = dataProvider?.loading ?? false
   $: data = dataProvider?.rows || []
   $: fullSchema = dataProvider?.schema ?? {}
-  $: fields = getFields(fullSchema, columns, false)
+  $: primaryDisplay = dataProvider?.primaryDisplay
+  $: fields = getFields(fullSchema, columns, false, primaryDisplay)
   $: schema = getFilteredSchema(fullSchema, fields, hasChildren)
   $: setSorting = getAction(
     dataProvider?.id,
@@ -55,18 +56,13 @@
     }
   }
 
-  const getFields = (schema, customColumns, showAutoColumns) => {
-    // Check for an invalid column selection
-    let invalid = false
-    customColumns?.forEach(column => {
-      const columnName = typeof column === "string" ? column : column.name
-      if (schema[columnName] == null) {
-        invalid = true
-      }
-    })
-
-    // Use column selection if it exists
-    if (!invalid && customColumns?.length) {
+  const getFields = (
+    schema,
+    customColumns,
+    showAutoColumns,
+    primaryDisplay
+  ) => {
+    if (customColumns?.length) {
       return customColumns
     }
 
@@ -74,13 +70,38 @@
     let columns = []
     let autoColumns = []
     Object.entries(schema).forEach(([field, fieldSchema]) => {
+      if (!fieldSchema.visible) {
+        return
+      }
       if (!fieldSchema?.autocolumn) {
         columns.push(field)
       } else if (showAutoColumns) {
         autoColumns.push(field)
       }
     })
-    return columns.concat(autoColumns)
+
+    // Sort columns to respect grid metadata
+    const allCols = columns.concat(autoColumns)
+    return allCols.sort((a, b) => {
+      if (a === primaryDisplay) {
+        return -1
+      }
+      if (b === primaryDisplay) {
+        return 1
+      }
+      const aOrder = schema[a].order
+      const bOrder = schema[b].order
+      if (aOrder === bOrder) {
+        return 0
+      }
+      if (aOrder == null) {
+        return 1
+      }
+      if (bOrder == null) {
+        return -1
+      }
+      return aOrder < bOrder ? -1 : 1
+    })
   }
 
   const getFilteredSchema = (schema, fields, hasChildren) => {

From 8d2530d4c859963c9f36d6b31534e1c992d269f1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 09:19:05 +0100
Subject: [PATCH 36/95] Swap let for const

---
 packages/bbui/src/Table/Table.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte
index 885dd75671..2610d6106c 100644
--- a/packages/bbui/src/Table/Table.svelte
+++ b/packages/bbui/src/Table/Table.svelte
@@ -109,7 +109,7 @@
 
       // Delete numeric only widths as these are grid widths and should be
       // ignored
-      let width = fixedSchema[fieldName].width
+      const width = fixedSchema[fieldName].width
       if (width != null && `${width}`.trim().match(/^[0-9]+$/)) {
         delete fixedSchema[fieldName].width
       }

From c462b072243c4fb8fc831773ce68c622ba4012a4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 10:34:38 +0200
Subject: [PATCH 37/95] Create deploy qa workflow

---
 .github/workflows/deploy-qa.yml | 35 +++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 .github/workflows/deploy-qa.yml

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
new file mode 100644
index 0000000000..9fa8a0cd3f
--- /dev/null
+++ b/.github/workflows/deploy-qa.yml
@@ -0,0 +1,35 @@
+name: Deploy QA
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+env:
+  BRANCH: ${{ github.event.pull_request.head.ref }}
+  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  NX_BASE_BRANCH: origin/${{ github.base_ref }}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+
+jobs:
+  trigger-deploy-to-qa-env:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get the current budibase release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_VERSION: ${{ github.sha }}
+          REF_NAME: ${{ github.ref_name}}
+        with:
+          repository: budibase/budibase-deploys
+          event: budicloud-qa-deploy
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

From ec76388de4a1093bdd9a00e64b280cac491c937a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 10:37:22 +0200
Subject: [PATCH 38/95] Use event v2

---
 .github/workflows/deploy-qa.yml | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index 9fa8a0cd3f..97a662d2f8 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -6,30 +6,15 @@ on:
       - master
   workflow_dispatch:
 
-env:
-  BRANCH: ${{ github.event.pull_request.head.ref }}
-  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  NX_BASE_BRANCH: origin/${{ github.base_ref }}
-  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
 jobs:
   trigger-deploy-to-qa-env:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Get the current budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - uses: passeidireto/trigger-external-workflow-action@main
+      - uses: peter-evans/repository-dispatch@v2
         env:
           PAYLOAD_VERSION: ${{ github.sha }}
           REF_NAME: ${{ github.ref_name}}
         with:
           repository: budibase/budibase-deploys
-          event: budicloud-qa-deploy
-          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
+          event-type: budicloud-qa-deploy-v2
+          token: ${{ secrets.GH_ACCESS_TOKEN }}

From d9e9f369858fe11c92f4b52c9c07bf10488da56e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 10:38:01 +0200
Subject: [PATCH 39/95] test

---
 .github/workflows/deploy-qa.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index 97a662d2f8..3cbe366f08 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - BUDI-7641/remove-npm-from-apps-worker-docker
   workflow_dispatch:
 
 jobs:

From c5b37ae0c73c9b42ec682c8fdbb5791d3a232163 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 10:42:45 +0200
Subject: [PATCH 40/95] Push

---
 .github/workflows/deploy-qa.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index 3cbe366f08..74b06faefe 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -3,8 +3,8 @@ name: Deploy QA
 on:
   push:
     branches:
-      - master
       - BUDI-7641/remove-npm-from-apps-worker-docker
+      - master
   workflow_dispatch:
 
 jobs:

From 65fea6d680038e52c422076737cf4bdbbc36effd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 10:44:24 +0200
Subject: [PATCH 41/95] Fix branch name

---
 .github/workflows/deploy-qa.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index 74b06faefe..d4bbb1c36a 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -3,8 +3,8 @@ name: Deploy QA
 on:
   push:
     branches:
-      - BUDI-7641/remove-npm-from-apps-worker-docker
       - master
+      - BUDI-7641/push_v2_images_to_qa
   workflow_dispatch:
 
 jobs:

From 7fd7943a5aeb9a289c3f9048c8042f638d225c07 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 11:07:37 +0100
Subject: [PATCH 42/95] Remove required prop from gridblock to ensure backwards
 compatiblity

---
 packages/client/manifest.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index ff0fbfac93..dfb6a3ef44 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5560,8 +5560,7 @@
       {
         "type": "dataSource",
         "label": "Data",
-        "key": "datasource",
-        "required": true
+        "key": "datasource"
       },
       {
         "type": "columns/grid",

From fa906291395c7a40d6088bb303a1ca75cf4d78bc Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 11:14:24 +0100
Subject: [PATCH 43/95] Improve grid datasource backwards compatibility

---
 packages/client/manifest.json                       | 3 ++-
 packages/client/src/components/app/GridBlock.svelte | 8 +++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index dfb6a3ef44..9f6468318a 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5560,7 +5560,8 @@
       {
         "type": "dataSource",
         "label": "Data",
-        "key": "datasource"
+        "key": "table",
+        "required": true
       },
       {
         "type": "columns/grid",
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index b3f7e826cd..8febaf44ff 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -4,7 +4,8 @@
   import { getContext } from "svelte"
   import { Grid } from "@budibase/frontend-core"
 
-  export let datasource
+  // table is actually any datasource, but called table for legacy compatibility
+  export let table
   export let allowAddRows = true
   export let allowEditRows = true
   export let allowDeleteRows = true
@@ -16,9 +17,6 @@
   export let columns = null
   export let onRowClick = null
 
-  // Legacy settings
-  export let table
-
   const component = getContext("component")
   const { styleable, API, builderStore, notificationStore } = getContext("sdk")
 
@@ -42,7 +40,7 @@
   class:in-builder={$builderStore.inBuilder}
 >
   <Grid
-    datasource={datasource || table}
+    datasource={table}
     {API}
     {stripeRows}
     {initialFilter}

From db286d431ac330517c93e5994bac3a52e37b8c07 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 20 Oct 2023 11:20:14 +0100
Subject: [PATCH 44/95] Fix dependsOn logic for gridblock after renaming
 setting key

---
 packages/client/manifest.json | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 9f6468318a..ae97cd30c6 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5568,9 +5568,9 @@
         "label": "Columns",
         "key": "columns",
         "dependsOn": [
-          "datasource",
+          "table",
           {
-            "setting": "datasource.type",
+            "setting": "table.type",
             "value": "custom",
             "invert": true
           }
@@ -5581,7 +5581,7 @@
         "label": "Filtering",
         "key": "initialFilter",
         "dependsOn": {
-          "setting": "datasource.type",
+          "setting": "table.type",
           "value": "custom",
           "invert": true
         }
@@ -5592,7 +5592,7 @@
         "key": "initialSortColumn",
         "placeholder": "Default",
         "dependsOn": {
-          "setting": "datasource.type",
+          "setting": "table.type",
           "value": "custom",
           "invert": true
         }
@@ -5642,7 +5642,7 @@
         "key": "allowAddRows",
         "defaultValue": true,
         "dependsOn": {
-          "setting": "datasource.type",
+          "setting": "table.type",
           "value": ["table", "viewV2"]
         }
       },
@@ -5652,7 +5652,7 @@
         "key": "allowEditRows",
         "defaultValue": true,
         "dependsOn": {
-          "setting": "datasource.type",
+          "setting": "table.type",
           "value": ["table", "viewV2"]
         }
       },
@@ -5662,7 +5662,7 @@
         "key": "allowDeleteRows",
         "defaultValue": true,
         "dependsOn": {
-          "setting": "datasource.type",
+          "setting": "table.type",
           "value": ["table", "viewV2"]
         }
       },

From e701bd2b89153ff446e943c265bfdbb65917b84d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 12:38:52 +0200
Subject: [PATCH 45/95] Ref name in master

---
 .github/workflows/deploy-qa.yml      | 2 +-
 .github/workflows/release-master.yml | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index d4bbb1c36a..a3fff65f35 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -17,5 +17,5 @@ jobs:
           REF_NAME: ${{ github.ref_name}}
         with:
           repository: budibase/budibase-deploys
-          event-type: budicloud-qa-deploy-v2
+          event-type: budicloud-qa-deploy
           token: ${{ secrets.GH_ACCESS_TOKEN }}
diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml
index 4acd5088d2..9ab8530341 100644
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@@ -123,6 +123,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
+          REF_NAME: ${{ github.ref_name}}
         with:
           repository: budibase/budibase-deploys
           event: budicloud-qa-deploy

From bc8e84b8156468b8440cda65a54d7169c8a6bacf Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 20 Oct 2023 12:19:04 +0100
Subject: [PATCH 46/95] Fixing datasource error states causing a datasource to
 fail to be built fully - this was simply a frontend issue of the error not
 being handled on creation.

---
 packages/builder/src/stores/backend/datasources.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js
index 11184f2caa..c0c37a7651 100644
--- a/packages/builder/src/stores/backend/datasources.js
+++ b/packages/builder/src/stores/backend/datasources.js
@@ -81,9 +81,9 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const updateDatasource = response => {
+  const updateDatasource = (response, { ignoreErrors } = {}) => {
     const { datasource, errors } = response
-    if (errors && Object.keys(errors).length > 0) {
+    if (!ignoreErrors && errors && Object.keys(errors).length > 0) {
       throw new TableImportError(errors)
     }
     replaceDatasource(datasource._id, datasource)
@@ -137,7 +137,7 @@ export function createDatasourcesStore() {
       fetchSchema: integration.plus,
     })
 
-    return updateDatasource(response)
+    return updateDatasource(response, { ignoreErrors: true })
   }
 
   const update = async ({ integration, datasource }) => {

From 1478ef503ade59f2aa172ec1c3a059019ee97e6a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 13:31:50 +0200
Subject: [PATCH 47/95] Don't use nx cloud locally

---
 nx.json | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/nx.json b/nx.json
index fef6893f9d..618395ec90 100644
--- a/nx.json
+++ b/nx.json
@@ -3,14 +3,16 @@
     "default": {
       "runner": "nx-cloud",
       "options": {
-        "cacheableOperations": ["build", "test", "check:types"],
-        "accessToken": "MmM4OGYxNzItMDBlYy00ZmE3LTk4MTYtNmJhYWMyZjBjZTUyfHJlYWQ="
+        "cacheableOperations": ["build", "test", "check:types"]
       }
     }
   },
   "targetDefaults": {
     "build": {
-      "inputs": ["{workspaceRoot}/scripts/build.js"]
+      "inputs": [
+        "{workspaceRoot}/scripts/build.js",
+        "{workspaceRoot}/lerna.json"
+      ]
     }
   }
 }

From 1b3ba9a4022b0a2a0d4767d63d68b075575d8634 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 13:32:54 +0200
Subject: [PATCH 48/95] Don't use nx cloud on ci

---
 .github/workflows/budibase_ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 1580be338e..c73d013b78 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -19,7 +19,6 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
 jobs:
   lint:

From f1365121843cf07aa489d2d9f4886e929999ccde Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 14:03:43 +0200
Subject: [PATCH 49/95] Clean pipeline

---
 .github/workflows/deploy-qa.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index a3fff65f35..d850d289ff 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - master
-      - BUDI-7641/push_v2_images_to_qa
   workflow_dispatch:
 
 jobs:

From 70f6f861d011bae826c6f10ebfd827dd51d740f5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 14:03:43 +0200
Subject: [PATCH 50/95] Clean pipeline

---
 .github/workflows/deploy-qa.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml
index a3fff65f35..d850d289ff 100644
--- a/.github/workflows/deploy-qa.yml
+++ b/.github/workflows/deploy-qa.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - master
-      - BUDI-7641/push_v2_images_to_qa
   workflow_dispatch:
 
 jobs:

From e5aefa83eccff6ee630b9aec3c7d571f36e7660c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 15:13:29 +0200
Subject: [PATCH 51/95] Fix single image dockerfile

---
 hosting/single/Dockerfile.v2 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2
index ad11545a22..a983f10044 100644
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@@ -26,6 +26,7 @@ RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json
 # We will never want to sync pro, but the script is still required
 RUN echo '' > scripts/syncProPackage.js
 RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
+RUN ./scripts/removeWorkspaceDependencies.sh package.json
 RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production
 
 # copy the actual code

From 36a33bc19299b3fdbb6e44670798b27d38449287 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 15:15:52 +0200
Subject: [PATCH 52/95] Use nx affected on PRs

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index c73d013b78..77867c8617 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
-  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   lint:

From 2d1e01998a320c9dfe048f228b0baf99d864caed Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 15:16:48 +0200
Subject: [PATCH 53/95] Use nx affected on PRs

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index c73d013b78..77867c8617 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -18,7 +18,7 @@ env:
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
-  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   lint:

From 0010e07b47112aba01c890b53415bcf09a9847e2 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 20 Oct 2023 13:29:13 +0000
Subject: [PATCH 54/95] Bump version to 2.11.40

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index e01e5ae03e..75db3b9b55 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.39",
+  "version": "2.11.40",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 961913b88a3208924eba14a9602b853d9e6de8ce Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 16:21:35 +0200
Subject: [PATCH 55/95] Use version from environment

---
 packages/backend-core/src/environment.ts | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 3894bdd0f7..4fb8b5fdca 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -49,7 +49,6 @@ function httpLogging() {
 }
 
 function getPackageJsonFields(): {
-  VERSION: string
   SERVICE_NAME: string
 } {
   function findFileInAncestors(
@@ -75,12 +74,11 @@ function getPackageJsonFields(): {
     const content = readFileSync(packageJsonFile!, "utf-8")
     const parsedContent = JSON.parse(content)
     return {
-      VERSION: parsedContent.version,
       SERVICE_NAME: parsedContent.name,
     }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return { VERSION: "", SERVICE_NAME: "" }
+    return { SERVICE_NAME: "" }
   }
 }
 
@@ -173,6 +171,7 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
+  VERSION: process.env.VERSION!,
   ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   OFFLINE_MODE: process.env.OFFLINE_MODE,

From 56ae857d498908c95c2c8e8146455ca170eaf8b0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 16:21:48 +0200
Subject: [PATCH 56/95] Add version to dev .env

---
 packages/server/scripts/dev/manage.js | 1 +
 packages/worker/scripts/dev/manage.js | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js
index 5db45040bf..13639b6bfd 100644
--- a/packages/server/scripts/dev/manage.js
+++ b/packages/server/scripts/dev/manage.js
@@ -47,6 +47,7 @@ async function init() {
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
       HTTP_MIGRATIONS: "0",
       HTTP_LOGGING: "0",
+      VERSION: "0.0.0+local",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {
diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js
index ecf5defd47..9e6a57d4bf 100644
--- a/packages/worker/scripts/dev/manage.js
+++ b/packages/worker/scripts/dev/manage.js
@@ -31,6 +31,7 @@ async function init() {
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
       ENABLE_EMAIL_TEST_MODE: 1,
       HTTP_LOGGING: 0,
+      VERSION: "0.0.0+local",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

From 04a2bbc61aba8016d84b649273a60dc3c7c6effb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 20 Oct 2023 15:45:35 +0100
Subject: [PATCH 57/95] Two fixes here - a quick fix for the builder side
 panel, making sure it fills up with users correctly (not all, but enough to
 make it look more pleasant) as well as dropping user search endpoint
 permissions to allow basic users to access it for user columns.

---
 packages/backend-core/src/users/db.ts              | 10 +++++-----
 packages/backend-core/src/users/users.ts           |  9 ++++++---
 .../_components/BuilderSidePanel.svelte            |  3 ++-
 packages/types/src/api/web/user.ts                 |  1 +
 .../worker/src/api/controllers/global/users.ts     |  7 ++++++-
 .../src/api/routes/global/tests/users.spec.ts      |  6 +++++-
 packages/worker/src/api/routes/global/users.ts     |  3 ++-
 packages/worker/src/tests/api/users.ts             | 14 ++++++++++----
 8 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index 8bb6300d4e..daa09bee6f 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -164,14 +164,14 @@ export class UserDB {
     }
   }
 
-  static async getUsersByAppAccess(appId?: string) {
-    const opts: any = {
+  static async getUsersByAppAccess(opts: { appId?: string; limit?: number }) {
+    const params: any = {
       include_docs: true,
-      limit: 50,
+      limit: opts.limit || 50,
     }
     let response: User[] = await usersCore.searchGlobalUsersByAppAccess(
-      appId,
-      opts
+      opts.appId,
+      params
     )
     return response
   }
diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index a64997224e..bad108ab84 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -19,6 +19,7 @@ import {
   SearchUsersRequest,
   User,
   ContextUser,
+  DatabaseQueryOpts,
 } from "@budibase/types"
 import { getGlobalDB } from "../context"
 import * as context from "../context"
@@ -241,12 +242,14 @@ export const paginatedUsers = async ({
   bookmark,
   query,
   appId,
+  limit,
 }: SearchUsersRequest = {}) => {
   const db = getGlobalDB()
+  const pageLimit = limit ? limit + 1 : PAGE_LIMIT + 1
   // get one extra document, to have the next page
-  const opts: any = {
+  const opts: DatabaseQueryOpts = {
     include_docs: true,
-    limit: PAGE_LIMIT + 1,
+    limit: pageLimit,
   }
   // add a startkey if the page was specified (anchor)
   if (bookmark) {
@@ -269,7 +272,7 @@ export const paginatedUsers = async ({
     const response = await db.allDocs(getGlobalUserParams(null, opts))
     userList = response.rows.map((row: any) => row.doc)
   }
-  return pagination(userList, PAGE_LIMIT, {
+  return pagination(userList, pageLimit, {
     paginate: true,
     property,
     getKey,
diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
index c93a41f541..a7d9584330 100644
--- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
@@ -114,8 +114,9 @@
       query: {
         appId: query || !filterByAppAccess ? null : prodAppId,
         email: query,
-        paginated: query || !filterByAppAccess ? null : false,
       },
+      limit: 50,
+      paginate: query || !filterByAppAccess ? null : false,
     })
     await usersFetch.refresh()
 
diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index a1e039cfd7..3a5bd16bdf 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -55,6 +55,7 @@ export interface SearchUsersRequest {
   bookmark?: string
   query?: SearchQuery
   appId?: string
+  limit?: number
   paginate?: boolean
 }
 
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 8de3a1444e..bfe9bdf284 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -189,7 +189,10 @@ export const destroy = async (ctx: any) => {
 
 export const getAppUsers = async (ctx: Ctx<SearchUsersRequest>) => {
   const body = ctx.request.body
-  const users = await userSdk.db.getUsersByAppAccess(body?.appId)
+  const users = await userSdk.db.getUsersByAppAccess({
+    appId: body.appId,
+    limit: body.limit,
+  })
 
   ctx.body = { data: users }
 }
@@ -203,8 +206,10 @@ export const search = async (ctx: Ctx<SearchUsersRequest>) => {
   }
 
   if (body.paginate === false) {
+    console.log("not paginated")
     await getAppUsers(ctx)
   } else {
+    console.log("paginated")
     const paginated = await userSdk.core.paginatedUsers(body)
     // user hashed password shouldn't ever be returned
     for (let user of paginated.data) {
diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts
index a446d10ed0..846b98a7ae 100644
--- a/packages/worker/src/api/routes/global/tests/users.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/users.spec.ts
@@ -569,9 +569,13 @@ describe("/api/global/users", () => {
         {
           query: { equal: { firstName: user.firstName } },
         },
-        501
+        { status: 501 }
       )
     })
+
+    it("should throw an error if public query performed", async () => {
+      await config.api.users.searchUsers({}, { status: 403, noHeaders: true })
+    })
   })
 
   describe("DELETE /api/global/users/:userId", () => {
diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts
index a57f7834ac..3c9cfd2f41 100644
--- a/packages/worker/src/api/routes/global/users.ts
+++ b/packages/worker/src/api/routes/global/users.ts
@@ -72,7 +72,8 @@ router
   )
 
   .get("/api/global/users", auth.builderOrAdmin, controller.fetch)
-  .post("/api/global/users/search", auth.builderOrAdmin, controller.search)
+  // search can be used by any user now, to retrieve users for user column
+  .post("/api/global/users/search", controller.search)
   .delete("/api/global/users/:id", auth.adminOnly, controller.destroy)
   .get(
     "/api/global/users/count/:appId",
diff --git a/packages/worker/src/tests/api/users.ts b/packages/worker/src/tests/api/users.ts
index b2a19bcb28..ca25e2f9ca 100644
--- a/packages/worker/src/tests/api/users.ts
+++ b/packages/worker/src/tests/api/users.ts
@@ -134,13 +134,19 @@ export class UserAPI extends TestAPI {
       .expect(status ? status : 200)
   }
 
-  searchUsers = ({ query }: { query?: SearchQuery }, status = 200) => {
-    return this.request
+  searchUsers = (
+    { query }: { query?: SearchQuery },
+    opts?: { status?: number; noHeaders?: boolean }
+  ) => {
+    const req = this.request
       .post("/api/global/users/search")
-      .set(this.config.defaultHeaders())
       .send({ query })
       .expect("Content-Type", /json/)
-      .expect(status ? status : 200)
+      .expect(opts?.status ? opts.status : 200)
+    if (!opts?.noHeaders) {
+      req.set(this.config.defaultHeaders())
+    }
+    return req
   }
 
   getUser = (userId: string, opts?: TestAPIOpts) => {

From 8c744ea7a99ab68e7388094ab74c6fc5ba708e78 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 20 Oct 2023 15:57:45 +0100
Subject: [PATCH 58/95] PR comments.

---
 packages/worker/src/api/controllers/global/users.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index bfe9bdf284..de1a605890 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -206,10 +206,8 @@ export const search = async (ctx: Ctx<SearchUsersRequest>) => {
   }
 
   if (body.paginate === false) {
-    console.log("not paginated")
     await getAppUsers(ctx)
   } else {
-    console.log("paginated")
     const paginated = await userSdk.core.paginatedUsers(body)
     // user hashed password shouldn't ever be returned
     for (let user of paginated.data) {

From 8a939dd3e60cb8dc5f0d9e8703fd5a41c7879e1c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:02:24 +0200
Subject: [PATCH 59/95] Update pro submodule ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 570d14aa44..56d968bfe6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76
+Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9

From 97fc794a2b198f2466dd1ec41e4371de8fd8a224 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 20 Oct 2023 15:15:17 +0000
Subject: [PATCH 60/95] Bump version to 2.11.41

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 75db3b9b55..4b2bbb05d5 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.40",
+  "version": "2.11.41",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 4c19ad1ca7f27c9713b9551bd4711aee6b081685 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:19:35 +0200
Subject: [PATCH 61/95] Add versions to server/worker docker v2

---
 hosting/docker-compose.build.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml
index e192620b59..19ce9b5473 100644
--- a/hosting/docker-compose.build.yaml
+++ b/hosting/docker-compose.build.yaml
@@ -7,6 +7,8 @@ services:
     build:
       context: ..
       dockerfile: packages/server/Dockerfile.v2
+      args:
+        - VERSION=0.0.0+dev-docker
     container_name: build-bbapps
     environment:
       SELF_HOSTED: 1
@@ -30,13 +32,13 @@ services:
     depends_on:
       - worker-service
       - redis-service
-  #    volumes:
-  #      - /some/path/to/plugins:/plugins
 
   worker-service:
     build:
       context: ..
       dockerfile: packages/worker/Dockerfile.v2
+      args:
+        - VERSION=0.0.0+dev-docker
     container_name: build-bbworker
     environment:
       SELF_HOSTED: 1

From 9dacb5849b126b69ae5006a39205719dd6e79a35 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:19:44 +0200
Subject: [PATCH 62/95] Add versions to server/worker docker v2

---
 packages/server/Dockerfile.v2 | 4 ++++
 packages/worker/Dockerfile.v2 | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2
index d5a86b037d..339cefcc45 100644
--- a/packages/server/Dockerfile.v2
+++ b/packages/server/Dockerfile.v2
@@ -67,6 +67,10 @@ COPY packages/server/docker_run.sh .
 COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
 
+ARG VERSION
+RUN test -n "$VERSION"
+ENV VERSION=$VERSION
+
 EXPOSE 4001
 
 # have to add node environment production after install
diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2
index 0d60db6fc5..20fcd4a69b 100644
--- a/packages/worker/Dockerfile.v2
+++ b/packages/worker/Dockerfile.v2
@@ -50,4 +50,8 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
+ARG VERSION
+RUN test -n "$VERSION"
+ENV VERSION=$VERSION
+
 CMD ["./docker_run.sh"]

From 133e4752576c27bd2f41f68578236aa31b7a48e4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:20:14 +0200
Subject: [PATCH 63/95] Add version to single image

---
 hosting/single/Dockerfile.v2 | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2
index a983f10044..ecc223be15 100644
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@@ -118,6 +118,9 @@ EXPOSE 443
 EXPOSE 2222
 VOLUME /data
 
+ARG VERSION
+RUN test -n "$VERSION"
+ENV VERSION=$VERSION
 
 HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
 

From 6ff11a82994719c556c3e68d46a5c5d35a5a57bf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:23:16 +0200
Subject: [PATCH 64/95] Fix local docker singleimage build

---
 scripts/build-single-image.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/build-single-image.sh b/scripts/build-single-image.sh
index cb2c29deac..216d758fae 100755
--- a/scripts/build-single-image.sh
+++ b/scripts/build-single-image.sh
@@ -1,3 +1,4 @@
 #!/bin/bash
 yarn build --scope @budibase/server --scope @budibase/worker
-docker build -f hosting/single/Dockerfile.v2 -t budibase:latest .
+version=$(./scripts/getCurrentVersion.sh)
+docker build -f hosting/single/Dockerfile.v2 -t budibase:latest --build-arg VERSION=$version .

From 5ee1d73e8160d13862a1dd8f6883d2073380745e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:25:30 +0200
Subject: [PATCH 65/95] Add args to build

---
 packages/server/package.json | 2 +-
 packages/worker/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 0bc3618a08..0636eebe77 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -18,7 +18,7 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg VERSION=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index 1eee3f020f..0a75eac522 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -20,7 +20,7 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg VERSION=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",

From 8db11ac174805355c1961defd7588629811de63f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:27:42 +0200
Subject: [PATCH 66/95] Add version to test image pipeline

---
 .github/workflows/release-singleimage-test.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-singleimage-test.yml b/.github/workflows/release-singleimage-test.yml
index 79b9afdd44..3d6c0d2b3c 100644
--- a/.github/workflows/release-singleimage-test.yml
+++ b/.github/workflows/release-singleimage-test.yml
@@ -54,6 +54,7 @@ jobs:
           push: true
           pull: true
           platforms: linux/amd64,linux/arm64
+          build-args: VERSION=0.0.0+test
           tags: budibase/budibase-test:test
           file: ./hosting/single/Dockerfile.v2
           cache-from: type=registry,ref=budibase/budibase-test:test
@@ -64,6 +65,8 @@ jobs:
           context: .
           push: true
           platforms: linux/amd64
-          build-args: TARGETBUILD=aas
+          build-args: |
+            TARGETBUILD=aas
+            VERSION=0.0.0+test
           tags: budibase/budibase-test:aas
           file: ./hosting/single/Dockerfile.v2

From d1f0e8fca0b4577e8ec0defa39f269727e793d77 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 17:53:23 +0200
Subject: [PATCH 67/95] Fix tests

---
 packages/server/src/tests/jestEnv.ts | 1 +
 packages/worker/src/tests/jestEnv.ts | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/server/src/tests/jestEnv.ts b/packages/server/src/tests/jestEnv.ts
index 34c51009aa..c1f32eeec9 100644
--- a/packages/server/src/tests/jestEnv.ts
+++ b/packages/server/src/tests/jestEnv.ts
@@ -9,3 +9,4 @@ process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"
+process.env.VERSION = "0.0.0+jest"
diff --git a/packages/worker/src/tests/jestEnv.ts b/packages/worker/src/tests/jestEnv.ts
index 9153676b8e..f6de4e0fd2 100644
--- a/packages/worker/src/tests/jestEnv.ts
+++ b/packages/worker/src/tests/jestEnv.ts
@@ -10,3 +10,4 @@ process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.INTERNAL_API_KEY = "tet"
 process.env.DISABLE_ACCOUNT_PORTAL = "0"
 process.env.MOCK_REDIS = "1"
+process.env.VERSION = "0.0.0+jest"

From 71869badfa26e57035582bca3f3d670a9fc7a8c0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 19:19:35 +0200
Subject: [PATCH 68/95] Fix tests

---
 packages/server/src/api/routes/tests/debug.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/debug.spec.ts b/packages/server/src/api/routes/tests/debug.spec.ts
index 23ee43fc73..26e98d93f9 100644
--- a/packages/server/src/api/routes/tests/debug.spec.ts
+++ b/packages/server/src/api/routes/tests/debug.spec.ts
@@ -41,7 +41,7 @@ describe("/component", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body).toEqual({
-        budibaseVersion: "0.0.0",
+        budibaseVersion: "0.0.0+jest",
         cpuArch: "arm64",
         cpuCores: 1,
         cpuInfo: "test",

From e7459bb5e704685338032509334933f9fd80dede Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 20 Oct 2023 20:42:58 +0200
Subject: [PATCH 69/95] Fix test

---
 packages/server/src/api/routes/tests/dev.spec.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/dev.spec.js b/packages/server/src/api/routes/tests/dev.spec.js
index 111f3dbd5b..af1dc82a9d 100644
--- a/packages/server/src/api/routes/tests/dev.spec.js
+++ b/packages/server/src/api/routes/tests/dev.spec.js
@@ -1,6 +1,6 @@
 const setup = require("./utilities")
 const { events } = require("@budibase/backend-core")
-const version = require("../../../../package.json").version
+
 
 describe("/dev", () => {
   let request = setup.getRequest()
@@ -32,9 +32,9 @@ describe("/dev", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.version).toBe(version)
+      expect(res.body.version).toBe('0.0.0+jest')
       expect(events.installation.versionChecked).toBeCalledTimes(1)
-      expect(events.installation.versionChecked).toBeCalledWith(version)
+      expect(events.installation.versionChecked).toBeCalledWith('0.0.0+jest')
     })
   })
 })
\ No newline at end of file

From 4ee1fd0f7ad53a5ed6c08d202826a2b227adf4c1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 23 Oct 2023 12:23:44 +0200
Subject: [PATCH 70/95] Default to package.json

---
 packages/backend-core/src/environment.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 4fb8b5fdca..a0905fb0f5 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -49,6 +49,7 @@ function httpLogging() {
 }
 
 function getPackageJsonFields(): {
+  VERSION: string
   SERVICE_NAME: string
 } {
   function findFileInAncestors(
@@ -74,11 +75,12 @@ function getPackageJsonFields(): {
     const content = readFileSync(packageJsonFile!, "utf-8")
     const parsedContent = JSON.parse(content)
     return {
+      VERSION: process.env.VERSION || parsedContent.version,
       SERVICE_NAME: parsedContent.name,
     }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return { SERVICE_NAME: "" }
+    return { VERSION: process.env.VERSION || "", SERVICE_NAME: "" }
   }
 }
 
@@ -171,7 +173,6 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  VERSION: process.env.VERSION!,
   ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   OFFLINE_MODE: process.env.OFFLINE_MODE,

From 648410348d2c2f0c9a3ce1e2d2a34b0ca4f5f80e Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Mon, 23 Oct 2023 13:27:59 +0200
Subject: [PATCH 71/95] [Revert] [Added] Per user per creator changes

---
 .../backend-core/src/cache/writethrough.ts    |   4 +-
 packages/backend-core/src/users/db.ts         | 116 ++++++++----------
 packages/backend-core/src/users/users.ts      |  13 --
 packages/backend-core/src/users/utils.ts      |   1 -
 .../tests/core/users/users.spec.js            |  54 --------
 .../core/utilities/structures/licenses.ts     |  13 --
 .../tests/core/utilities/structures/quotas.ts |   5 +-
 packages/pro                                  |   2 +-
 .../src/migrations/functions/syncQuotas.ts    |   2 -
 .../functions/usageQuotas/syncCreators.ts     |  13 --
 .../usageQuotas/tests/syncCreators.spec.ts    |  26 ----
 .../shared-core/src/sdk/documents/users.ts    |  25 ----
 packages/types/src/documents/global/quotas.ts |   1 -
 packages/types/src/sdk/featureFlag.ts         |   3 -
 packages/types/src/sdk/licensing/billing.ts   |   7 --
 packages/types/src/sdk/licensing/plan.ts      |   4 -
 packages/types/src/sdk/licensing/quota.ts     |   2 -
 17 files changed, 55 insertions(+), 236 deletions(-)
 delete mode 100644 packages/backend-core/tests/core/users/users.spec.js
 delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
 delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts

diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts
index c331d791a6..e64c116663 100644
--- a/packages/backend-core/src/cache/writethrough.ts
+++ b/packages/backend-core/src/cache/writethrough.ts
@@ -119,8 +119,8 @@ export class Writethrough {
     this.writeRateMs = writeRateMs
   }
 
-  async put(doc: any, writeRateMs: number = this.writeRateMs) {
-    return put(this.db, doc, writeRateMs)
+  async put(doc: any) {
+    return put(this.db, doc, this.writeRateMs)
   }
 
   async get(id: string) {
diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index daa09bee6f..a2539e836e 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -25,17 +25,12 @@ import {
 import {
   getAccountHolderFromUserIds,
   isAdmin,
-  isCreator,
   validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
 
-type QuotaUpdateFn = (
-  change: number,
-  creatorsChange: number,
-  cb?: () => Promise<any>
-) => Promise<any>
+type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
 type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
@@ -250,8 +245,7 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0
-    return UserDB.quotas.addUsers(change, creatorsChange, async () => {
+    return UserDB.quotas.addUsers(change, async () => {
       await validateUniqueUser(email, tenantId)
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
@@ -313,7 +307,6 @@ export class UserDB {
 
     let usersToSave: any[] = []
     let newUsers: any[] = []
-    let newCreators: any[] = []
 
     const emails = newUsersRequested.map((user: User) => user.email)
     const existingEmails = await searchExistingEmails(emails)
@@ -334,66 +327,59 @@ export class UserDB {
       }
       newUser.userGroups = groups
       newUsers.push(newUser)
-      if (isCreator(newUser)) {
-        newCreators.push(newUser)
-      }
     }
 
     const account = await accountSdk.getAccountByTenantId(tenantId)
-    return UserDB.quotas.addUsers(
-      newUsers.length,
-      newCreators.length,
-      async () => {
-        // create the promises array that will be called by bulkDocs
-        newUsers.forEach((user: any) => {
-          usersToSave.push(
-            UserDB.buildUser(
-              user,
-              {
-                hashPassword: true,
-                requirePassword: user.requirePassword,
-              },
-              tenantId,
-              undefined, // no dbUser
-              account
-            )
+    return UserDB.quotas.addUsers(newUsers.length, async () => {
+      // create the promises array that will be called by bulkDocs
+      newUsers.forEach((user: any) => {
+        usersToSave.push(
+          UserDB.buildUser(
+            user,
+            {
+              hashPassword: true,
+              requirePassword: user.requirePassword,
+            },
+            tenantId,
+            undefined, // no dbUser
+            account
           )
-        })
+        )
+      })
 
-        const usersToBulkSave = await Promise.all(usersToSave)
-        await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+      const usersToBulkSave = await Promise.all(usersToSave)
+      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
 
-        // Post-processing of bulk added users, e.g. events and cache operations
-        for (const user of usersToBulkSave) {
-          // TODO: Refactor to bulk insert users into the info db
-          // instead of relying on looping tenant creation
-          await platform.users.addUser(tenantId, user._id, user.email)
-          await eventHelpers.handleSaveEvents(user, undefined)
-        }
-
-        const saved = usersToBulkSave.map(user => {
-          return {
-            _id: user._id,
-            email: user.email,
-          }
-        })
-
-        // now update the groups
-        if (Array.isArray(saved) && groups) {
-          const groupPromises = []
-          const createdUserIds = saved.map(user => user._id)
-          for (let groupId of groups) {
-            groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
-          }
-          await Promise.all(groupPromises)
-        }
-
-        return {
-          successful: saved,
-          unsuccessful,
-        }
+      // Post-processing of bulk added users, e.g. events and cache operations
+      for (const user of usersToBulkSave) {
+        // TODO: Refactor to bulk insert users into the info db
+        // instead of relying on looping tenant creation
+        await platform.users.addUser(tenantId, user._id, user.email)
+        await eventHelpers.handleSaveEvents(user, undefined)
       }
-    )
+
+      const saved = usersToBulkSave.map(user => {
+        return {
+          _id: user._id,
+          email: user.email,
+        }
+      })
+
+      // now update the groups
+      if (Array.isArray(saved) && groups) {
+        const groupPromises = []
+        const createdUserIds = saved.map(user => user._id)
+        for (let groupId of groups) {
+          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+        }
+        await Promise.all(groupPromises)
+      }
+
+      return {
+        successful: saved,
+        unsuccessful,
+      }
+    })
   }
 
   static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
@@ -433,12 +419,11 @@ export class UserDB {
       _deleted: true,
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
-    const creatorsToDelete = usersToDelete.filter(isCreator)
 
+    await UserDB.quotas.removeUsers(toDelete.length)
     for (let user of usersToDelete) {
       await bulkDeleteProcessing(user)
     }
-    await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length)
 
     // Build Response
     // index users by id
@@ -487,8 +472,7 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev)
 
-    const creatorsToDelete = isCreator(dbUser) ? 1 : 0
-    await UserDB.quotas.removeUsers(1, creatorsToDelete)
+    await UserDB.quotas.removeUsers(1)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })
diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index bad108ab84..30b9e0a45f 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -287,19 +287,6 @@ export async function getUserCount() {
   return response.total_rows
 }
 
-export async function getCreatorCount() {
-  let creators = 0
-  async function iterate(startPage?: string) {
-    const page = await paginatedUsers({ bookmark: startPage })
-    creators += page.data.filter(isCreator).length
-    if (page.hasNextPage) {
-      await iterate(page.nextPage)
-    }
-  }
-  await iterate()
-  return creators
-}
-
 // used to remove the builder/admin permissions, for processing the
 // user as an app user (they may have some specific role/group
 export function removePortalUserPermissions(user: User | ContextUser) {
diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts
index 0ef4b77998..af0e8e10c7 100644
--- a/packages/backend-core/src/users/utils.ts
+++ b/packages/backend-core/src/users/utils.ts
@@ -10,7 +10,6 @@ import { getAccountByTenantId } from "../accounts"
 // extract from shared-core to make easily accessible from backend-core
 export const isBuilder = sdk.users.isBuilder
 export const isAdmin = sdk.users.isAdmin
-export const isCreator = sdk.users.isCreator
 export const isGlobalBuilder = sdk.users.isGlobalBuilder
 export const isAdminOrBuilder = sdk.users.isAdminOrBuilder
 export const hasAdminPermissions = sdk.users.hasAdminPermissions
diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js
deleted file mode 100644
index ae7109344a..0000000000
--- a/packages/backend-core/tests/core/users/users.spec.js
+++ /dev/null
@@ -1,54 +0,0 @@
-const _ = require('lodash/fp')
-const {structures} = require("../../../tests")
-
-jest.mock("../../../src/context")
-jest.mock("../../../src/db")
-
-const context = require("../../../src/context")
-const db = require("../../../src/db")
-
-const {getCreatorCount} = require('../../../src/users/users')
-
-describe("Users", () => {
-
-  let getGlobalDBMock
-  let getGlobalUserParamsMock
-  let paginationMock
-
-  beforeEach(() => {
-    jest.resetAllMocks()
-
-    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
-    getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams")
-    paginationMock = jest.spyOn(db, "pagination")
-  })
-
-  it("Retrieves the number of creators", async () => {
-    const getUsers = (offset, limit, creators = false) => {
-      const range = _.range(offset, limit)
-      const opts = creators ? {builder: {global: true}} : undefined
-      return range.map(() => structures.users.user(opts))
-    }
-    const page1Data = getUsers(0, 8)
-    const page2Data = getUsers(8, 12, true)
-    getGlobalDBMock.mockImplementation(() => ({
-      name   : "fake-db",
-      allDocs: () => ({
-        rows: [...page1Data, ...page2Data]
-      })
-    }))
-    paginationMock.mockImplementationOnce(() => ({
-      data: page1Data,
-      hasNextPage: true,
-      nextPage: "1"
-    }))
-    paginationMock.mockImplementation(() => ({
-      data: page2Data,
-      hasNextPage: false,
-      nextPage: undefined
-    }))
-    const creatorsCount = await getCreatorCount()
-    expect(creatorsCount).toBe(4)
-    expect(paginationMock).toHaveBeenCalledTimes(2)
-  })
-})
diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts
index bb452f9ad5..5cce84edfd 100644
--- a/packages/backend-core/tests/core/utilities/structures/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts
@@ -72,11 +72,6 @@ export function quotas(): Quotas {
           value: 1,
           triggers: [],
         },
-        creators: {
-          name: "Creators",
-          value: 1,
-          triggers: [],
-        },
         userGroups: {
           name: "User Groups",
           value: 1,
@@ -123,10 +118,6 @@ export function customer(): Customer {
 export function subscription(): Subscription {
   return {
     amount: 10000,
-    amounts: {
-      user: 10000,
-      creator: 0,
-    },
     cancelAt: undefined,
     currency: "usd",
     currentPeriodEnd: 0,
@@ -135,10 +126,6 @@ export function subscription(): Subscription {
     duration: PriceDuration.MONTHLY,
     pastDueAt: undefined,
     quantity: 0,
-    quantities: {
-      user: 0,
-      creator: 0,
-    },
     status: "active",
   }
 }
diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts
index 8d0b05fe1e..e82117053f 100644
--- a/packages/backend-core/tests/core/utilities/structures/quotas.ts
+++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts
@@ -1,6 +1,6 @@
 import { MonthlyQuotaName, QuotaUsage } from "@budibase/types"
 
-export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
+export const usage = (): QuotaUsage => {
   return {
     _id: "usage_quota",
     quotaReset: new Date().toISOString(),
@@ -58,8 +58,7 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
     usageQuota: {
       apps: 0,
       plugins: 0,
-      users,
-      creators,
+      users: 0,
       userGroups: 0,
       rows: 0,
       triggers: {},
diff --git a/packages/pro b/packages/pro
index 56d968bfe6..1e43861b7e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9
+Subproject commit 1e43861b7ed7141284ab63a0cf5bd494ec3ab2d1
diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts
index 83a7670e78..67f38ba929 100644
--- a/packages/server/src/migrations/functions/syncQuotas.ts
+++ b/packages/server/src/migrations/functions/syncQuotas.ts
@@ -3,7 +3,6 @@ import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
 import * as syncPlugins from "./usageQuotas/syncPlugins"
 import * as syncUsers from "./usageQuotas/syncUsers"
-import * as syncCreators from "./usageQuotas/syncCreators"
 
 /**
  * Synchronise quotas to the state of the db.
@@ -14,6 +13,5 @@ export const run = async () => {
     await syncRows.run()
     await syncPlugins.run()
     await syncUsers.run()
-    await syncCreators.run()
   })
 }
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
deleted file mode 100644
index ce53be925a..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { users } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
-  const creatorCount = await users.getCreatorCount()
-  console.log(`Syncing creator count: ${creatorCount}`)
-  await quotas.setUsage(
-    creatorCount,
-    StaticQuotaName.CREATORS,
-    QuotaUsageType.STATIC
-  )
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
deleted file mode 100644
index 75fa9f217e..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncCreators from "../syncCreators"
-import { quotas } from "@budibase/pro"
-
-describe("syncCreators", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-  })
-
-  afterAll(config.end)
-
-  it("syncs creators", async () => {
-    return config.doInContext(null, async () => {
-      await config.createUser({ admin: true })
-
-      await syncCreators.run()
-
-      const usageDoc = await quotas.getQuotaUsage()
-      // default + additional creator
-      const creatorsCount = 2
-      expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
-    })
-  })
-})
diff --git a/packages/shared-core/src/sdk/documents/users.ts b/packages/shared-core/src/sdk/documents/users.ts
index b58994aa46..03d86daa85 100644
--- a/packages/shared-core/src/sdk/documents/users.ts
+++ b/packages/shared-core/src/sdk/documents/users.ts
@@ -6,7 +6,6 @@ import {
   InternalTable,
 } from "@budibase/types"
 import { getProdAppID } from "./applications"
-import * as _ from "lodash/fp"
 
 // checks if a user is specifically a builder, given an app ID
 export function isBuilder(user: User | ContextUser, appId?: string): boolean {
@@ -59,18 +58,6 @@ export function hasAppBuilderPermissions(user?: User | ContextUser): boolean {
   return !isGlobalBuilder && appLength != null && appLength > 0
 }
 
-export function hasAppCreatorPermissions(user?: User | ContextUser): boolean {
-  if (!user) {
-    return false
-  }
-  return _.flow(
-    _.get("roles"),
-    _.values,
-    _.find(x => ["CREATOR", "ADMIN"].includes(x)),
-    x => !!x
-  )(user)
-}
-
 // checks if a user is capable of building any app
 export function hasBuilderPermissions(user?: User | ContextUser): boolean {
   if (!user) {
@@ -87,18 +74,6 @@ export function hasAdminPermissions(user?: User | ContextUser): boolean {
   return !!user.admin?.global
 }
 
-export function isCreator(user?: User | ContextUser): boolean {
-  if (!user) {
-    return false
-  }
-  return (
-    isGlobalBuilder(user) ||
-    hasAdminPermissions(user) ||
-    hasAppBuilderPermissions(user) ||
-    hasAppCreatorPermissions(user)
-  )
-}
-
 export function getGlobalUserID(userId?: string): string | undefined {
   if (typeof userId !== "string") {
     return userId
diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts
index 4eb1168f7d..61410f7435 100644
--- a/packages/types/src/documents/global/quotas.ts
+++ b/packages/types/src/documents/global/quotas.ts
@@ -32,7 +32,6 @@ export interface StaticUsage {
   [StaticQuotaName.APPS]: number
   [StaticQuotaName.PLUGINS]: number
   [StaticQuotaName.USERS]: number
-  [StaticQuotaName.CREATORS]: number
   [StaticQuotaName.USER_GROUPS]: number
   [StaticQuotaName.ROWS]: number
   triggers: {
diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts
index e3935bc7ee..53aa4842c4 100644
--- a/packages/types/src/sdk/featureFlag.ts
+++ b/packages/types/src/sdk/featureFlag.ts
@@ -1,8 +1,5 @@
 export enum FeatureFlag {
   LICENSING = "LICENSING",
-  // Feature IDs in Posthog
-  PER_CREATOR_PER_USER_PRICE = "18873",
-  PER_CREATOR_PER_USER_PRICE_ALERT = "18530",
 }
 
 export interface TenantFeatureFlags {
diff --git a/packages/types/src/sdk/licensing/billing.ts b/packages/types/src/sdk/licensing/billing.ts
index bcbc7abd18..35f366c811 100644
--- a/packages/types/src/sdk/licensing/billing.ts
+++ b/packages/types/src/sdk/licensing/billing.ts
@@ -5,17 +5,10 @@ export interface Customer {
   currency: string | null | undefined
 }
 
-export interface SubscriptionItems {
-  user: number | undefined
-  creator: number | undefined
-}
-
 export interface Subscription {
   amount: number
-  amounts: SubscriptionItems | undefined
   currency: string
   quantity: number
-  quantities: SubscriptionItems | undefined
   duration: PriceDuration
   cancelAt: number | null | undefined
   currentPeriodStart: number
diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts
index 1604dfb8af..3e214a01ff 100644
--- a/packages/types/src/sdk/licensing/plan.ts
+++ b/packages/types/src/sdk/licensing/plan.ts
@@ -4,9 +4,7 @@ export enum PlanType {
   PRO = "pro",
   /** @deprecated */
   TEAM = "team",
-  /** @deprecated */
   PREMIUM = "premium",
-  PREMIUM_PLUS = "premium_plus",
   BUSINESS = "business",
   ENTERPRISE = "enterprise",
 }
@@ -28,12 +26,10 @@ export interface AvailablePrice {
   currency: string
   duration: PriceDuration
   priceId: string
-  type?: string
 }
 
 export enum PlanModel {
   PER_USER = "perUser",
-  PER_CREATOR_PER_USER = "per_creator_per_user",
   DAY_PASS = "dayPass",
 }
 
diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 85700f167b..73afa1ed05 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -14,7 +14,6 @@ export enum StaticQuotaName {
   ROWS = "rows",
   APPS = "apps",
   USERS = "users",
-  CREATORS = "creators",
   USER_GROUPS = "userGroups",
   PLUGINS = "plugins",
 }
@@ -68,7 +67,6 @@ export type StaticQuotas = {
   [StaticQuotaName.ROWS]: Quota
   [StaticQuotaName.APPS]: Quota
   [StaticQuotaName.USERS]: Quota
-  [StaticQuotaName.CREATORS]: Quota
   [StaticQuotaName.USER_GROUPS]: Quota
   [StaticQuotaName.PLUGINS]: Quota
 }

From a285fb1d4027ca70d77a953ba16bcb74179ba8e1 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Mon, 23 Oct 2023 13:33:27 +0200
Subject: [PATCH 72/95] Missed code in merge

---
 packages/backend-core/src/users/users.ts | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index 30b9e0a45f..d259d5b560 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -14,16 +14,15 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
+  ContextUser,
   SearchQuery,
   SearchQueryOperators,
   SearchUsersRequest,
   User,
-  ContextUser,
   DatabaseQueryOpts,
 } from "@budibase/types"
-import { getGlobalDB } from "../context"
 import * as context from "../context"
-import { isCreator } from "./utils"
+import { getGlobalDB } from "../context"
 
 type GetOpts = { cleanup?: boolean }
 

From a22aea63a372f32a9a2089fd783cd72ced667378 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Mon, 23 Oct 2023 13:33:56 +0200
Subject: [PATCH 73/95] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 1e43861b7e..ee222c4694 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1e43861b7ed7141284ab63a0cf5bd494ec3ab2d1
+Subproject commit ee222c4694c71716eeb1257f37467fc2231ee166

From 73f3b76c01a924e122e0f2f1de4ab132a333e05f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 23 Oct 2023 12:40:29 +0100
Subject: [PATCH 74/95] Update default column visibility check

---
 packages/client/src/components/app/table/Table.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/components/app/table/Table.svelte b/packages/client/src/components/app/table/Table.svelte
index 34fa8c204d..7c133f4e17 100644
--- a/packages/client/src/components/app/table/Table.svelte
+++ b/packages/client/src/components/app/table/Table.svelte
@@ -70,7 +70,7 @@
     let columns = []
     let autoColumns = []
     Object.entries(schema).forEach(([field, fieldSchema]) => {
-      if (!fieldSchema.visible) {
+      if (fieldSchema.visible === false) {
         return
       }
       if (!fieldSchema?.autocolumn) {

From 0cc7cc67a388db38fbb03bbcf2adb1ce3ed3b364 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 23 Oct 2023 13:53:18 +0200
Subject: [PATCH 75/95] Renames

---
 .github/workflows/release-singleimage-test.yml | 4 ++--
 hosting/docker-compose.build.yaml              | 4 ++--
 hosting/single/Dockerfile.v2                   | 6 +++---
 packages/server/Dockerfile.v2                  | 6 +++---
 packages/server/package.json                   | 2 +-
 packages/worker/Dockerfile.v2                  | 6 +++---
 packages/worker/package.json                   | 2 +-
 scripts/build-single-image.sh                  | 2 +-
 8 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/release-singleimage-test.yml b/.github/workflows/release-singleimage-test.yml
index 3d6c0d2b3c..c3a14226ce 100644
--- a/.github/workflows/release-singleimage-test.yml
+++ b/.github/workflows/release-singleimage-test.yml
@@ -54,7 +54,7 @@ jobs:
           push: true
           pull: true
           platforms: linux/amd64,linux/arm64
-          build-args: VERSION=0.0.0+test
+          build-args: BUDIBASE_VERSION=0.0.0+test
           tags: budibase/budibase-test:test
           file: ./hosting/single/Dockerfile.v2
           cache-from: type=registry,ref=budibase/budibase-test:test
@@ -67,6 +67,6 @@ jobs:
           platforms: linux/amd64
           build-args: |
             TARGETBUILD=aas
-            VERSION=0.0.0+test
+            BUDIBASE_VERSION=0.0.0+test
           tags: budibase/budibase-test:aas
           file: ./hosting/single/Dockerfile.v2
diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml
index 19ce9b5473..7ead001a1c 100644
--- a/hosting/docker-compose.build.yaml
+++ b/hosting/docker-compose.build.yaml
@@ -8,7 +8,7 @@ services:
       context: ..
       dockerfile: packages/server/Dockerfile.v2
       args:
-        - VERSION=0.0.0+dev-docker
+        - BUDIBASE_VERSION=0.0.0+dev-docker
     container_name: build-bbapps
     environment:
       SELF_HOSTED: 1
@@ -38,7 +38,7 @@ services:
       context: ..
       dockerfile: packages/worker/Dockerfile.v2
       args:
-        - VERSION=0.0.0+dev-docker
+        - BUDIBASE_VERSION=0.0.0+dev-docker
     container_name: build-bbworker
     environment:
       SELF_HOSTED: 1
diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2
index ecc223be15..3b994ce0e6 100644
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@@ -118,9 +118,9 @@ EXPOSE 443
 EXPOSE 2222
 VOLUME /data
 
-ARG VERSION
-RUN test -n "$VERSION"
-ENV VERSION=$VERSION
+ARG BUDIBASE_VERSION
+RUN test -n "$BUDIBASE_VERSION"
+ENV BUDIBASE_VERSION=$VERSION
 
 HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
 
diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2
index 339cefcc45..341fe5ee34 100644
--- a/packages/server/Dockerfile.v2
+++ b/packages/server/Dockerfile.v2
@@ -67,9 +67,9 @@ COPY packages/server/docker_run.sh .
 COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
 
-ARG VERSION
-RUN test -n "$VERSION"
-ENV VERSION=$VERSION
+ARG BUDIBASE_VERSION
+RUN test -n "$BUDIBASE_VERSION"
+ENV BUDIBASE_VERSION=$VERSION
 
 EXPOSE 4001
 
diff --git a/packages/server/package.json b/packages/server/package.json
index 0636eebe77..4a858f3be9 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -18,7 +18,7 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg VERSION=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2
index 20fcd4a69b..ed3817e89d 100644
--- a/packages/worker/Dockerfile.v2
+++ b/packages/worker/Dockerfile.v2
@@ -50,8 +50,8 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
-ARG VERSION
-RUN test -n "$VERSION"
-ENV VERSION=$VERSION
+ARG BUDIBASE_VERSION
+RUN test -n "$BUDIBASE_VERSION"
+ENV BUDIBASE_VERSION=$VERSION
 
 CMD ["./docker_run.sh"]
diff --git a/packages/worker/package.json b/packages/worker/package.json
index 0a75eac522..205bf3309a 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -20,7 +20,7 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg VERSION=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",
diff --git a/scripts/build-single-image.sh b/scripts/build-single-image.sh
index 216d758fae..ed3d9a8ed6 100755
--- a/scripts/build-single-image.sh
+++ b/scripts/build-single-image.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 yarn build --scope @budibase/server --scope @budibase/worker
 version=$(./scripts/getCurrentVersion.sh)
-docker build -f hosting/single/Dockerfile.v2 -t budibase:latest --build-arg VERSION=$version .
+docker build -f hosting/single/Dockerfile.v2 -t budibase:latest --build-arg BUDIBASE_VERSION=$version .

From 3c16e820354c9b8b9ee03ad8e34ad304fb504d60 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 23 Oct 2023 13:55:23 +0200
Subject: [PATCH 76/95] Comment docker

---
 hosting/single/Dockerfile.v2  | 1 +
 packages/server/Dockerfile.v2 | 1 +
 packages/worker/Dockerfile.v2 | 1 +
 3 files changed, 3 insertions(+)

diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2
index 3b994ce0e6..66ce0f1c4d 100644
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@@ -119,6 +119,7 @@ EXPOSE 2222
 VOLUME /data
 
 ARG BUDIBASE_VERSION
+# Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
 ENV BUDIBASE_VERSION=$VERSION
 
diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2
index 341fe5ee34..5035dc748f 100644
--- a/packages/server/Dockerfile.v2
+++ b/packages/server/Dockerfile.v2
@@ -68,6 +68,7 @@ COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
 
 ARG BUDIBASE_VERSION
+# Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
 ENV BUDIBASE_VERSION=$VERSION
 
diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2
index ed3817e89d..78eba2d3d8 100644
--- a/packages/worker/Dockerfile.v2
+++ b/packages/worker/Dockerfile.v2
@@ -51,6 +51,7 @@ ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
 ARG BUDIBASE_VERSION
+# Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
 ENV BUDIBASE_VERSION=$VERSION
 

From 3c85f1ce9e4b476062ffc5147f9c1e8bd73fc1f8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 23 Oct 2023 13:55:58 +0200
Subject: [PATCH 77/95] Fix after renames

---
 packages/backend-core/src/environment.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index a0905fb0f5..ed882fe96a 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -75,12 +75,12 @@ function getPackageJsonFields(): {
     const content = readFileSync(packageJsonFile!, "utf-8")
     const parsedContent = JSON.parse(content)
     return {
-      VERSION: process.env.VERSION || parsedContent.version,
+      VERSION: process.env.BUDIBASE_VERSION || parsedContent.version,
       SERVICE_NAME: parsedContent.name,
     }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return { VERSION: process.env.VERSION || "", SERVICE_NAME: "" }
+    return { VERSION: process.env.BUDIBASE_VERSION || "", SERVICE_NAME: "" }
   }
 }
 

From e3b342da57eac0f62b061f3c089a8f7c8f91bd0a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 23 Oct 2023 13:57:23 +0200
Subject: [PATCH 78/95] Fix after renames

---
 hosting/single/Dockerfile.v2  | 2 +-
 packages/server/Dockerfile.v2 | 2 +-
 packages/worker/Dockerfile.v2 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2
index 66ce0f1c4d..5b07a51b27 100644
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@@ -121,7 +121,7 @@ VOLUME /data
 ARG BUDIBASE_VERSION
 # Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
-ENV BUDIBASE_VERSION=$VERSION
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
 
 HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
 
diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2
index 5035dc748f..881c21299e 100644
--- a/packages/server/Dockerfile.v2
+++ b/packages/server/Dockerfile.v2
@@ -70,7 +70,7 @@ COPY packages/server/client/ client/
 ARG BUDIBASE_VERSION
 # Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
-ENV BUDIBASE_VERSION=$VERSION
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
 
 EXPOSE 4001
 
diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2
index 78eba2d3d8..a8be432827 100644
--- a/packages/worker/Dockerfile.v2
+++ b/packages/worker/Dockerfile.v2
@@ -53,6 +53,6 @@ ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 ARG BUDIBASE_VERSION
 # Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
-ENV BUDIBASE_VERSION=$VERSION
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
 
 CMD ["./docker_run.sh"]

From 453c0744be8d8b3721a2d760d56dcd616d8abf76 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 23 Oct 2023 15:20:44 +0200
Subject: [PATCH 79/95] Fix tests

---
 packages/server/src/tests/jestEnv.ts | 2 +-
 packages/worker/src/tests/jestEnv.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/tests/jestEnv.ts b/packages/server/src/tests/jestEnv.ts
index c1f32eeec9..4763208c54 100644
--- a/packages/server/src/tests/jestEnv.ts
+++ b/packages/server/src/tests/jestEnv.ts
@@ -9,4 +9,4 @@ process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"
-process.env.VERSION = "0.0.0+jest"
+process.env.BUDIBASE_VERSION = "0.0.0+jest"
diff --git a/packages/worker/src/tests/jestEnv.ts b/packages/worker/src/tests/jestEnv.ts
index f6de4e0fd2..9bd258c43c 100644
--- a/packages/worker/src/tests/jestEnv.ts
+++ b/packages/worker/src/tests/jestEnv.ts
@@ -10,4 +10,4 @@ process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.INTERNAL_API_KEY = "tet"
 process.env.DISABLE_ACCOUNT_PORTAL = "0"
 process.env.MOCK_REDIS = "1"
-process.env.VERSION = "0.0.0+jest"
+process.env.BUDIBASE_VERSION = "0.0.0+jest"

From bc4dd8e1c99fafb335dc79daa78e47ded56e9498 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Mon, 23 Oct 2023 15:43:44 +0200
Subject: [PATCH 80/95] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ee222c4694..f7e7cffe42 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ee222c4694c71716eeb1257f37467fc2231ee166
+Subproject commit f7e7cffe422086d9449c2075a74a378c16caff9d

From 7623ffd11e6340bb732b6e00398f4dc37862e1f9 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 23 Oct 2023 14:13:18 +0000
Subject: [PATCH 81/95] Bump version to 2.11.42

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 4b2bbb05d5..4fd5161891 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.41",
+  "version": "2.11.42",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From f643bc5ee8e94ba6c9db62b25a2320589699090f Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 24 Oct 2023 10:24:00 +0200
Subject: [PATCH 82/95] Keep only migrations

---
 packages/backend-core/src/users/users.ts      | 16 +++++++++++-
 packages/backend-core/src/users/utils.ts      |  1 +
 .../core/utilities/structures/licenses.ts     |  5 ++++
 .../tests/core/utilities/structures/quotas.ts |  5 ++--
 packages/pro                                  |  2 +-
 .../src/migrations/functions/syncQuotas.ts    |  2 ++
 .../functions/usageQuotas/syncCreators.ts     | 14 ++++++++++
 .../usageQuotas/tests/syncCreators.spec.ts    | 26 +++++++++++++++++++
 .../shared-core/src/sdk/documents/users.ts    | 25 ++++++++++++++++++
 packages/types/src/documents/global/quotas.ts |  1 +
 packages/types/src/sdk/licensing/quota.ts     |  2 ++
 11 files changed, 95 insertions(+), 4 deletions(-)
 create mode 100644 packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
 create mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts

diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index d259d5b560..6237c23972 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -21,8 +21,9 @@ import {
   User,
   DatabaseQueryOpts,
 } from "@budibase/types"
-import * as context from "../context"
 import { getGlobalDB } from "../context"
+import * as context from "../context"
+import { isCreator } from "./utils"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -286,6 +287,19 @@ export async function getUserCount() {
   return response.total_rows
 }
 
+export async function getCreatorCount() {
+  let creators = 0
+  async function iterate(startPage?: string) {
+    const page = await paginatedUsers({ bookmark: startPage })
+    creators += page.data.filter(isCreator).length
+    if (page.hasNextPage) {
+      await iterate(page.nextPage)
+    }
+  }
+  await iterate()
+  return creators
+}
+
 // used to remove the builder/admin permissions, for processing the
 // user as an app user (they may have some specific role/group
 export function removePortalUserPermissions(user: User | ContextUser) {
diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts
index af0e8e10c7..0ef4b77998 100644
--- a/packages/backend-core/src/users/utils.ts
+++ b/packages/backend-core/src/users/utils.ts
@@ -10,6 +10,7 @@ import { getAccountByTenantId } from "../accounts"
 // extract from shared-core to make easily accessible from backend-core
 export const isBuilder = sdk.users.isBuilder
 export const isAdmin = sdk.users.isAdmin
+export const isCreator = sdk.users.isCreator
 export const isGlobalBuilder = sdk.users.isGlobalBuilder
 export const isAdminOrBuilder = sdk.users.isAdminOrBuilder
 export const hasAdminPermissions = sdk.users.hasAdminPermissions
diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts
index 5cce84edfd..0e34f2e9bb 100644
--- a/packages/backend-core/tests/core/utilities/structures/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts
@@ -72,6 +72,11 @@ export function quotas(): Quotas {
           value: 1,
           triggers: [],
         },
+        creators: {
+          name: "Creators",
+          value: 1,
+          triggers: [],
+        },
         userGroups: {
           name: "User Groups",
           value: 1,
diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts
index e82117053f..8d0b05fe1e 100644
--- a/packages/backend-core/tests/core/utilities/structures/quotas.ts
+++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts
@@ -1,6 +1,6 @@
 import { MonthlyQuotaName, QuotaUsage } from "@budibase/types"
 
-export const usage = (): QuotaUsage => {
+export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
   return {
     _id: "usage_quota",
     quotaReset: new Date().toISOString(),
@@ -58,7 +58,8 @@ export const usage = (): QuotaUsage => {
     usageQuota: {
       apps: 0,
       plugins: 0,
-      users: 0,
+      users,
+      creators,
       userGroups: 0,
       rows: 0,
       triggers: {},
diff --git a/packages/pro b/packages/pro
index f7e7cffe42..271c1b38a5 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit f7e7cffe422086d9449c2075a74a378c16caff9d
+Subproject commit 271c1b38a54ba2fd787caa741e437b80d032c9c9
diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts
index 67f38ba929..83a7670e78 100644
--- a/packages/server/src/migrations/functions/syncQuotas.ts
+++ b/packages/server/src/migrations/functions/syncQuotas.ts
@@ -3,6 +3,7 @@ import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
 import * as syncPlugins from "./usageQuotas/syncPlugins"
 import * as syncUsers from "./usageQuotas/syncUsers"
+import * as syncCreators from "./usageQuotas/syncCreators"
 
 /**
  * Synchronise quotas to the state of the db.
@@ -13,5 +14,6 @@ export const run = async () => {
     await syncRows.run()
     await syncPlugins.run()
     await syncUsers.run()
+    await syncCreators.run()
   })
 }
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
new file mode 100644
index 0000000000..bcc94bf139
--- /dev/null
+++ b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
@@ -0,0 +1,14 @@
+import { users } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+
+export const run = async () => {
+  console.log("XXXXXXXXXXXXXXXXXXX2222222222")
+  const creatorCount = await users.getCreatorCount()
+  console.log(`Syncing creator count: ${creatorCount}`)
+  await quotas.setUsage(
+    creatorCount,
+    StaticQuotaName.CREATORS,
+    QuotaUsageType.STATIC
+  )
+}
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
new file mode 100644
index 0000000000..75fa9f217e
--- /dev/null
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
@@ -0,0 +1,26 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncCreators from "../syncCreators"
+import { quotas } from "@budibase/pro"
+
+describe("syncCreators", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("syncs creators", async () => {
+    return config.doInContext(null, async () => {
+      await config.createUser({ admin: true })
+
+      await syncCreators.run()
+
+      const usageDoc = await quotas.getQuotaUsage()
+      // default + additional creator
+      const creatorsCount = 2
+      expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
+    })
+  })
+})
diff --git a/packages/shared-core/src/sdk/documents/users.ts b/packages/shared-core/src/sdk/documents/users.ts
index 03d86daa85..b58994aa46 100644
--- a/packages/shared-core/src/sdk/documents/users.ts
+++ b/packages/shared-core/src/sdk/documents/users.ts
@@ -6,6 +6,7 @@ import {
   InternalTable,
 } from "@budibase/types"
 import { getProdAppID } from "./applications"
+import * as _ from "lodash/fp"
 
 // checks if a user is specifically a builder, given an app ID
 export function isBuilder(user: User | ContextUser, appId?: string): boolean {
@@ -58,6 +59,18 @@ export function hasAppBuilderPermissions(user?: User | ContextUser): boolean {
   return !isGlobalBuilder && appLength != null && appLength > 0
 }
 
+export function hasAppCreatorPermissions(user?: User | ContextUser): boolean {
+  if (!user) {
+    return false
+  }
+  return _.flow(
+    _.get("roles"),
+    _.values,
+    _.find(x => ["CREATOR", "ADMIN"].includes(x)),
+    x => !!x
+  )(user)
+}
+
 // checks if a user is capable of building any app
 export function hasBuilderPermissions(user?: User | ContextUser): boolean {
   if (!user) {
@@ -74,6 +87,18 @@ export function hasAdminPermissions(user?: User | ContextUser): boolean {
   return !!user.admin?.global
 }
 
+export function isCreator(user?: User | ContextUser): boolean {
+  if (!user) {
+    return false
+  }
+  return (
+    isGlobalBuilder(user) ||
+    hasAdminPermissions(user) ||
+    hasAppBuilderPermissions(user) ||
+    hasAppCreatorPermissions(user)
+  )
+}
+
 export function getGlobalUserID(userId?: string): string | undefined {
   if (typeof userId !== "string") {
     return userId
diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts
index 61410f7435..4eb1168f7d 100644
--- a/packages/types/src/documents/global/quotas.ts
+++ b/packages/types/src/documents/global/quotas.ts
@@ -32,6 +32,7 @@ export interface StaticUsage {
   [StaticQuotaName.APPS]: number
   [StaticQuotaName.PLUGINS]: number
   [StaticQuotaName.USERS]: number
+  [StaticQuotaName.CREATORS]: number
   [StaticQuotaName.USER_GROUPS]: number
   [StaticQuotaName.ROWS]: number
   triggers: {
diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 73afa1ed05..85700f167b 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -14,6 +14,7 @@ export enum StaticQuotaName {
   ROWS = "rows",
   APPS = "apps",
   USERS = "users",
+  CREATORS = "creators",
   USER_GROUPS = "userGroups",
   PLUGINS = "plugins",
 }
@@ -67,6 +68,7 @@ export type StaticQuotas = {
   [StaticQuotaName.ROWS]: Quota
   [StaticQuotaName.APPS]: Quota
   [StaticQuotaName.USERS]: Quota
+  [StaticQuotaName.CREATORS]: Quota
   [StaticQuotaName.USER_GROUPS]: Quota
   [StaticQuotaName.PLUGINS]: Quota
 }

From dbc903f3e69648b41cfa8d46de16c4785408d6ef Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 24 Oct 2023 12:03:07 +0200
Subject: [PATCH 83/95] Remove console.log

---
 .../server/src/migrations/functions/usageQuotas/syncCreators.ts  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
index bcc94bf139..ce53be925a 100644
--- a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
+++ b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
@@ -3,7 +3,6 @@ import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
-  console.log("XXXXXXXXXXXXXXXXXXX2222222222")
   const creatorCount = await users.getCreatorCount()
   console.log(`Syncing creator count: ${creatorCount}`)
   await quotas.setUsage(

From 5c1710b5dabd1eb44f26958c9cca6e110e1e66fc Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 24 Oct 2023 12:15:28 +0200
Subject: [PATCH 84/95] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 271c1b38a5..d24c0dc3a3 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 271c1b38a54ba2fd787caa741e437b80d032c9c9
+Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376

From 0610117d87b2be2843d78c3265f78fdc2a18824a Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 24 Oct 2023 10:54:33 +0000
Subject: [PATCH 85/95] Bump version to 2.11.43

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 4fd5161891..11e4e7627a 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.42",
+  "version": "2.11.43",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From b53158ac083046f0522855a3cb7628e6bd76d526 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 16:25:14 +0100
Subject: [PATCH 86/95] Moving the client library to be retrieved using the
 same mechanism as we do in development, through the server instance (same as
 the builder itself). This should help with the CDN issues that we have been
 having, although attachments are still currently affected.

---
 .../src/objectStore/buckets/app.ts            | 50 ++++++++-------
 .../src/objectStore/buckets/plugins.ts        | 16 ++---
 .../src/objectStore/objectStore.ts            | 62 ++++++++++++-------
 .../src/api/controllers/static/index.ts       | 49 +++++++++------
 4 files changed, 102 insertions(+), 75 deletions(-)

diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts
index be9fddeaa6..931cbeb78b 100644
--- a/packages/backend-core/src/objectStore/buckets/app.ts
+++ b/packages/backend-core/src/objectStore/buckets/app.ts
@@ -1,37 +1,41 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
+import qs from "querystring"
+
+export function clientLibraryPath(appId: string) {
+  return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+}
 
 /**
- * In production the client library is stored in the object store, however in development
- * we use the symlinked version produced by lerna, located in node modules. We link to this
- * via a specific endpoint (under /api/assets/client).
- * @param appId In production we need the appId to look up the correct bucket, as the
- * version of the client lib may differ between apps.
- * @param version The version to retrieve.
- * @return The URL to be inserted into appPackage response or server rendered
- * app index file.
+ * Previously we used to serve the client library directly from Cloudfront, however
+ * due to issues with the domain we were unable to continue doing this - keeping
+ * incase we are able to switch back to CDN path again in future.
  */
-export const clientLibraryUrl = (appId: string, version: string) => {
-  if (env.isProd()) {
-    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
-    if (env.CLOUDFRONT_CDN) {
-      // append app version to bust the cache
-      if (version) {
-        file += `?v=${version}`
-      }
-      // don't need to use presigned for client with cloudfront
-      // file is public
-      return cloudfront.getUrl(file)
-    } else {
-      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+export function clientLibraryCDNUrl(appId: string, version: string) {
+  let file = clientLibraryPath(appId)
+  if (env.CLOUDFRONT_CDN) {
+    // append app version to bust the cache
+    if (version) {
+      file += `?v=${version}`
     }
+    // don't need to use presigned for client with cloudfront
+    // file is public
+    return cloudfront.getUrl(file)
   } else {
-    return `/api/assets/client`
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
   }
 }
 
-export const getAppFileUrl = (s3Key: string) => {
+export function clientLibraryUrl(appId: string, version: string) {
+  const queryString = qs.encode({
+    appId,
+    version,
+  })
+  return `/api/assets/client?${queryString}`
+}
+
+export function getAppFileUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts
index f7721afb23..6f1b7116ae 100644
--- a/packages/backend-core/src/objectStore/buckets/plugins.ts
+++ b/packages/backend-core/src/objectStore/buckets/plugins.ts
@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export const enrichPluginURLs = (plugins: Plugin[]) => {
+export function enrichPluginURLs(plugins: Plugin[]) {
   if (!plugins || !plugins.length) {
     return []
   }
@@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => {
   })
 }
 
-const getPluginJSUrl = (plugin: Plugin) => {
+function getPluginJSUrl(plugin: Plugin) {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+function getPluginIconUrl(plugin: Plugin): string | undefined {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
@@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => {
   return getPluginUrl(s3Key)
 }
 
-const getPluginUrl = (s3Key: string) => {
+function getPluginUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
@@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => {
 
 // S3 KEYS
 
-export const getPluginJSKey = (plugin: Plugin) => {
+export function getPluginJSKey(plugin: Plugin) {
   return getPluginS3Key(plugin, "plugin.min.js")
 }
 
-export const getPluginIconKey = (plugin: Plugin) => {
+export function getPluginIconKey(plugin: Plugin) {
   // stored iconUrl is deprecated - hardcode to icon.svg in this case
   const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
   if (!iconFileName) {
@@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => {
   return getPluginS3Key(plugin, iconFileName)
 }
 
-const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+function getPluginS3Key(plugin: Plugin, fileName: string) {
   const s3Key = getPluginS3Dir(plugin.name)
   return `${s3Key}/${fileName}`
 }
 
-export const getPluginS3Dir = (pluginName: string) => {
+export function getPluginS3Dir(pluginName: string) {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
     const tenantId = context.getTenantId()
diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts
index c36a09915e..76d2dd6689 100644
--- a/packages/backend-core/src/objectStore/objectStore.ts
+++ b/packages/backend-core/src/objectStore/objectStore.ts
@@ -1,6 +1,6 @@
 const sanitize = require("sanitize-s3-objectkey")
 import AWS from "aws-sdk"
-import stream from "stream"
+import stream, { Readable } from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
 import zlib from "zlib"
@@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) {
  * @return an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (
+export function ObjectStore(
   bucket: string,
   opts: { presigning: boolean } = { presigning: false }
-) => {
+) {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -104,7 +104,7 @@ export const ObjectStore = (
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export const makeSureBucketExists = async (client: any, bucketName: string) => {
+export async function makeSureBucketExists(client: any, bucketName: string) {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-export const upload = async ({
+export async function upload({
   bucket: bucketName,
   filename,
   path,
   type,
   metadata,
-}: UploadParams) => {
+}: UploadParams) {
   const extension = filename.split(".").pop()
   const fileBytes = fs.readFileSync(path)
 
@@ -180,12 +180,12 @@ export const upload = async ({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export const streamUpload = async (
+export async function streamUpload(
   bucketName: string,
   filename: string,
   stream: any,
   extra = {}
-) => {
+) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
 
@@ -215,7 +215,7 @@ export const streamUpload = async (
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export const retrieve = async (bucketName: string, filepath: string) => {
+export async function retrieve(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => {
   }
 }
 
-export const listAllObjects = async (bucketName: string, path: string) => {
+export async function listAllObjects(bucketName: string, path: string) {
   const objectStore = ObjectStore(bucketName)
   const list = (params: ListParams = {}) => {
     return objectStore
@@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => {
 /**
  * Generate a presigned url with a default TTL of 1 hour
  */
-export const getPresignedUrl = (
+export function getPresignedUrl(
   bucketName: string,
   key: string,
   durationSeconds: number = 3600
-) => {
+) {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -291,7 +291,7 @@ export const getPresignedUrl = (
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-export const retrieveToTmp = async (bucketName: string, filepath: string) => {
+export async function retrieveToTmp(bucketName: string, filepath: string) {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
@@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => {
   return outputPath
 }
 
-export const retrieveDirectory = async (bucketName: string, path: string) => {
+export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
@@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => {
 /**
  * Delete a single file.
  */
-export const deleteFile = async (bucketName: string, filepath: string) => {
+export async function deleteFile(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   return objectStore.deleteObject(params).promise()
 }
 
-export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
+export async function deleteFiles(bucketName: string, filepaths: string[]) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
 /**
  * Delete a path, including everything within.
  */
-export const deleteFolder = async (
+export async function deleteFolder(
   bucketName: string,
   folder: string
-): Promise<any> => {
+): Promise<any> {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
   const client = ObjectStore(bucketName)
@@ -383,11 +383,11 @@ export const deleteFolder = async (
   }
 }
 
-export const uploadDirectory = async (
+export async function uploadDirectory(
   bucketName: string,
   localPath: string,
   bucketPath: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
@@ -404,11 +404,11 @@ export const uploadDirectory = async (
   return files
 }
 
-export const downloadTarballDirect = async (
+export async function downloadTarballDirect(
   url: string,
   path: string,
   headers = {}
-) => {
+) {
   path = sanitizeKey(path)
   const response = await fetch(url, { headers })
   if (!response.ok) {
@@ -418,11 +418,11 @@ export const downloadTarballDirect = async (
   await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
 }
 
-export const downloadTarball = async (
+export async function downloadTarball(
   url: string,
   bucketName: string,
   path: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
@@ -438,3 +438,17 @@ export const downloadTarball = async (
   // return the temporary path incase there is a use for it
   return tmpPath
 }
+
+export async function getReadStream(
+  bucketName: string,
+  path: string
+): Promise<Readable> {
+  bucketName = sanitizeBucket(bucketName)
+  path = sanitizeKey(path)
+  const client = ObjectStore(bucketName)
+  const params = {
+    Bucket: bucketName,
+    Key: path,
+  }
+  return client.getObject(params).createReadStream()
+}
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index bbf9dd34f5..d2e887dcf8 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -16,7 +16,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App } from "@budibase/types"
+import { App, Ctx } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   }
 }
 
-export const toggleBetaUiFeature = async function (ctx: any) {
+export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
   if (ctx.cookies.get(cookieName)) {
@@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) {
   }
 }
 
-export const serveBuilder = async function (ctx: any) {
+export const serveBuilder = async function (ctx: Ctx) {
   const builderPath = join(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: any) {
-  let files =
-    ctx.request.files.file.length > 1
-      ? Array.from(ctx.request.files.file)
-      : [ctx.request.files.file]
+export const uploadFile = async function (ctx: Ctx) {
+  const file = ctx.request?.files?.file
+  let files = file && Array.isArray(file) ? Array.from(file) : [file]
 
   const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
@@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) {
   ctx.body = await Promise.all(uploads)
 }
 
-export const deleteObjects = async function (ctx: any) {
+export const deleteObjects = async function (ctx: Ctx) {
   ctx.body = await objectStore.deleteFiles(
     ObjectStoreBuckets.APPS,
     ctx.request.body.keys
   )
 }
 
-export const serveApp = async function (ctx: any) {
+export const serveApp = async function (ctx: Ctx) {
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) {
   }
 }
 
-export const serveBuilderPreview = async function (ctx: any) {
+export const serveBuilderPreview = async function (ctx: Ctx) {
   const db = context.getAppDB({ skip_setup: true })
   const appInfo = await db.get<App>(DocumentType.APP_METADATA)
 
@@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) {
   }
 }
 
-export const serveClientLibrary = async function (ctx: any) {
+export const serveClientLibrary = async function (ctx: Ctx) {
+  const appId = context.getAppId() || (ctx.request.query.appId as string)
   let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
-  // incase running from TS directly
-  if (env.isDev() && !fs.existsSync(rootPath)) {
-    rootPath = join(require.resolve("@budibase/client"), "..")
+  if (!appId) {
+    ctx.throw(400, "No app ID provided - cannot fetch client library.")
+  }
+  if (env.isProd()) {
+    ctx.body = await objectStore.getReadStream(
+      ObjectStoreBuckets.APPS,
+      objectStore.clientLibraryPath(appId!)
+    )
+  } else if (env.isDev()) {
+    // incase running from TS directly
+    const tsPath = join(require.resolve("@budibase/client"), "..")
+    return send(ctx, "budibase-client.js", {
+      root: !fs.existsSync(rootPath) ? tsPath : rootPath,
+    })
+  } else {
+    ctx.throw(500, "Unable to retrieve client library.")
   }
-  return send(ctx, "budibase-client.js", {
-    root: rootPath,
-  })
 }
 
-export const getSignedUploadURL = async function (ctx: any) {
+export const getSignedUploadURL = async function (ctx: Ctx) {
   // Ensure datasource is valid
   let datasource
   try {
@@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) {
       const params = { Bucket: bucket, Key: key }
       signedUrl = s3.getSignedUrl("putObject", params)
       publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
-    } catch (error) {
+    } catch (error: any) {
       ctx.throw(400, error)
     }
   }

From 570eaa8e5347a41de09b78fb283b50c3d585def1 Mon Sep 17 00:00:00 2001
From: Mitch-Budibase <mitch@budibase.com>
Date: Tue, 24 Oct 2023 17:22:49 +0100
Subject: [PATCH 87/95] Update package.json + remove license.manage.spec.ts

Update package.json
- Updating "test:self:ci"
- Now includes license. Needed to prevent license tests from running where an account portal is not setup

Removing license.manage.spec.ts
- This will be included in another PR
- Work required to finish this test
- Removing so the license activation test can be merged
---
 qa-core/package.json                          |  2 +-
 .../tests/licensing/license.manage.spec.ts    | 57 -------------------
 2 files changed, 1 insertion(+), 58 deletions(-)
 delete mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts

diff --git a/qa-core/package.json b/qa-core/package.json
index 3c789d89e6..d266ca9def 100644
--- a/qa-core/package.json
+++ b/qa-core/package.json
@@ -17,7 +17,7 @@
     "test:notify": "node scripts/testResultsWebhook",
     "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.",
     "test:cloud:qa": "yarn run test",
-    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\.",
+    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.",
     "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
     "serve": "start-server-and-test dev:built http://localhost:4001/health",
     "dev:built": "cd ../ && yarn dev:built"
diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
deleted file mode 100644
index 3f87838ee4..0000000000
--- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-import TestConfiguration from "../../config/TestConfiguration"
-import * as fixtures from "../../fixtures"
-import { Hosting, PlanType } from "@budibase/types"
-
-describe("license management", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await config.beforeAll()
-  })
-
-  afterAll(async () => {
-    await config.afterAll()
-  })
-
-  it("retrieves plans, creates checkout session, and updates license", async () => {
-    // Create cloud account
-    const createAccountRequest = fixtures.accounts.generateAccount({
-      hosting: Hosting.CLOUD,
-    })
-
-    // Self response has free license
-    const [selfRes, selfBody] = await config.api.accounts.self()
-    expect(selfBody.license.plan.type).toBe(PlanType.FREE)
-
-    // Retrieve plans
-    const [plansRes, planBody] = await config.api.licenses.getPlans()
-
-    // Select priceId from premium plan
-    let premiumPriceId = null
-    for (const plan of planBody) {
-      if (plan.type === PlanType.PREMIUM) {
-        premiumPriceId = plan.prices[0].priceId
-        break
-      }
-    }
-
-    // Create checkout session for price
-    const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
-      premiumPriceId
-    )
-    const checkoutSessionUrl = checkoutSessionRes[1].url
-    expect(checkoutSessionUrl).toContain("checkout.stripe.com")
-
-    // TODO: Mimic checkout success
-    // Create stripe customer
-    // Create subscription for premium plan
-    // Assert license updated from free to premium
-
-    // Create portal session
-    //await config.api.stripe.createPortalSession()
-
-    // Update from free to business license
-
-    // License updated
-  })
-})

From ba5b1d7cfa0b9f9fc228a3d22c64b8b8bb53934a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 17:27:58 +0100
Subject: [PATCH 88/95] Adding tenant ID to query string and fixing test cases.

---
 .../src/objectStore/buckets/app.ts            | 19 ++++++---
 .../src/objectStore/buckets/tests/app.spec.ts | 42 +++++++------------
 2 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts
index 931cbeb78b..95b73b9104 100644
--- a/packages/backend-core/src/objectStore/buckets/app.ts
+++ b/packages/backend-core/src/objectStore/buckets/app.ts
@@ -2,6 +2,7 @@ import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
 import qs from "querystring"
+import { getTenantId, DEFAULT_TENANT_ID } from "../../context"
 
 export function clientLibraryPath(appId: string) {
   return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
@@ -28,11 +29,19 @@ export function clientLibraryCDNUrl(appId: string, version: string) {
 }
 
 export function clientLibraryUrl(appId: string, version: string) {
-  const queryString = qs.encode({
-    appId,
-    version,
-  })
-  return `/api/assets/client?${queryString}`
+  let tenantId, qsParams: { appId: string; version: string; tenantId?: string }
+  try {
+    tenantId = getTenantId()
+  } finally {
+    qsParams = {
+      appId,
+      version,
+    }
+  }
+  if (tenantId) {
+    qsParams.tenantId = tenantId
+  }
+  return `/api/assets/client?${qs.encode(qsParams)}`
 }
 
 export function getAppFileUrl(s3Key: string) {
diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
index aaa07ec9d3..cbbbee6255 100644
--- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
+++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
@@ -1,5 +1,4 @@
 import * as app from "../app"
-import { getAppFileUrl } from "../app"
 import { testEnv } from "../../../../tests/extra"
 
 describe("app", () => {
@@ -7,6 +6,15 @@ describe("app", () => {
     testEnv.nodeJest()
   })
 
+  function baseCheck(url: string, tenantId?: string) {
+    expect(url).toContain("/api/assets/client")
+    if (tenantId) {
+      expect(url).toContain(`tenantId=${tenantId}`)
+    }
+    expect(url).toContain("appId=app_123")
+    expect(url).toContain("version=2.0.0")
+  }
+
   describe("clientLibraryUrl", () => {
     function getClientUrl() {
       return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
@@ -20,31 +28,19 @@ describe("app", () => {
       it("gets url in dev", () => {
         testEnv.nodeDev()
         const url = getClientUrl()
-        expect(url).toBe("/api/assets/client")
-      })
-
-      it("gets url with embedded minio", () => {
-        testEnv.withMinio()
-        const url = getClientUrl()
-        expect(url).toBe(
-          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
+        baseCheck(url)
       })
 
       it("gets url with custom S3", () => {
         testEnv.withS3()
         const url = getClientUrl()
-        expect(url).toBe(
-          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
+        baseCheck(url)
       })
 
       it("gets url with cloudfront + s3", () => {
         testEnv.withCloudfront()
         const url = getClientUrl()
-        expect(url).toBe(
-          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-        )
+        baseCheck(url)
       })
     })
 
@@ -57,7 +53,7 @@ describe("app", () => {
         testEnv.nodeDev()
         await testEnv.withTenant(tenantId => {
           const url = getClientUrl()
-          expect(url).toBe("/api/assets/client")
+          baseCheck(url, tenantId)
         })
       })
 
@@ -65,9 +61,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withMinio()
           const url = getClientUrl()
-          expect(url).toBe(
-            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
+          baseCheck(url, tenantId)
         })
       })
 
@@ -75,9 +69,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withS3()
           const url = getClientUrl()
-          expect(url).toBe(
-            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
+          baseCheck(url, tenantId)
         })
       })
 
@@ -85,9 +77,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withCloudfront()
           const url = getClientUrl()
-          expect(url).toBe(
-            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-          )
+          baseCheck(url, tenantId)
         })
       })
     })

From 701d30abb4f3c292fbddd5ec50cf4f8c31fae789 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 17:30:57 +0100
Subject: [PATCH 89/95] Don't pass through default tenant ID.

---
 packages/backend-core/src/objectStore/buckets/app.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts
index 95b73b9104..43bc965c65 100644
--- a/packages/backend-core/src/objectStore/buckets/app.ts
+++ b/packages/backend-core/src/objectStore/buckets/app.ts
@@ -2,7 +2,7 @@ import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
 import qs from "querystring"
-import { getTenantId, DEFAULT_TENANT_ID } from "../../context"
+import { DEFAULT_TENANT_ID, getTenantId } from "../../context"
 
 export function clientLibraryPath(appId: string) {
   return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
@@ -38,7 +38,7 @@ export function clientLibraryUrl(appId: string, version: string) {
       version,
     }
   }
-  if (tenantId) {
+  if (tenantId && tenantId !== DEFAULT_TENANT_ID) {
     qsParams.tenantId = tenantId
   }
   return `/api/assets/client?${qs.encode(qsParams)}`

From e2e5c9044c6fe708f520a1c66075f492c85f45b4 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 18:12:20 +0100
Subject: [PATCH 90/95] Revert "Client library fetching through server"

---
 .../src/objectStore/buckets/app.ts            | 59 +++++++-----------
 .../src/objectStore/buckets/plugins.ts        | 16 ++---
 .../src/objectStore/buckets/tests/app.spec.ts | 42 ++++++++-----
 .../src/objectStore/objectStore.ts            | 62 +++++++------------
 .../src/api/controllers/static/index.ts       | 49 ++++++---------
 5 files changed, 101 insertions(+), 127 deletions(-)

diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts
index 43bc965c65..be9fddeaa6 100644
--- a/packages/backend-core/src/objectStore/buckets/app.ts
+++ b/packages/backend-core/src/objectStore/buckets/app.ts
@@ -1,50 +1,37 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
-import qs from "querystring"
-import { DEFAULT_TENANT_ID, getTenantId } from "../../context"
-
-export function clientLibraryPath(appId: string) {
-  return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
-}
 
 /**
- * Previously we used to serve the client library directly from Cloudfront, however
- * due to issues with the domain we were unable to continue doing this - keeping
- * incase we are able to switch back to CDN path again in future.
+ * In production the client library is stored in the object store, however in development
+ * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * via a specific endpoint (under /api/assets/client).
+ * @param appId In production we need the appId to look up the correct bucket, as the
+ * version of the client lib may differ between apps.
+ * @param version The version to retrieve.
+ * @return The URL to be inserted into appPackage response or server rendered
+ * app index file.
  */
-export function clientLibraryCDNUrl(appId: string, version: string) {
-  let file = clientLibraryPath(appId)
-  if (env.CLOUDFRONT_CDN) {
-    // append app version to bust the cache
-    if (version) {
-      file += `?v=${version}`
+export const clientLibraryUrl = (appId: string, version: string) => {
+  if (env.isProd()) {
+    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+    if (env.CLOUDFRONT_CDN) {
+      // append app version to bust the cache
+      if (version) {
+        file += `?v=${version}`
+      }
+      // don't need to use presigned for client with cloudfront
+      // file is public
+      return cloudfront.getUrl(file)
+    } else {
+      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
     }
-    // don't need to use presigned for client with cloudfront
-    // file is public
-    return cloudfront.getUrl(file)
   } else {
-    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+    return `/api/assets/client`
   }
 }
 
-export function clientLibraryUrl(appId: string, version: string) {
-  let tenantId, qsParams: { appId: string; version: string; tenantId?: string }
-  try {
-    tenantId = getTenantId()
-  } finally {
-    qsParams = {
-      appId,
-      version,
-    }
-  }
-  if (tenantId && tenantId !== DEFAULT_TENANT_ID) {
-    qsParams.tenantId = tenantId
-  }
-  return `/api/assets/client?${qs.encode(qsParams)}`
-}
-
-export function getAppFileUrl(s3Key: string) {
+export const getAppFileUrl = (s3Key: string) => {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts
index 6f1b7116ae..f7721afb23 100644
--- a/packages/backend-core/src/objectStore/buckets/plugins.ts
+++ b/packages/backend-core/src/objectStore/buckets/plugins.ts
@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins: Plugin[]) {
+export const enrichPluginURLs = (plugins: Plugin[]) => {
   if (!plugins || !plugins.length) {
     return []
   }
@@ -17,12 +17,12 @@ export function enrichPluginURLs(plugins: Plugin[]) {
   })
 }
 
-function getPluginJSUrl(plugin: Plugin) {
+const getPluginJSUrl = (plugin: Plugin) => {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-function getPluginIconUrl(plugin: Plugin): string | undefined {
+const getPluginIconUrl = (plugin: Plugin): string | undefined => {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
@@ -30,7 +30,7 @@ function getPluginIconUrl(plugin: Plugin): string | undefined {
   return getPluginUrl(s3Key)
 }
 
-function getPluginUrl(s3Key: string) {
+const getPluginUrl = (s3Key: string) => {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
@@ -40,11 +40,11 @@ function getPluginUrl(s3Key: string) {
 
 // S3 KEYS
 
-export function getPluginJSKey(plugin: Plugin) {
+export const getPluginJSKey = (plugin: Plugin) => {
   return getPluginS3Key(plugin, "plugin.min.js")
 }
 
-export function getPluginIconKey(plugin: Plugin) {
+export const getPluginIconKey = (plugin: Plugin) => {
   // stored iconUrl is deprecated - hardcode to icon.svg in this case
   const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
   if (!iconFileName) {
@@ -53,12 +53,12 @@ export function getPluginIconKey(plugin: Plugin) {
   return getPluginS3Key(plugin, iconFileName)
 }
 
-function getPluginS3Key(plugin: Plugin, fileName: string) {
+const getPluginS3Key = (plugin: Plugin, fileName: string) => {
   const s3Key = getPluginS3Dir(plugin.name)
   return `${s3Key}/${fileName}`
 }
 
-export function getPluginS3Dir(pluginName: string) {
+export const getPluginS3Dir = (pluginName: string) => {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
     const tenantId = context.getTenantId()
diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
index cbbbee6255..aaa07ec9d3 100644
--- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
+++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
@@ -1,4 +1,5 @@
 import * as app from "../app"
+import { getAppFileUrl } from "../app"
 import { testEnv } from "../../../../tests/extra"
 
 describe("app", () => {
@@ -6,15 +7,6 @@ describe("app", () => {
     testEnv.nodeJest()
   })
 
-  function baseCheck(url: string, tenantId?: string) {
-    expect(url).toContain("/api/assets/client")
-    if (tenantId) {
-      expect(url).toContain(`tenantId=${tenantId}`)
-    }
-    expect(url).toContain("appId=app_123")
-    expect(url).toContain("version=2.0.0")
-  }
-
   describe("clientLibraryUrl", () => {
     function getClientUrl() {
       return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
@@ -28,19 +20,31 @@ describe("app", () => {
       it("gets url in dev", () => {
         testEnv.nodeDev()
         const url = getClientUrl()
-        baseCheck(url)
+        expect(url).toBe("/api/assets/client")
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
       })
 
       it("gets url with custom S3", () => {
         testEnv.withS3()
         const url = getClientUrl()
-        baseCheck(url)
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
       })
 
       it("gets url with cloudfront + s3", () => {
         testEnv.withCloudfront()
         const url = getClientUrl()
-        baseCheck(url)
+        expect(url).toBe(
+          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+        )
       })
     })
 
@@ -53,7 +57,7 @@ describe("app", () => {
         testEnv.nodeDev()
         await testEnv.withTenant(tenantId => {
           const url = getClientUrl()
-          baseCheck(url, tenantId)
+          expect(url).toBe("/api/assets/client")
         })
       })
 
@@ -61,7 +65,9 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withMinio()
           const url = getClientUrl()
-          baseCheck(url, tenantId)
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
         })
       })
 
@@ -69,7 +75,9 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withS3()
           const url = getClientUrl()
-          baseCheck(url, tenantId)
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
         })
       })
 
@@ -77,7 +85,9 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withCloudfront()
           const url = getClientUrl()
-          baseCheck(url, tenantId)
+          expect(url).toBe(
+            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+          )
         })
       })
     })
diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts
index 76d2dd6689..c36a09915e 100644
--- a/packages/backend-core/src/objectStore/objectStore.ts
+++ b/packages/backend-core/src/objectStore/objectStore.ts
@@ -1,6 +1,6 @@
 const sanitize = require("sanitize-s3-objectkey")
 import AWS from "aws-sdk"
-import stream, { Readable } from "stream"
+import stream from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
 import zlib from "zlib"
@@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) {
  * @return an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export function ObjectStore(
+export const ObjectStore = (
   bucket: string,
   opts: { presigning: boolean } = { presigning: false }
-) {
+) => {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -104,7 +104,7 @@ export function ObjectStore(
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export async function makeSureBucketExists(client: any, bucketName: string) {
+export const makeSureBucketExists = async (client: any, bucketName: string) => {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -139,13 +139,13 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-export async function upload({
+export const upload = async ({
   bucket: bucketName,
   filename,
   path,
   type,
   metadata,
-}: UploadParams) {
+}: UploadParams) => {
   const extension = filename.split(".").pop()
   const fileBytes = fs.readFileSync(path)
 
@@ -180,12 +180,12 @@ export async function upload({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export async function streamUpload(
+export const streamUpload = async (
   bucketName: string,
   filename: string,
   stream: any,
   extra = {}
-) {
+) => {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
 
@@ -215,7 +215,7 @@ export async function streamUpload(
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export async function retrieve(bucketName: string, filepath: string) {
+export const retrieve = async (bucketName: string, filepath: string) => {
   const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -230,7 +230,7 @@ export async function retrieve(bucketName: string, filepath: string) {
   }
 }
 
-export async function listAllObjects(bucketName: string, path: string) {
+export const listAllObjects = async (bucketName: string, path: string) => {
   const objectStore = ObjectStore(bucketName)
   const list = (params: ListParams = {}) => {
     return objectStore
@@ -261,11 +261,11 @@ export async function listAllObjects(bucketName: string, path: string) {
 /**
  * Generate a presigned url with a default TTL of 1 hour
  */
-export function getPresignedUrl(
+export const getPresignedUrl = (
   bucketName: string,
   key: string,
   durationSeconds: number = 3600
-) {
+) => {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -291,7 +291,7 @@ export function getPresignedUrl(
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-export async function retrieveToTmp(bucketName: string, filepath: string) {
+export const retrieveToTmp = async (bucketName: string, filepath: string) => {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
@@ -300,7 +300,7 @@ export async function retrieveToTmp(bucketName: string, filepath: string) {
   return outputPath
 }
 
-export async function retrieveDirectory(bucketName: string, path: string) {
+export const retrieveDirectory = async (bucketName: string, path: string) => {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
@@ -324,7 +324,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
 /**
  * Delete a single file.
  */
-export async function deleteFile(bucketName: string, filepath: string) {
+export const deleteFile = async (bucketName: string, filepath: string) => {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -334,7 +334,7 @@ export async function deleteFile(bucketName: string, filepath: string) {
   return objectStore.deleteObject(params).promise()
 }
 
-export async function deleteFiles(bucketName: string, filepaths: string[]) {
+export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -349,10 +349,10 @@ export async function deleteFiles(bucketName: string, filepaths: string[]) {
 /**
  * Delete a path, including everything within.
  */
-export async function deleteFolder(
+export const deleteFolder = async (
   bucketName: string,
   folder: string
-): Promise<any> {
+): Promise<any> => {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
   const client = ObjectStore(bucketName)
@@ -383,11 +383,11 @@ export async function deleteFolder(
   }
 }
 
-export async function uploadDirectory(
+export const uploadDirectory = async (
   bucketName: string,
   localPath: string,
   bucketPath: string
-) {
+) => {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
@@ -404,11 +404,11 @@ export async function uploadDirectory(
   return files
 }
 
-export async function downloadTarballDirect(
+export const downloadTarballDirect = async (
   url: string,
   path: string,
   headers = {}
-) {
+) => {
   path = sanitizeKey(path)
   const response = await fetch(url, { headers })
   if (!response.ok) {
@@ -418,11 +418,11 @@ export async function downloadTarballDirect(
   await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
 }
 
-export async function downloadTarball(
+export const downloadTarball = async (
   url: string,
   bucketName: string,
   path: string
-) {
+) => {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
@@ -438,17 +438,3 @@ export async function downloadTarball(
   // return the temporary path incase there is a use for it
   return tmpPath
 }
-
-export async function getReadStream(
-  bucketName: string,
-  path: string
-): Promise<Readable> {
-  bucketName = sanitizeBucket(bucketName)
-  path = sanitizeKey(path)
-  const client = ObjectStore(bucketName)
-  const params = {
-    Bucket: bucketName,
-    Key: path,
-  }
-  return client.getObject(params).createReadStream()
-}
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index d2e887dcf8..bbf9dd34f5 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -16,7 +16,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx } from "@budibase/types"
+import { App } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   }
 }
 
-export const toggleBetaUiFeature = async function (ctx: Ctx) {
+export const toggleBetaUiFeature = async function (ctx: any) {
   const cookieName = `beta:${ctx.params.feature}`
 
   if (ctx.cookies.get(cookieName)) {
@@ -67,14 +67,16 @@ export const toggleBetaUiFeature = async function (ctx: Ctx) {
   }
 }
 
-export const serveBuilder = async function (ctx: Ctx) {
+export const serveBuilder = async function (ctx: any) {
   const builderPath = join(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: Ctx) {
-  const file = ctx.request?.files?.file
-  let files = file && Array.isArray(file) ? Array.from(file) : [file]
+export const uploadFile = async function (ctx: any) {
+  let files =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
 
   const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
@@ -91,14 +93,14 @@ export const uploadFile = async function (ctx: Ctx) {
   ctx.body = await Promise.all(uploads)
 }
 
-export const deleteObjects = async function (ctx: Ctx) {
+export const deleteObjects = async function (ctx: any) {
   ctx.body = await objectStore.deleteFiles(
     ObjectStoreBuckets.APPS,
     ctx.request.body.keys
   )
 }
 
-export const serveApp = async function (ctx: Ctx) {
+export const serveApp = async function (ctx: any) {
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -179,7 +181,7 @@ export const serveApp = async function (ctx: Ctx) {
   }
 }
 
-export const serveBuilderPreview = async function (ctx: Ctx) {
+export const serveBuilderPreview = async function (ctx: any) {
   const db = context.getAppDB({ skip_setup: true })
   const appInfo = await db.get<App>(DocumentType.APP_METADATA)
 
@@ -195,29 +197,18 @@ export const serveBuilderPreview = async function (ctx: Ctx) {
   }
 }
 
-export const serveClientLibrary = async function (ctx: Ctx) {
-  const appId = context.getAppId() || (ctx.request.query.appId as string)
+export const serveClientLibrary = async function (ctx: any) {
   let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
-  if (!appId) {
-    ctx.throw(400, "No app ID provided - cannot fetch client library.")
-  }
-  if (env.isProd()) {
-    ctx.body = await objectStore.getReadStream(
-      ObjectStoreBuckets.APPS,
-      objectStore.clientLibraryPath(appId!)
-    )
-  } else if (env.isDev()) {
-    // incase running from TS directly
-    const tsPath = join(require.resolve("@budibase/client"), "..")
-    return send(ctx, "budibase-client.js", {
-      root: !fs.existsSync(rootPath) ? tsPath : rootPath,
-    })
-  } else {
-    ctx.throw(500, "Unable to retrieve client library.")
+  // incase running from TS directly
+  if (env.isDev() && !fs.existsSync(rootPath)) {
+    rootPath = join(require.resolve("@budibase/client"), "..")
   }
+  return send(ctx, "budibase-client.js", {
+    root: rootPath,
+  })
 }
 
-export const getSignedUploadURL = async function (ctx: Ctx) {
+export const getSignedUploadURL = async function (ctx: any) {
   // Ensure datasource is valid
   let datasource
   try {
@@ -256,7 +247,7 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
       const params = { Bucket: bucket, Key: key }
       signedUrl = s3.getSignedUrl("putObject", params)
       publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
-    } catch (error: any) {
+    } catch (error) {
       ctx.throw(400, error)
     }
   }

From c6b377c5659067bb220006d94abfacc18a2b575b Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 18:13:36 +0100
Subject: [PATCH 91/95] Revert "Revert "Client library fetching through
 server""

---
 .../src/objectStore/buckets/app.ts            | 59 +++++++++++-------
 .../src/objectStore/buckets/plugins.ts        | 16 ++---
 .../src/objectStore/buckets/tests/app.spec.ts | 42 +++++--------
 .../src/objectStore/objectStore.ts            | 62 ++++++++++++-------
 .../src/api/controllers/static/index.ts       | 49 +++++++++------
 5 files changed, 127 insertions(+), 101 deletions(-)

diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts
index be9fddeaa6..43bc965c65 100644
--- a/packages/backend-core/src/objectStore/buckets/app.ts
+++ b/packages/backend-core/src/objectStore/buckets/app.ts
@@ -1,37 +1,50 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
+import qs from "querystring"
+import { DEFAULT_TENANT_ID, getTenantId } from "../../context"
+
+export function clientLibraryPath(appId: string) {
+  return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+}
 
 /**
- * In production the client library is stored in the object store, however in development
- * we use the symlinked version produced by lerna, located in node modules. We link to this
- * via a specific endpoint (under /api/assets/client).
- * @param appId In production we need the appId to look up the correct bucket, as the
- * version of the client lib may differ between apps.
- * @param version The version to retrieve.
- * @return The URL to be inserted into appPackage response or server rendered
- * app index file.
+ * Previously we used to serve the client library directly from Cloudfront, however
+ * due to issues with the domain we were unable to continue doing this - keeping
+ * incase we are able to switch back to CDN path again in future.
  */
-export const clientLibraryUrl = (appId: string, version: string) => {
-  if (env.isProd()) {
-    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
-    if (env.CLOUDFRONT_CDN) {
-      // append app version to bust the cache
-      if (version) {
-        file += `?v=${version}`
-      }
-      // don't need to use presigned for client with cloudfront
-      // file is public
-      return cloudfront.getUrl(file)
-    } else {
-      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+export function clientLibraryCDNUrl(appId: string, version: string) {
+  let file = clientLibraryPath(appId)
+  if (env.CLOUDFRONT_CDN) {
+    // append app version to bust the cache
+    if (version) {
+      file += `?v=${version}`
     }
+    // don't need to use presigned for client with cloudfront
+    // file is public
+    return cloudfront.getUrl(file)
   } else {
-    return `/api/assets/client`
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
   }
 }
 
-export const getAppFileUrl = (s3Key: string) => {
+export function clientLibraryUrl(appId: string, version: string) {
+  let tenantId, qsParams: { appId: string; version: string; tenantId?: string }
+  try {
+    tenantId = getTenantId()
+  } finally {
+    qsParams = {
+      appId,
+      version,
+    }
+  }
+  if (tenantId && tenantId !== DEFAULT_TENANT_ID) {
+    qsParams.tenantId = tenantId
+  }
+  return `/api/assets/client?${qs.encode(qsParams)}`
+}
+
+export function getAppFileUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts
index f7721afb23..6f1b7116ae 100644
--- a/packages/backend-core/src/objectStore/buckets/plugins.ts
+++ b/packages/backend-core/src/objectStore/buckets/plugins.ts
@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export const enrichPluginURLs = (plugins: Plugin[]) => {
+export function enrichPluginURLs(plugins: Plugin[]) {
   if (!plugins || !plugins.length) {
     return []
   }
@@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => {
   })
 }
 
-const getPluginJSUrl = (plugin: Plugin) => {
+function getPluginJSUrl(plugin: Plugin) {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+function getPluginIconUrl(plugin: Plugin): string | undefined {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
@@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => {
   return getPluginUrl(s3Key)
 }
 
-const getPluginUrl = (s3Key: string) => {
+function getPluginUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
@@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => {
 
 // S3 KEYS
 
-export const getPluginJSKey = (plugin: Plugin) => {
+export function getPluginJSKey(plugin: Plugin) {
   return getPluginS3Key(plugin, "plugin.min.js")
 }
 
-export const getPluginIconKey = (plugin: Plugin) => {
+export function getPluginIconKey(plugin: Plugin) {
   // stored iconUrl is deprecated - hardcode to icon.svg in this case
   const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
   if (!iconFileName) {
@@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => {
   return getPluginS3Key(plugin, iconFileName)
 }
 
-const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+function getPluginS3Key(plugin: Plugin, fileName: string) {
   const s3Key = getPluginS3Dir(plugin.name)
   return `${s3Key}/${fileName}`
 }
 
-export const getPluginS3Dir = (pluginName: string) => {
+export function getPluginS3Dir(pluginName: string) {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
     const tenantId = context.getTenantId()
diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
index aaa07ec9d3..cbbbee6255 100644
--- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
+++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
@@ -1,5 +1,4 @@
 import * as app from "../app"
-import { getAppFileUrl } from "../app"
 import { testEnv } from "../../../../tests/extra"
 
 describe("app", () => {
@@ -7,6 +6,15 @@ describe("app", () => {
     testEnv.nodeJest()
   })
 
+  function baseCheck(url: string, tenantId?: string) {
+    expect(url).toContain("/api/assets/client")
+    if (tenantId) {
+      expect(url).toContain(`tenantId=${tenantId}`)
+    }
+    expect(url).toContain("appId=app_123")
+    expect(url).toContain("version=2.0.0")
+  }
+
   describe("clientLibraryUrl", () => {
     function getClientUrl() {
       return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
@@ -20,31 +28,19 @@ describe("app", () => {
       it("gets url in dev", () => {
         testEnv.nodeDev()
         const url = getClientUrl()
-        expect(url).toBe("/api/assets/client")
-      })
-
-      it("gets url with embedded minio", () => {
-        testEnv.withMinio()
-        const url = getClientUrl()
-        expect(url).toBe(
-          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
+        baseCheck(url)
       })
 
       it("gets url with custom S3", () => {
         testEnv.withS3()
         const url = getClientUrl()
-        expect(url).toBe(
-          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
+        baseCheck(url)
       })
 
       it("gets url with cloudfront + s3", () => {
         testEnv.withCloudfront()
         const url = getClientUrl()
-        expect(url).toBe(
-          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-        )
+        baseCheck(url)
       })
     })
 
@@ -57,7 +53,7 @@ describe("app", () => {
         testEnv.nodeDev()
         await testEnv.withTenant(tenantId => {
           const url = getClientUrl()
-          expect(url).toBe("/api/assets/client")
+          baseCheck(url, tenantId)
         })
       })
 
@@ -65,9 +61,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withMinio()
           const url = getClientUrl()
-          expect(url).toBe(
-            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
+          baseCheck(url, tenantId)
         })
       })
 
@@ -75,9 +69,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withS3()
           const url = getClientUrl()
-          expect(url).toBe(
-            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
+          baseCheck(url, tenantId)
         })
       })
 
@@ -85,9 +77,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withCloudfront()
           const url = getClientUrl()
-          expect(url).toBe(
-            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-          )
+          baseCheck(url, tenantId)
         })
       })
     })
diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts
index c36a09915e..76d2dd6689 100644
--- a/packages/backend-core/src/objectStore/objectStore.ts
+++ b/packages/backend-core/src/objectStore/objectStore.ts
@@ -1,6 +1,6 @@
 const sanitize = require("sanitize-s3-objectkey")
 import AWS from "aws-sdk"
-import stream from "stream"
+import stream, { Readable } from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
 import zlib from "zlib"
@@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) {
  * @return an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (
+export function ObjectStore(
   bucket: string,
   opts: { presigning: boolean } = { presigning: false }
-) => {
+) {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -104,7 +104,7 @@ export const ObjectStore = (
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export const makeSureBucketExists = async (client: any, bucketName: string) => {
+export async function makeSureBucketExists(client: any, bucketName: string) {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-export const upload = async ({
+export async function upload({
   bucket: bucketName,
   filename,
   path,
   type,
   metadata,
-}: UploadParams) => {
+}: UploadParams) {
   const extension = filename.split(".").pop()
   const fileBytes = fs.readFileSync(path)
 
@@ -180,12 +180,12 @@ export const upload = async ({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export const streamUpload = async (
+export async function streamUpload(
   bucketName: string,
   filename: string,
   stream: any,
   extra = {}
-) => {
+) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
 
@@ -215,7 +215,7 @@ export const streamUpload = async (
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export const retrieve = async (bucketName: string, filepath: string) => {
+export async function retrieve(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => {
   }
 }
 
-export const listAllObjects = async (bucketName: string, path: string) => {
+export async function listAllObjects(bucketName: string, path: string) {
   const objectStore = ObjectStore(bucketName)
   const list = (params: ListParams = {}) => {
     return objectStore
@@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => {
 /**
  * Generate a presigned url with a default TTL of 1 hour
  */
-export const getPresignedUrl = (
+export function getPresignedUrl(
   bucketName: string,
   key: string,
   durationSeconds: number = 3600
-) => {
+) {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -291,7 +291,7 @@ export const getPresignedUrl = (
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-export const retrieveToTmp = async (bucketName: string, filepath: string) => {
+export async function retrieveToTmp(bucketName: string, filepath: string) {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
@@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => {
   return outputPath
 }
 
-export const retrieveDirectory = async (bucketName: string, path: string) => {
+export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
@@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => {
 /**
  * Delete a single file.
  */
-export const deleteFile = async (bucketName: string, filepath: string) => {
+export async function deleteFile(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   return objectStore.deleteObject(params).promise()
 }
 
-export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
+export async function deleteFiles(bucketName: string, filepaths: string[]) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
 /**
  * Delete a path, including everything within.
  */
-export const deleteFolder = async (
+export async function deleteFolder(
   bucketName: string,
   folder: string
-): Promise<any> => {
+): Promise<any> {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
   const client = ObjectStore(bucketName)
@@ -383,11 +383,11 @@ export const deleteFolder = async (
   }
 }
 
-export const uploadDirectory = async (
+export async function uploadDirectory(
   bucketName: string,
   localPath: string,
   bucketPath: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
@@ -404,11 +404,11 @@ export const uploadDirectory = async (
   return files
 }
 
-export const downloadTarballDirect = async (
+export async function downloadTarballDirect(
   url: string,
   path: string,
   headers = {}
-) => {
+) {
   path = sanitizeKey(path)
   const response = await fetch(url, { headers })
   if (!response.ok) {
@@ -418,11 +418,11 @@ export const downloadTarballDirect = async (
   await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
 }
 
-export const downloadTarball = async (
+export async function downloadTarball(
   url: string,
   bucketName: string,
   path: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
@@ -438,3 +438,17 @@ export const downloadTarball = async (
   // return the temporary path incase there is a use for it
   return tmpPath
 }
+
+export async function getReadStream(
+  bucketName: string,
+  path: string
+): Promise<Readable> {
+  bucketName = sanitizeBucket(bucketName)
+  path = sanitizeKey(path)
+  const client = ObjectStore(bucketName)
+  const params = {
+    Bucket: bucketName,
+    Key: path,
+  }
+  return client.getObject(params).createReadStream()
+}
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index bbf9dd34f5..d2e887dcf8 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -16,7 +16,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App } from "@budibase/types"
+import { App, Ctx } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   }
 }
 
-export const toggleBetaUiFeature = async function (ctx: any) {
+export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
   if (ctx.cookies.get(cookieName)) {
@@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) {
   }
 }
 
-export const serveBuilder = async function (ctx: any) {
+export const serveBuilder = async function (ctx: Ctx) {
   const builderPath = join(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: any) {
-  let files =
-    ctx.request.files.file.length > 1
-      ? Array.from(ctx.request.files.file)
-      : [ctx.request.files.file]
+export const uploadFile = async function (ctx: Ctx) {
+  const file = ctx.request?.files?.file
+  let files = file && Array.isArray(file) ? Array.from(file) : [file]
 
   const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
@@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) {
   ctx.body = await Promise.all(uploads)
 }
 
-export const deleteObjects = async function (ctx: any) {
+export const deleteObjects = async function (ctx: Ctx) {
   ctx.body = await objectStore.deleteFiles(
     ObjectStoreBuckets.APPS,
     ctx.request.body.keys
   )
 }
 
-export const serveApp = async function (ctx: any) {
+export const serveApp = async function (ctx: Ctx) {
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) {
   }
 }
 
-export const serveBuilderPreview = async function (ctx: any) {
+export const serveBuilderPreview = async function (ctx: Ctx) {
   const db = context.getAppDB({ skip_setup: true })
   const appInfo = await db.get<App>(DocumentType.APP_METADATA)
 
@@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) {
   }
 }
 
-export const serveClientLibrary = async function (ctx: any) {
+export const serveClientLibrary = async function (ctx: Ctx) {
+  const appId = context.getAppId() || (ctx.request.query.appId as string)
   let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
-  // incase running from TS directly
-  if (env.isDev() && !fs.existsSync(rootPath)) {
-    rootPath = join(require.resolve("@budibase/client"), "..")
+  if (!appId) {
+    ctx.throw(400, "No app ID provided - cannot fetch client library.")
+  }
+  if (env.isProd()) {
+    ctx.body = await objectStore.getReadStream(
+      ObjectStoreBuckets.APPS,
+      objectStore.clientLibraryPath(appId!)
+    )
+  } else if (env.isDev()) {
+    // incase running from TS directly
+    const tsPath = join(require.resolve("@budibase/client"), "..")
+    return send(ctx, "budibase-client.js", {
+      root: !fs.existsSync(rootPath) ? tsPath : rootPath,
+    })
+  } else {
+    ctx.throw(500, "Unable to retrieve client library.")
   }
-  return send(ctx, "budibase-client.js", {
-    root: rootPath,
-  })
 }
 
-export const getSignedUploadURL = async function (ctx: any) {
+export const getSignedUploadURL = async function (ctx: Ctx) {
   // Ensure datasource is valid
   let datasource
   try {
@@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) {
       const params = { Bucket: bucket, Key: key }
       signedUrl = s3.getSignedUrl("putObject", params)
       publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
-    } catch (error) {
+    } catch (error: any) {
       ctx.throw(400, error)
     }
   }

From 5fabe324345a4e20ae03130a8d7047ad33c5220b Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 20:35:25 +0100
Subject: [PATCH 92/95] Making sure client endpoint available in prod.

---
 packages/server/src/api/routes/static.ts | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts
index 0012764b40..bd3f1aba2e 100644
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@@ -27,15 +27,9 @@ router.param("file", async (file: any, ctx: any, next: any) => {
   return next()
 })
 
-// only used in development for retrieving the client library,
-// in production the client lib is always stored in the object store.
-if (env.isDev()) {
-  router.get("/api/assets/client", controller.serveClientLibrary)
-}
-
 router
-  // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
+  .get("/api/assets/client", controller.serveClientLibrary)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post(
     "/api/attachments/delete",

From 9983d67163e576d8e202b633017d394722efb84f Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 24 Oct 2023 22:13:56 +0100
Subject: [PATCH 93/95] Adding mimetype for client library response (correcting
 from octet-stream).

---
 packages/server/src/api/controllers/static/index.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index d2e887dcf8..984cb16c06 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -206,6 +206,7 @@ export const serveClientLibrary = async function (ctx: Ctx) {
       ObjectStoreBuckets.APPS,
       objectStore.clientLibraryPath(appId!)
     )
+    ctx.set("Content-Type", "application/javascript")
   } else if (env.isDev()) {
     // incase running from TS directly
     const tsPath = join(require.resolve("@budibase/client"), "..")

From 65111c81efacc95b9389f24a952ca8fed4c0c219 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Wed, 25 Oct 2023 08:44:59 +0200
Subject: [PATCH 94/95] Reactivate sync quotas migration

---
 packages/types/src/sdk/migrations.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts
index 4667ed0c8f..0692b27f8e 100644
--- a/packages/types/src/sdk/migrations.ts
+++ b/packages/types/src/sdk/migrations.ts
@@ -46,7 +46,7 @@ export enum MigrationName {
   GLOBAL_INFO_SYNC_USERS = "global_info_sync_users",
   TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions",
   // increment this number to re-activate this migration
-  SYNC_QUOTAS = "sync_quotas_1",
+  SYNC_QUOTAS = "sync_quotas_2",
 }
 
 export interface MigrationDefinition {

From 659cf3006024b938521f0fb18bbf07be3858e061 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 25 Oct 2023 08:59:28 +0000
Subject: [PATCH 95/95] Bump version to 2.11.44

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 11e4e7627a..7d14875c97 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.11.43",
+  "version": "2.11.44",
   "npmClient": "yarn",
   "packages": [
     "packages/*"