From f9e5a9f8ca127fe1d1cff799b1b86843709b1723 Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Tue, 13 Aug 2024 11:27:59 +0100
Subject: [PATCH 001/101] Added frontend type for premium

---
 packages/frontend-core/src/constants.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js
index 22e5e8583a..c2706c69b1 100644
--- a/packages/frontend-core/src/constants.js
+++ b/packages/frontend-core/src/constants.js
@@ -72,6 +72,7 @@ export const PlanType = {
   TEAM: "team",
   PRO: "pro",
   BUSINESS: "business",
+  PREMIUM: "premium",
   ENTERPRISE: "enterprise",
   ENTERPRISE_BASIC_TRIAL: "enterprise_basic_trial",
 }

From 7b3e02506dc7fc84312fedbb4ae05326c7b0f749 Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Tue, 13 Aug 2024 11:28:12 +0100
Subject: [PATCH 002/101] Changed from Business to premium

---
 packages/builder/src/components/common/RoleSelect.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 4605b0c182..6ed77f383c 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -74,7 +74,7 @@
         name: "Can edit",
         tag:
           !$licensing.perAppBuildersEnabled &&
-          capitalise(Constants.PlanType.BUSINESS),
+          capitalise(Constants.PlanType.PREMIUM),
       })
     }
 

From bd5789f59cbeada57ea9112ad5f1c338f4de3faa Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Tue, 13 Aug 2024 11:36:15 +0100
Subject: [PATCH 003/101] Changed label to enterprise.

---
 packages/builder/src/components/common/RoleSelect.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 6ed77f383c..49b349a00c 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -74,7 +74,7 @@
         name: "Can edit",
         tag:
           !$licensing.perAppBuildersEnabled &&
-          capitalise(Constants.PlanType.PREMIUM),
+          capitalise(Constants.PlanType.ENTERPRISE),
       })
     }
 

From 9a6a1abb84e999437137156e18f9fa89eb608d8b Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Tue, 13 Aug 2024 12:39:56 +0100
Subject: [PATCH 004/101] Fix logic, enterprise is the only tier that can use
 this

---
 .../src/components/common/RoleSelect.svelte   | 40 ++++++++++++++-----
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 49b349a00c..6eb7b705be 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -24,6 +24,7 @@
 
   const dispatch = createEventDispatcher()
   const RemoveID = "remove"
+  const subType = $licensing.license.plan.type
 
   $: enrichLabel = label => (labelPrefix ? `${labelPrefix} ${label}` : label)
   $: options = getOptions(
@@ -68,13 +69,19 @@
     }))
 
     // Add creator if required
-    if (allowCreator) {
+    if (
+      allowCreator ||
+      subType === Constants.PlanType.ENTERPRISE ||
+      subType === Constants.PlanType.ENTERPRISE_BASIC
+    ) {
       options.unshift({
         _id: Constants.Roles.CREATOR,
         name: "Can edit",
         tag:
-          !$licensing.perAppBuildersEnabled &&
-          capitalise(Constants.PlanType.ENTERPRISE),
+          subType === Constants.PlanType.ENTERPRISE ||
+          subType === Constants.PlanType.ENTERPRISE_BASIC
+            ? null
+            : capitalise(Constants.PlanType.ENTERPRISE),
       })
     }
 
@@ -134,9 +141,16 @@
     getOptionValue={role => role._id}
     getOptionColour={getColor}
     getOptionIcon={getIcon}
-    isOptionEnabled={option =>
-      option._id !== Constants.Roles.CREATOR ||
-      $licensing.perAppBuildersEnabled}
+    isOptionEnabled={option => {
+      if (option._id === Constants.Roles.CREATOR) {
+        return (
+          subType === Constants.PlanType.ENTERPRISE ||
+          (subType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL &&
+            $licensing.perAppBuildersEnabled)
+        )
+      }
+      return true
+    }}
     {placeholder}
     {error}
   />
@@ -154,10 +168,16 @@
     getOptionValue={role => role._id}
     getOptionColour={getColor}
     getOptionIcon={getIcon}
-    isOptionEnabled={option =>
-      (option._id !== Constants.Roles.CREATOR ||
-        $licensing.perAppBuildersEnabled) &&
-      option.enabled !== false}
+    isOptionEnabled={option => {
+      if (option._id === Constants.Roles.CREATOR) {
+        return (
+          subType === Constants.PlanType.ENTERPRISE ||
+          (subType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL &&
+            $licensing.perAppBuildersEnabled)
+        )
+      }
+      return option.enabled !== false
+    }}
     {placeholder}
     {error}
   />

From 7456596c7ac338405b54d5b533d7018c58321cb9 Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Tue, 13 Aug 2024 16:36:42 +0100
Subject: [PATCH 005/101] Refactor based on feedback

---
 .../src/components/common/RoleSelect.svelte   | 36 ++++++++-----------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 6eb7b705be..3a37e61138 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -24,7 +24,7 @@
 
   const dispatch = createEventDispatcher()
   const RemoveID = "remove"
-  const subType = $licensing.license.plan.type
+  const subType = $licensing.license.plan.type ?? null
 
   $: enrichLabel = label => (labelPrefix ? `${labelPrefix} ${label}` : label)
   $: options = getOptions(
@@ -69,19 +69,13 @@
     }))
 
     // Add creator if required
-    if (
-      allowCreator ||
-      subType === Constants.PlanType.ENTERPRISE ||
-      subType === Constants.PlanType.ENTERPRISE_BASIC
-    ) {
+    if (allowCreator || isEnterprisePlan(subType)) {
       options.unshift({
         _id: Constants.Roles.CREATOR,
         name: "Can edit",
-        tag:
-          subType === Constants.PlanType.ENTERPRISE ||
-          subType === Constants.PlanType.ENTERPRISE_BASIC
-            ? null
-            : capitalise(Constants.PlanType.ENTERPRISE),
+        tag: isEnterprisePlan(subType)
+          ? null
+          : capitalise(Constants.PlanType.ENTERPRISE),
       })
     }
 
@@ -124,6 +118,14 @@
       dispatch("change", e.detail)
     }
   }
+
+  function isEnterprisePlan(subType) {
+    return (
+      subType === Constants.PlanType.ENTERPRISE ||
+      subType === Constants.PlanType.ENTERPRISE_BASIC ||
+      subType === Constants.PlanType.ENTERPRISE_BASIC_trial
+    )
+  }
 </script>
 
 {#if fancySelect}
@@ -143,11 +145,7 @@
     getOptionIcon={getIcon}
     isOptionEnabled={option => {
       if (option._id === Constants.Roles.CREATOR) {
-        return (
-          subType === Constants.PlanType.ENTERPRISE ||
-          (subType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL &&
-            $licensing.perAppBuildersEnabled)
-        )
+        return isEnterprisePlan(subType)
       }
       return true
     }}
@@ -170,11 +168,7 @@
     getOptionIcon={getIcon}
     isOptionEnabled={option => {
       if (option._id === Constants.Roles.CREATOR) {
-        return (
-          subType === Constants.PlanType.ENTERPRISE ||
-          (subType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL &&
-            $licensing.perAppBuildersEnabled)
-        )
+        return isEnterprisePlan(subType)
       }
       return option.enabled !== false
     }}

From abaa40a2723c2bcbf65d96aaaceb294b5d14e5e4 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Tue, 10 Sep 2024 17:09:42 +0100
Subject: [PATCH 006/101] automation steps using names

---
 .../SetupPanel/AutomationBlockSetup.svelte    | 40 ++++++---
 .../src/helpers/automations/nameHelpers.js    | 85 +++++++++++++++++++
 .../builder/src/stores/builder/automations.js | 43 ++++++++--
 .../server/src/definitions/automations.ts     |  1 +
 packages/server/src/threads/automation.ts     | 31 ++++++-
 5 files changed, 180 insertions(+), 20 deletions(-)
 create mode 100644 packages/builder/src/helpers/automations/nameHelpers.js

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index 6c4865b539..1bd65c82f8 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -49,6 +49,7 @@
   import {
     getSchemaForDatasourcePlus,
     getEnvironmentBindings,
+    runtimeToReadableBinding,
   } from "dataBinding"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { onMount } from "svelte"
@@ -595,9 +596,13 @@
     let loopBlockCount = 0
     const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
       if (!name) return
-      const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock)
+      const runtimeBinding = determineRuntimeBinding(
+        name,
+        idx,
+        isLoopBlock,
+        bindingName
+      )
       const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
-
       bindings.push(
         createBindingObject(
           name,
@@ -613,7 +618,7 @@
       )
     }
 
-    const determineRuntimeBinding = (name, idx, isLoopBlock) => {
+    const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => {
       let runtimeName
 
       /* Begin special cases for generating custom schemas based on triggers */
@@ -634,12 +639,18 @@
       }
       /* End special cases for generating custom schemas based on triggers */
 
+      let hasUserDefinedName =
+        automation.stepNames?.[allSteps[idx - loopBlockCount]]?.id
       if (isLoopBlock) {
         runtimeName = `loop.${name}`
       } else if (block.name.startsWith("JS")) {
-        runtimeName = `steps[${idx - loopBlockCount}].${name}`
+        runtimeName = hasUserDefinedName
+          ? `stepsByName.${bindingName}.${name}`
+          : `steps[${idx - loopBlockCount}].${name}`
       } else {
-        runtimeName = `steps.${idx - loopBlockCount}.${name}`
+        runtimeName = hasUserDefinedName
+          ? `stepsByName.${bindingName}.${name}`
+          : `steps.${idx - loopBlockCount}.${name}`
       }
       return idx === 0 ? `trigger.${name}` : runtimeName
     }
@@ -666,11 +677,12 @@
       const field = Object.values(FIELDS).find(
         field => field.type === value.type && field.subtype === value.subtype
       )
-
+      console.log(bindingName)
       return {
-        readableBinding: bindingName
-          ? `${bindingName}.${name}`
-          : runtimeBinding,
+        readableBinding:
+          bindingName && !isLoopBlock
+            ? `steps.${bindingName}.${name}`
+            : runtimeBinding,
         runtimeBinding,
         type: value.type,
         description: value.description,
@@ -690,8 +702,15 @@
         allSteps[idx]?.stepId === ActionStepID.LOOP &&
         allSteps.some(x => x.blockToLoop === block.id)
       let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {}
+      if (wasLoopBlock) {
+        break
+      }
       let bindingName =
-        automation.stepNames?.[allSteps[idx - loopBlockCount].id]
+        automation.stepNames?.[allSteps[idx - loopBlockCount].id] ||
+        !isLoopBlock
+          ? allSteps[idx]?.name
+          : allSteps[idx - 1]?.name
+      console.log(idx == 4 && bindingName)
 
       if (isLoopBlock) {
         schema = {
@@ -746,7 +765,6 @@
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)
       )
     }
-
     return bindings
   }
 
diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js
new file mode 100644
index 0000000000..da5c999b2f
--- /dev/null
+++ b/packages/builder/src/helpers/automations/nameHelpers.js
@@ -0,0 +1,85 @@
+import { AutomationActionStepId } from "@budibase/types"
+
+export const updateBindingsInInputs = (inputs, oldName, newName) => {
+  if (typeof inputs === "string") {
+    return inputs.replace(
+      new RegExp(`stepsByName\\.${oldName}\\.`, "g"),
+      `stepsByName.${newName}.`
+    )
+  }
+
+  if (Array.isArray(inputs)) {
+    return inputs.map(item => updateBindingsInInputs(item, oldName, newName))
+  }
+
+  if (typeof inputs === "object" && inputs !== null) {
+    const updatedInputs = {}
+    for (const [key, value] of Object.entries(inputs)) {
+      updatedInputs[key] = updateBindingsInInputs(value, oldName, newName)
+    }
+    return updatedInputs
+  }
+
+  return inputs
+}
+
+export const updateBindingsInSteps = (steps, oldName, newName) => {
+  return steps.map(step => {
+    const updatedStep = {
+      ...step,
+      inputs: updateBindingsInInputs(step.inputs, oldName, newName),
+    }
+
+    // Handle branch steps
+    if ("branches" in updatedStep.inputs) {
+      updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({
+        ...branch,
+        condition: updateBindingsInInputs(branch.condition, oldName, newName),
+      }))
+
+      if (updatedStep.inputs.children) {
+        for (const [key, childSteps] of Object.entries(
+          updatedStep.inputs.children
+        )) {
+          updatedStep.inputs.children[key] = updateBindingsInSteps(
+            childSteps,
+            oldName,
+            newName
+          )
+        }
+      }
+    }
+
+    return updatedStep
+  })
+}
+
+export const getNewStepName = (automation, step) => {
+  const baseName = step.name
+
+  const countExistingSteps = steps => {
+    return steps.reduce((count, currentStep) => {
+      if (currentStep.name && currentStep.name.startsWith(baseName)) {
+        count++
+      }
+      if (
+        currentStep.stepId === AutomationActionStepId.BRANCH &&
+        currentStep.inputs &&
+        currentStep.inputs.children
+      ) {
+        Object.values(currentStep.inputs.children).forEach(branchSteps => {
+          count += countExistingSteps(branchSteps)
+        })
+      }
+      return count
+    }, 0)
+  }
+
+  const existingCount = countExistingSteps(automation.definition.steps)
+
+  if (existingCount === 0) {
+    return baseName
+  }
+
+  return `${baseName} ${existingCount + 1}`
+}
diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js
index fdb0991911..eaf565c4f2 100644
--- a/packages/builder/src/stores/builder/automations.js
+++ b/packages/builder/src/stores/builder/automations.js
@@ -6,6 +6,10 @@ import { createHistoryStore } from "stores/builder/history"
 import { notifications } from "@budibase/bbui"
 import { updateReferencesInObject } from "dataBinding"
 import { AutomationTriggerStepId } from "@budibase/types"
+import {
+  updateBindingsInSteps,
+  getNewStepName,
+} from "helpers/automations/nameHelpers"
 
 const initialAutomationState = {
   automations: [],
@@ -275,13 +279,17 @@ const automationActions = store => ({
     await store.actions.save(newAutomation)
   },
   constructBlock(type, stepId, blockDefinition) {
-    return {
+    let newName
+    const newStep = {
       ...blockDefinition,
       inputs: blockDefinition.inputs || {},
       stepId,
       type,
       id: generate(),
     }
+    newName = getNewStepName(get(selectedAutomation), newStep)
+    newStep.name = newName
+    return newStep
   },
   addBlockToAutomation: async (block, blockIdx) => {
     const automation = get(selectedAutomation)
@@ -301,15 +309,36 @@ const automationActions = store => ({
   saveAutomationName: async (blockId, name) => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)
-    if (!automation) {
+    if (!newAutomation) {
       return
     }
-    newAutomation.definition.stepNames = {
-      ...newAutomation.definition.stepNames,
-      [blockId]: name.trim(),
-    }
 
-    await store.actions.save(newAutomation)
+    const stepIndex = newAutomation.definition.steps.findIndex(
+      step => step.id === blockId
+    )
+
+    if (stepIndex !== -1) {
+      const oldName = newAutomation.definition.steps[stepIndex].name
+      const newName = name.trim()
+
+      // Update stepNames
+      newAutomation.definition.stepNames = {
+        ...newAutomation.definition.stepNames,
+        [blockId]: newName,
+      }
+
+      // Update the name in the step itself
+      newAutomation.definition.steps[stepIndex].name = newName
+
+      // Update bindings in all steps
+      newAutomation.definition.steps = updateBindingsInSteps(
+        newAutomation.definition.steps,
+        oldName,
+        newName
+      )
+
+      await store.actions.save(newAutomation)
+    }
   },
   deleteAutomationName: async blockId => {
     const automation = get(selectedAutomation)
diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts
index e84eecea51..6488e604e9 100644
--- a/packages/server/src/definitions/automations.ts
+++ b/packages/server/src/definitions/automations.ts
@@ -15,6 +15,7 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
+  stepsByName?: Record<string, any>
   env?: Record<string, string>
   trigger: any
 }
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index f374ff159a..3e5ac32b0a 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -89,7 +89,12 @@ class Orchestrator {
     delete triggerOutput.appId
     delete triggerOutput.metadata
     // step zero is never used as the template string is zero indexed for customer facing
-    this.context = { steps: [{}], trigger: triggerOutput }
+    this.context = {
+      steps: [{}],
+      stepsByName: {},
+      trigger: triggerOutput,
+    }
+
     this.automation = automation
     // create an emitter which has the chain count for this automation run in it, so it can block
     // excessive chaining if required
@@ -449,6 +454,11 @@ class Orchestrator {
         outputs: tempOutput,
         inputs: steps[stepToLoopIndex].inputs,
       })
+      console.log(this.context)
+
+      const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id
+      this.context.stepsByName![stepName] = tempOutput
+      console.log(this.context)
       this.context.steps[this.context.steps.length] = tempOutput
       this.context.steps = this.context.steps.filter(
         item => !item.hasOwnProperty.call(item, "currentItem")
@@ -542,8 +552,11 @@ class Orchestrator {
             loopIteration
           )
         }
+        console.log(this.context)
+
         const stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await processObject(originalStepInput, this.context)
+        let inputs = await this.testProcesss(originalStepInput, this.context)
+
         inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
 
         const outputs = await stepFn({
@@ -570,6 +583,18 @@ class Orchestrator {
     return null
   }
 
+  private async testProcesss(inputs: any, context: any) {
+    const processContext = {
+      ...context,
+      steps: {
+        ...context.steps,
+        ...context.stepsByName,
+      },
+    }
+
+    return processObject(inputs, processContext)
+  }
+
   private handleStepOutput(
     step: AutomationStep,
     outputs: any,
@@ -587,6 +612,8 @@ class Orchestrator {
     } else {
       this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       this.context.steps[this.context.steps.length] = outputs
+      const stepName = step.name || step.id
+      this.context.stepsByName![stepName] = outputs
     }
   }
 }

From ae1e2dd2f2e916550d7c04aa843691aab6fa1509 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 10 Sep 2024 17:41:33 +0100
Subject: [PATCH 007/101] Add column rename test.

---
 .../integrations/tests/googlesheets.spec.ts   | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 9781b97972..3b8ea49685 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -10,6 +10,7 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import { GoogleSheetsMock } from "./utils/googlesheets"
+import exp from "constants"
 
 describe("Google Sheets Integration", () => {
   const config = new TestConfiguration()
@@ -299,5 +300,42 @@ describe("Google Sheets Integration", () => {
       expect(mock.cell("A2")).toEqual("Test Contact Updated")
       expect(mock.cell("B2")).toEqual("original description updated")
     })
+
+    it("should be able to rename a column", async () => {
+      const row = await config.api.row.save(table._id!, {
+        name: "Test Contact",
+        description: "original description",
+      })
+
+      const { name, ...otherColumns } = table.schema
+      const renamedTable = await config.api.table.save({
+        ...table,
+        schema: {
+          ...otherColumns,
+          renamed: {
+            ...table.schema.name,
+          },
+        },
+        _rename: {
+          old: "name",
+          updated: "renamed",
+        },
+      })
+
+      expect(renamedTable.schema.name).not.toBeDefined()
+      expect(renamedTable.schema.renamed).toBeDefined()
+
+      expect(mock.cell("A1")).toEqual("renamed")
+      expect(mock.cell("B1")).toEqual("description")
+      expect(mock.cell("A2")).toEqual("Test Contact")
+      expect(mock.cell("B2")).toEqual("original description")
+      expect(mock.cell("A3")).toEqual(null)
+      expect(mock.cell("B3")).toEqual(null)
+
+      const renamedRow = await config.api.row.get(table._id!, row._id!)
+      expect(renamedRow.renamed).toEqual("Test Contact")
+      expect(renamedRow.description).toEqual("original description")
+      expect(renamedRow.name).not.toBeDefined()
+    })
   })
 })

From 69b8661d8a2e404b20be7f881ad05d6c4b7e4ac2 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 10 Sep 2024 18:29:11 +0100
Subject: [PATCH 008/101] Working toward delete tests.

---
 .../integrations/tests/googlesheets.spec.ts   | 116 +++++++++
 .../integrations/tests/utils/googlesheets.ts  | 231 ++++++++++++------
 2 files changed, 273 insertions(+), 74 deletions(-)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 3b8ea49685..3e7e0cddc4 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import {
   Datasource,
   FieldType,
+  Row,
   SourceName,
   Table,
   TableSourceType,
@@ -337,5 +338,120 @@ describe("Google Sheets Integration", () => {
       expect(renamedRow.description).toEqual("original description")
       expect(renamedRow.name).not.toBeDefined()
     })
+
+    // TODO: this gets the error "Sheet is not large enough to fit 27 columns. Resize the sheet first."
+    // eslint-disable-next-line jest/no-commented-out-tests
+    // it("should be able to add a new column", async () => {
+    //   const updatedTable = await config.api.table.save({
+    //     ...table,
+    //     schema: {
+    //       ...table.schema,
+    //       newColumn: {
+    //         name: "newColumn",
+    //         type: FieldType.STRING,
+    //       },
+    //     },
+    //   })
+
+    //   expect(updatedTable.schema.newColumn).toBeDefined()
+
+    //   expect(mock.cell("A1")).toEqual("name")
+    //   expect(mock.cell("B1")).toEqual("description")
+    //   expect(mock.cell("C1")).toEqual("newColumn")
+    // })
+
+    it("should be able to delete a column", async () => {
+      const row = await config.api.row.save(table._id!, {
+        name: "Test Contact",
+        description: "original description",
+      })
+
+      const updatedTable = await config.api.table.save({
+        ...table,
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+        },
+      })
+
+      expect(updatedTable.schema.name).toBeDefined()
+      expect(updatedTable.schema.description).not.toBeDefined()
+
+      // TODO: we don't delete data in deleted columns yet, should we?
+      // expect(mock.cell("A1")).toEqual("name")
+      // expect(mock.cell("B1")).toEqual(null)
+
+      const updatedRow = await config.api.row.get(table._id!, row._id!)
+      expect(updatedRow.name).toEqual("Test Contact")
+      expect(updatedRow.description).not.toBeDefined()
+    })
+  })
+
+  describe("delete", () => {
+    let table: Table
+    beforeEach(async () => {
+      table = await config.api.table.save({
+        name: "Test Table",
+        type: "table",
+        sourceId: datasource._id!,
+        sourceType: TableSourceType.EXTERNAL,
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+            constraints: {
+              type: "string",
+            },
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+            constraints: {
+              type: "string",
+            },
+          },
+        },
+      })
+
+      await config.api.row.bulkImport(table._id!, {
+        rows: [
+          {
+            name: "Test Contact 1",
+            description: "original description 1",
+          },
+          {
+            name: "Test Contact 2",
+            description: "original description 2",
+          },
+        ],
+      })
+    })
+
+    it.skip("can delete a table", async () => {
+      await config.api.table.destroy(table._id!, table._rev!)
+      expect(mock.cell("A1")).toEqual(null)
+      expect(mock.cell("B1")).toEqual(null)
+    })
+
+    it.skip("can delete a row", async () => {
+      const rows = await config.api.row.fetch(table._id!)
+      expect(rows.length).toEqual(2)
+
+      for (const row of rows) {
+        await config.api.row.delete(table._id!, { _id: row._id! })
+      }
+
+      expect(mock.cell("A1")).toEqual("name")
+      expect(mock.cell("B1")).toEqual("description")
+      expect(mock.cell("A2")).toEqual(null)
+      expect(mock.cell("B2")).toEqual(null)
+      expect(mock.cell("A3")).toEqual(null)
+      expect(mock.cell("B3")).toEqual(null)
+
+      const emptyRows = await config.api.row.fetch(table._id!)
+      expect(emptyRows.length).toEqual(0)
+    })
   })
 })
diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts
index c58066bee5..90f3b3652c 100644
--- a/packages/server/src/integrations/tests/utils/googlesheets.ts
+++ b/packages/server/src/integrations/tests/utils/googlesheets.ts
@@ -21,6 +21,7 @@ import type {
   CellFormat,
   CellPadding,
   Color,
+  GridRange,
 } from "google-spreadsheet/src/lib/types/sheets-types"
 
 const BLACK: Color = { red: 0, green: 0, blue: 0 }
@@ -88,11 +89,32 @@ interface UpdateValuesResponse {
   updatedData: ValueRange
 }
 
+// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest
+interface AddSheetRequest {
+  properties: WorksheetProperties
+}
+
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse
 interface AddSheetResponse {
   properties: WorksheetProperties
 }
 
+interface DeleteRangeRequest {
+  range: GridRange
+  shiftDimension: WorksheetDimension
+}
+
+// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request
+interface BatchUpdateRequest {
+  requests: {
+    addSheet?: AddSheetRequest
+    deleteRange?: DeleteRangeRequest
+  }[]
+  includeSpreadsheetInResponse: boolean
+  responseRanges: string[]
+  responseIncludeGridData: boolean
+}
+
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response
 interface BatchUpdateResponse {
   spreadsheetId: string
@@ -102,23 +124,6 @@ interface BatchUpdateResponse {
   updatedSpreadsheet: Spreadsheet
 }
 
-// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest
-interface AddSheetRequest {
-  properties: WorksheetProperties
-}
-
-interface Request {
-  addSheet?: AddSheetRequest
-}
-
-// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request
-interface BatchUpdateRequest {
-  requests: Request[]
-  includeSpreadsheetInResponse: boolean
-  responseRanges: string[]
-  responseIncludeGridData: boolean
-}
-
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData
 interface RowData {
   values: CellData[]
@@ -369,13 +374,17 @@ export class GoogleSheetsMock {
 
   private handleValueAppend(request: AppendRequest): AppendResponse {
     const { range, params, body } = request
-    const { sheet, bottomRight } = this.parseA1Notation(range)
+    const { sheetId, endRowIndex } = this.parseA1Notation(range)
+    const sheet = this.getSheetById(sheetId)
+    if (!sheet) {
+      throw new Error(`Sheet ${sheetId} not found`)
+    }
 
     const newRows = body.values.map(v => this.valuesToRowData(v))
     const toDelete =
       params.insertDataOption === "INSERT_ROWS" ? newRows.length : 0
-    sheet.data[0].rowData.splice(bottomRight.row + 1, toDelete, ...newRows)
-    sheet.data[0].rowMetadata.splice(bottomRight.row + 1, toDelete, {
+    sheet.data[0].rowData.splice(endRowIndex + 1, toDelete, ...newRows)
+    sheet.data[0].rowMetadata.splice(endRowIndex + 1, toDelete, {
       hiddenByUser: false,
       hiddenByFilter: false,
       pixelSize: 100,
@@ -384,17 +393,13 @@ export class GoogleSheetsMock {
 
     // It's important to give back a correct updated range because the API
     // library we use makes use of it to assign the correct row IDs to rows.
-    const updatedRange = this.createA1FromRanges(
-      sheet,
-      {
-        row: bottomRight.row + 1,
-        column: 0,
-      },
-      {
-        row: bottomRight.row + newRows.length,
-        column: 0,
-      }
-    )
+    const updatedRange = this.createA1({
+      sheetId,
+      startRowIndex: endRowIndex + 1,
+      startColumnIndex: 0,
+      endRowIndex: endRowIndex + newRows.length,
+      endColumnIndex: 0,
+    })
 
     return {
       spreadsheetId: this.spreadsheet.spreadsheetId,
@@ -438,6 +443,9 @@ export class GoogleSheetsMock {
           addSheet: this.handleAddSheet(request.addSheet),
         })
       }
+      if (request.deleteRange) {
+        this.handleDeleteRange(request.deleteRange)
+      }
     }
 
     return response
@@ -474,12 +482,24 @@ export class GoogleSheetsMock {
     return { properties: properties as WorksheetProperties }
   }
 
+  private handleDeleteRange(request: DeleteRangeRequest) {
+    const { range, shiftDimension } = request
+
+    if (shiftDimension !== "ROWS") {
+      throw new Error("Only row-based deletes are supported")
+    }
+
+    this.iterateRange(range, (cell, value) => {
+      cell.userEnteredValue = this.createValue(null)
+    })
+  }
+
   private handleGetSpreadsheet(): Spreadsheet {
     return this.spreadsheet
   }
 
   private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse {
-    this.iterateCells(valueRange, (cell, value) => {
+    this.iterateValueRange(valueRange, (cell, value) => {
       cell.userEnteredValue = this.createValue(value)
     })
 
@@ -494,7 +514,30 @@ export class GoogleSheetsMock {
     return response
   }
 
-  private iterateCells(
+  private iterateRange(
+    range: Required<GridRange>,
+    cb: (cell: CellData) => void
+  ) {
+    const {
+      sheetId,
+      startRowIndex,
+      endRowIndex,
+      startColumnIndex,
+      endColumnIndex,
+    } = range
+
+    for (let row = startRowIndex; row <= endRowIndex; row++) {
+      for (let col = startColumnIndex; col <= endColumnIndex; col++) {
+        const cell = this.getCellNumericIndexes(sheetId, row, col)
+        if (!cell) {
+          throw new Error("Cell not found")
+        }
+        cb(cell)
+      }
+    }
+  }
+
+  private iterateValueRange(
     valueRange: ValueRange,
     cb: (cell: CellData, value: Value) => void
   ) {
@@ -502,33 +545,46 @@ export class GoogleSheetsMock {
       throw new Error("Only row-major updates are supported")
     }
 
-    const { sheet, topLeft, bottomRight } = this.parseA1Notation(
-      valueRange.range
-    )
-    for (let row = topLeft.row; row <= bottomRight.row; row++) {
-      for (let col = topLeft.column; col <= bottomRight.column; col++) {
-        const cell = this.getCellNumericIndexes(sheet, row, col)
+    const {
+      sheetId,
+      startColumnIndex,
+      startRowIndex,
+      endColumnIndex,
+      endRowIndex,
+    } = this.parseA1Notation(valueRange.range)
+
+    for (let row = startRowIndex; row <= endRowIndex; row++) {
+      for (let col = startColumnIndex; col <= endColumnIndex; col++) {
+        const cell = this.getCellNumericIndexes(sheetId, row, col)
         if (!cell) {
           throw new Error("Cell not found")
         }
-        const value = valueRange.values[row - topLeft.row][col - topLeft.column]
+        const value =
+          valueRange.values[row - startRowIndex][col - startColumnIndex]
         cb(cell, value)
       }
     }
   }
 
   private getValueRange(range: string): ValueRange {
-    const { sheet, topLeft, bottomRight } = this.parseA1Notation(range)
+    const {
+      sheetId,
+      startRowIndex,
+      endRowIndex,
+      startColumnIndex,
+      endColumnIndex,
+    } = this.parseA1Notation(range)
+
     const valueRange: ValueRange = {
       range,
       majorDimension: "ROWS",
       values: [],
     }
 
-    for (let row = topLeft.row; row <= bottomRight.row; row++) {
+    for (let row = startRowIndex; row <= endRowIndex; row++) {
       const values: Value[] = []
-      for (let col = topLeft.column; col <= bottomRight.column; col++) {
-        const cell = this.getCellNumericIndexes(sheet, row, col)
+      for (let col = startColumnIndex; col <= endColumnIndex; col++) {
+        const cell = this.getCellNumericIndexes(sheetId, row, col)
         if (!cell) {
           throw new Error("Cell not found")
         }
@@ -693,11 +749,9 @@ export class GoogleSheetsMock {
   }
 
   private cellData(cell: string): CellData | undefined {
-    const {
-      sheet,
-      topLeft: { row, column },
-    } = this.parseA1Notation(cell)
-    return this.getCellNumericIndexes(sheet, row, column)
+    const { sheetId, startColumnIndex, startRowIndex } =
+      this.parseA1Notation(cell)
+    return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex)
   }
 
   cell(cell: string): Value | undefined {
@@ -709,10 +763,18 @@ export class GoogleSheetsMock {
   }
 
   private getCellNumericIndexes(
-    sheet: Sheet,
+    sheet: Sheet | number,
     row: number,
     column: number
   ): CellData | undefined {
+    if (typeof sheet === "number") {
+      const foundSheet = this.getSheetById(sheet)
+      if (!foundSheet) {
+        return undefined
+      }
+      sheet = foundSheet
+    }
+
     const data = sheet.data[0]
     const rowData = data.rowData[row]
     if (!rowData) {
@@ -751,11 +813,7 @@ export class GoogleSheetsMock {
   //   "Sheet1!A:B"    -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 1 } }
   //   "Sheet1!1:1"    -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 25 } }
   //   "Sheet1!1:2"    -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 1, column: 25 } }
-  private parseA1Notation(range: string): {
-    sheet: Sheet
-    topLeft: Range
-    bottomRight: Range
-  } {
+  private parseA1Notation(range: string): Required<GridRange> {
     let sheet: Sheet
     let rest: string
     if (!range.includes("!")) {
@@ -793,35 +851,54 @@ export class GoogleSheetsMock {
       parsedBottomRight = parsedTopLeft
     }
 
-    if (parsedTopLeft && parsedTopLeft.row === undefined) {
-      parsedTopLeft.row = 0
-    }
-    if (parsedTopLeft && parsedTopLeft.column === undefined) {
-      parsedTopLeft.column = 0
-    }
-    if (parsedBottomRight && parsedBottomRight.row === undefined) {
-      parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1
-    }
-    if (parsedBottomRight && parsedBottomRight.column === undefined) {
-      parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1
+    return this.ensureGridRange({
+      sheetId: sheet.properties.sheetId,
+      startRowIndex: parsedTopLeft.row,
+      endRowIndex: parsedBottomRight.row,
+      startColumnIndex: parsedTopLeft.column,
+      endColumnIndex: parsedBottomRight.column,
+    })
+  }
+
+  private ensureGridRange(range: GridRange): Required<GridRange> {
+    const sheet = this.getSheetById(range.sheetId)
+    if (!sheet) {
+      throw new Error(`Sheet ${range.sheetId} not found`)
     }
 
     return {
-      sheet,
-      topLeft: parsedTopLeft as Range,
-      bottomRight: parsedBottomRight as Range,
+      sheetId: range.sheetId,
+      startRowIndex: range.startRowIndex ?? 0,
+      endRowIndex:
+        range.endRowIndex ?? sheet.properties.gridProperties.rowCount - 1,
+      startColumnIndex: range.startColumnIndex ?? 0,
+      endColumnIndex:
+        range.endColumnIndex ?? sheet.properties.gridProperties.columnCount - 1,
     }
   }
 
-  private createA1FromRanges(sheet: Sheet, topLeft: Range, bottomRight: Range) {
+  private createA1(range: Required<GridRange>) {
+    const {
+      sheetId,
+      startColumnIndex,
+      startRowIndex,
+      endColumnIndex,
+      endRowIndex,
+    } = range
+
+    const sheet = this.getSheetById(sheetId)
+    if (!sheet) {
+      throw new Error(`Sheet ${range.sheetId} not found`)
+    }
+
     let title = sheet.properties.title
     if (title.includes(" ")) {
       title = `'${title}'`
     }
-    const topLeftLetter = this.numberToLetter(topLeft.column)
-    const bottomRightLetter = this.numberToLetter(bottomRight.column)
-    const topLeftRow = topLeft.row + 1
-    const bottomRightRow = bottomRight.row + 1
+    const topLeftLetter = this.numberToLetter(startColumnIndex)
+    const bottomRightLetter = this.numberToLetter(endColumnIndex)
+    const topLeftRow = startRowIndex + 1
+    const bottomRightRow = endRowIndex + 1
     return `${title}!${topLeftLetter}${topLeftRow}:${bottomRightLetter}${bottomRightRow}`
   }
 
@@ -860,4 +937,10 @@ export class GoogleSheetsMock {
       sheet => sheet.properties.title === name
     )
   }
+
+  private getSheetById(id: number): Sheet | undefined {
+    return this.spreadsheet.sheets.find(
+      sheet => sheet.properties.sheetId === id
+    )
+  }
 }

From 8a5d0560eb6bff84067d181478f6d42fa653faec Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Wed, 11 Sep 2024 10:01:20 +0100
Subject: [PATCH 009/101] Fixed PlanType typo

---
 packages/builder/src/components/common/RoleSelect.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte
index 3a37e61138..6006b8ab8d 100644
--- a/packages/builder/src/components/common/RoleSelect.svelte
+++ b/packages/builder/src/components/common/RoleSelect.svelte
@@ -123,7 +123,7 @@
     return (
       subType === Constants.PlanType.ENTERPRISE ||
       subType === Constants.PlanType.ENTERPRISE_BASIC ||
-      subType === Constants.PlanType.ENTERPRISE_BASIC_trial
+      subType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL
     )
   }
 </script>

From bf98d61ea6fbab5f9f8a51bf1cc4b930abd79259 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 11:35:46 +0100
Subject: [PATCH 010/101] add tests for new binding update code

---
 .../SetupPanel/AutomationBlockSetup.svelte    |   3 -
 .../src/helpers/automations/nameHelpers.js    |  59 ++++--
 .../src/helpers/tests/nameHelpers.spec.js     | 177 ++++++++++++++++++
 .../builder/src/stores/builder/automations.js |   6 +-
 4 files changed, 224 insertions(+), 21 deletions(-)
 create mode 100644 packages/builder/src/helpers/tests/nameHelpers.spec.js

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index 1bd65c82f8..1e2e409243 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -49,7 +49,6 @@
   import {
     getSchemaForDatasourcePlus,
     getEnvironmentBindings,
-    runtimeToReadableBinding,
   } from "dataBinding"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { onMount } from "svelte"
@@ -677,7 +676,6 @@
       const field = Object.values(FIELDS).find(
         field => field.type === value.type && field.subtype === value.subtype
       )
-      console.log(bindingName)
       return {
         readableBinding:
           bindingName && !isLoopBlock
@@ -710,7 +708,6 @@
         !isLoopBlock
           ? allSteps[idx]?.name
           : allSteps[idx - 1]?.name
-      console.log(idx == 4 && bindingName)
 
       if (isLoopBlock) {
         schema = {
diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js
index da5c999b2f..b863796a54 100644
--- a/packages/builder/src/helpers/automations/nameHelpers.js
+++ b/packages/builder/src/helpers/automations/nameHelpers.js
@@ -1,40 +1,71 @@
 import { AutomationActionStepId } from "@budibase/types"
 
-export const updateBindingsInInputs = (inputs, oldName, newName) => {
+export const updateBindingsInInputs = (inputs, oldName, newName, stepIndex) => {
   if (typeof inputs === "string") {
-    return inputs.replace(
-      new RegExp(`stepsByName\\.${oldName}\\.`, "g"),
-      `stepsByName.${newName}.`
-    )
+    return inputs
+      .replace(
+        new RegExp(`stepsByName\\.${oldName}\\.`, "g"),
+        `stepsByName.${newName}.`
+      )
+      .replace(
+        new RegExp(`steps\\.${stepIndex}\\.`, "g"),
+        `stepsByName.${newName}.`
+      )
   }
 
   if (Array.isArray(inputs)) {
-    return inputs.map(item => updateBindingsInInputs(item, oldName, newName))
+    return inputs.map(item =>
+      updateBindingsInInputs(item, oldName, newName, stepIndex)
+    )
   }
 
   if (typeof inputs === "object" && inputs !== null) {
     const updatedInputs = {}
     for (const [key, value] of Object.entries(inputs)) {
-      updatedInputs[key] = updateBindingsInInputs(value, oldName, newName)
+      const updatedKey = updateBindingsInInputs(
+        key,
+        oldName,
+        newName,
+        stepIndex
+      )
+      updatedInputs[updatedKey] = updateBindingsInInputs(
+        value,
+        oldName,
+        newName,
+        stepIndex
+      )
     }
     return updatedInputs
   }
-
   return inputs
 }
 
-export const updateBindingsInSteps = (steps, oldName, newName) => {
+export const updateBindingsInSteps = (
+  steps,
+  oldName,
+  newName,
+  changedStepIndex
+) => {
   return steps.map(step => {
     const updatedStep = {
       ...step,
-      inputs: updateBindingsInInputs(step.inputs, oldName, newName),
+      inputs: updateBindingsInInputs(
+        step.inputs,
+        oldName,
+        newName,
+        changedStepIndex
+      ),
     }
 
-    // Handle branch steps
     if ("branches" in updatedStep.inputs) {
       updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({
         ...branch,
-        condition: updateBindingsInInputs(branch.condition, oldName, newName),
+        condition: updateBindingsInInputs(
+          branch.condition,
+          oldName,
+          newName,
+          changedStepIndex
+        ),
       }))
 
       if (updatedStep.inputs.children) {
@@ -44,7 +75,8 @@ export const updateBindingsInSteps = (steps, oldName, newName) => {
           updatedStep.inputs.children[key] = updateBindingsInSteps(
             childSteps,
             oldName,
-            newName
+            newName,
+            changedStepIndex
           )
         }
       }
@@ -53,7 +85,6 @@ export const updateBindingsInSteps = (steps, oldName, newName) => {
     return updatedStep
   })
 }
-
 export const getNewStepName = (automation, step) => {
   const baseName = step.name
 
diff --git a/packages/builder/src/helpers/tests/nameHelpers.spec.js b/packages/builder/src/helpers/tests/nameHelpers.spec.js
new file mode 100644
index 0000000000..1ce2d1987a
--- /dev/null
+++ b/packages/builder/src/helpers/tests/nameHelpers.spec.js
@@ -0,0 +1,177 @@
+import { cloneDeep } from "lodash"
+import {
+  updateBindingsInInputs,
+  updateBindingsInSteps,
+} from "../automations/nameHelpers"
+describe("Automation Binding Update Functions", () => {
+  const sampleAutomation = {
+    definition: {
+      steps: [
+        {
+          name: "First Step",
+          inputs: {
+            text: "Starting automation",
+          },
+          id: "step1",
+        },
+        {
+          name: "Second Step",
+          inputs: {
+            text: "{{ steps.0.success }} and {{ stepsByName.First Step.message }}",
+          },
+          id: "step2",
+        },
+        {
+          name: "Branch",
+          inputs: {
+            branches: [
+              {
+                name: "branch1",
+                condition: {
+                  equal: {
+                    "steps.1.success": true,
+                  },
+                },
+              },
+            ],
+            children: {
+              branch1: [
+                {
+                  name: "Nested Step",
+                  inputs: {
+                    text: "{{ stepsByName.Second Step.message }} and {{ steps.1.success }}",
+                  },
+                  id: "nestedStep",
+                },
+              ],
+            },
+          },
+          id: "branchStep",
+        },
+      ],
+      stepNames: {
+        step1: "First Step",
+        step2: "Second Step",
+        branchStep: "Branch",
+      },
+    },
+  }
+
+  it("updateBindingsInInputs updates string bindings correctly", () => {
+    const input = "{{ stepsByName.oldName.success }} and {{ steps.1.message }}"
+    const result = updateBindingsInInputs(input, "oldName", "newName", 1)
+    expect(result).toBe(
+      "{{ stepsByName.newName.success }} and {{ stepsByName.newName.message }}"
+    )
+  })
+
+  it("updateBindingsInInputs handles nested objects", () => {
+    const input = {
+      text: "{{ stepsByName.oldName.success }}",
+      nested: {
+        value: "{{ steps.1.message }}",
+      },
+    }
+    const result = updateBindingsInInputs(input, "oldName", "newName", 1)
+    expect(result).toEqual({
+      text: "{{ stepsByName.newName.success }}",
+      nested: {
+        value: "{{ stepsByName.newName.message }}",
+      },
+    })
+  })
+
+  it("updateBindingsInSteps updates bindings in all steps", () => {
+    const steps = cloneDeep(sampleAutomation.definition.steps)
+    const result = updateBindingsInSteps(
+      steps,
+      "Second Step",
+      "Renamed Step",
+      1
+    )
+
+    expect(result[1].name).toBe("Second Step")
+
+    expect(result[2].inputs.branches[0].condition.equal).toEqual({
+      "stepsByName.Renamed Step.success": true,
+    })
+
+    const nestedStepText = result[2].inputs.children.branch1[0].inputs.text
+    expect(nestedStepText).toBe(
+      "{{ stepsByName.Renamed Step.message }} and {{ stepsByName.Renamed Step.success }}"
+    )
+  })
+
+  it("updateBindingsInSteps handles steps with no bindings", () => {
+    const steps = [
+      {
+        name: "No Binding Step",
+        inputs: {
+          text: "Plain text",
+        },
+        id: "noBindingStep",
+      },
+    ]
+    const result = updateBindingsInSteps(steps, "Old Name", "New Name", 0)
+    expect(result).toEqual(steps)
+  })
+
+  it("updateBindingsInSteps updates bindings in deeply nested branches", () => {
+    const deeplyNestedStep = {
+      name: "Deep Branch",
+      inputs: {
+        branches: [
+          {
+            name: "deepBranch",
+            condition: {
+              equal: {
+                "stepsByName.Second Step.success": true,
+              },
+            },
+          },
+        ],
+        children: {
+          deepBranch: [
+            {
+              name: "Deep Log",
+              inputs: {
+                text: "{{ steps.1.message }}",
+              },
+            },
+          ],
+        },
+      },
+    }
+
+    const steps = [...sampleAutomation.definition.steps, deeplyNestedStep]
+    const result = updateBindingsInSteps(
+      steps,
+      "Second Step",
+      "Renamed Step",
+      1
+    )
+
+    expect(
+      result[3].inputs.branches[0].condition.equal[
+        "stepsByName.Renamed Step.success"
+      ]
+    ).toBe(true)
+    expect(result[3].inputs.children.deepBranch[0].inputs.text).toBe(
+      "{{ stepsByName.Renamed Step.message }}"
+    )
+  })
+
+  it("updateBindingsInSteps does not affect unrelated bindings", () => {
+    const steps = cloneDeep(sampleAutomation.definition.steps)
+    const result = updateBindingsInSteps(
+      steps,
+      "Second Step",
+      "Renamed Step",
+      1
+    )
+
+    expect(result[1].inputs.text).toBe(
+      "{{ steps.0.success }} and {{ stepsByName.First Step.message }}"
+    )
+  })
+})
diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js
index eaf565c4f2..6627c67080 100644
--- a/packages/builder/src/stores/builder/automations.js
+++ b/packages/builder/src/stores/builder/automations.js
@@ -321,20 +321,18 @@ const automationActions = store => ({
       const oldName = newAutomation.definition.steps[stepIndex].name
       const newName = name.trim()
 
-      // Update stepNames
       newAutomation.definition.stepNames = {
         ...newAutomation.definition.stepNames,
         [blockId]: newName,
       }
 
-      // Update the name in the step itself
       newAutomation.definition.steps[stepIndex].name = newName
 
-      // Update bindings in all steps
       newAutomation.definition.steps = updateBindingsInSteps(
         newAutomation.definition.steps,
         oldName,
-        newName
+        newName,
+        stepIndex
       )
 
       await store.actions.save(newAutomation)

From 2d4ac7fced1121dd49768426038372bfbed55e04 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 11:40:06 +0100
Subject: [PATCH 011/101] remove logs

---
 packages/server/src/threads/automation.ts | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 3e5ac32b0a..24833d0db9 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -454,11 +454,9 @@ class Orchestrator {
         outputs: tempOutput,
         inputs: steps[stepToLoopIndex].inputs,
       })
-      console.log(this.context)
 
       const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id
       this.context.stepsByName![stepName] = tempOutput
-      console.log(this.context)
       this.context.steps[this.context.steps.length] = tempOutput
       this.context.steps = this.context.steps.filter(
         item => !item.hasOwnProperty.call(item, "currentItem")
@@ -552,7 +550,6 @@ class Orchestrator {
             loopIteration
           )
         }
-        console.log(this.context)
 
         const stepFn = await this.getStepFunctionality(step.stepId)
         let inputs = await this.testProcesss(originalStepInput, this.context)

From 0c11924027595cdd12d2534f00cae0d1ab8d2f7e Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 11:40:52 +0100
Subject: [PATCH 012/101] refs

---
 packages/account-portal | 2 +-
 packages/pro            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index 048c37ecd9..c24374879d 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 048c37ecd921340614bf61a76a774aaa46176569
+Subproject commit c24374879d2b61516fabc24d7404e7da235be05e
diff --git a/packages/pro b/packages/pro
index ec1d2bda75..a4d1d15d9c 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c
+Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758

From 27b65e22efac6e78692a7c7990473bf59dc71dca Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 11:54:42 +0100
Subject: [PATCH 013/101] fix error with updating existinh name count

---
 packages/builder/src/helpers/automations/nameHelpers.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js
index b863796a54..ad18213ab0 100644
--- a/packages/builder/src/helpers/automations/nameHelpers.js
+++ b/packages/builder/src/helpers/automations/nameHelpers.js
@@ -105,9 +105,10 @@ export const getNewStepName = (automation, step) => {
       return count
     }, 0)
   }
-
-  const existingCount = countExistingSteps(automation.definition.steps)
-
+  let existingCount = 0
+  if (automation?.definition) {
+    existingCount = countExistingSteps(automation.definition.steps)
+  }
   if (existingCount === 0) {
     return baseName
   }

From e4918aea60752f69c032db921379d85e11b50063 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 11:55:31 +0100
Subject: [PATCH 014/101] refs

---
 packages/account-portal | 2 +-
 packages/pro            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index c24374879d..048c37ecd9 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit c24374879d2b61516fabc24d7404e7da235be05e
+Subproject commit 048c37ecd921340614bf61a76a774aaa46176569
diff --git a/packages/pro b/packages/pro
index a4d1d15d9c..ec1d2bda75 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758
+Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c

From 60dd500ecbf569dd70924332e39eb2738f360763 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 15:12:50 +0100
Subject: [PATCH 015/101] rename func

---
 packages/server/src/threads/automation.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 24833d0db9..288524b099 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -552,7 +552,10 @@ class Orchestrator {
         }
 
         const stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await this.testProcesss(originalStepInput, this.context)
+        let inputs = await this.addContextAndProcess(
+          originalStepInput,
+          this.context
+        )
 
         inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
 
@@ -580,7 +583,7 @@ class Orchestrator {
     return null
   }
 
-  private async testProcesss(inputs: any, context: any) {
+  private async addContextAndProcess(inputs: any, context: any) {
     const processContext = {
       ...context,
       steps: {

From d4d4f8a1501bf02fa45cc70f388642355328ce65 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Wed, 11 Sep 2024 15:24:49 +0100
Subject: [PATCH 016/101] ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 7899d07904..cd64894684 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 7899d07904d89d48954dd500da7b5dec32b781dd
+Subproject commit cd648946847aade0ac1f53a1498596670c8a0aee

From 47110e095fec3d1ed05afcdcb9f269dbaa78cf72 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 11 Sep 2024 17:42:08 +0100
Subject: [PATCH 017/101] updating budibase apps image to use alpine

---
 packages/server/Dockerfile | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile
index 30507fc827..2206d49d9b 100644
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20-slim
+FROM node:20-alpine
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
@@ -15,37 +15,35 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 ENV TOP_LEVEL_PATH=/
 
-# handle node-gyp
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ make python3 jq
-RUN yarn global add pm2
+# handle node-gyp and install postgres client for pg_dump utils
+RUN apk add --no-cache \
+    g++ \
+    make \
+    python3 \
+    jq \
+    bash \
+    postgresql-client \
+    git
 
-# Install postgres client for pg_dump utils
-RUN apt update && apt upgrade -y \
-    && apt install software-properties-common apt-transport-https curl gpg -y \
-    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
-    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
-    && apt update -y \
-    && apt install postgresql-client-15 -y \
-    && apt remove software-properties-common apt-transport-https curl gpg -y
+RUN yarn global add pm2
 
 WORKDIR /
 
 COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
 
-
 WORKDIR /app
 COPY packages/server/package.json .
 COPY packages/server/dist/yarn.lock .
 
 COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
-RUN ./scripts/removeWorkspaceDependencies.sh  package.json
+RUN ./scripts/removeWorkspaceDependencies.sh package.json
 
+# Install yarn packages with caching
 RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 \
-    # Remove unneeded data from file system to reduce image size
-    && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \
+    && yarn cache clean \
+    && apk del g++ make python3 jq \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
 
 COPY packages/server/dist/ dist/
@@ -69,7 +67,7 @@ EXPOSE 4001
 # due to this causing yarn to stop installing dev dependencies
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
-# this is required for isolated-vm to work on Node 20+
+# This is required for isolated-vm to work on Node 20+
 ENV NODE_OPTIONS="--no-node-snapshot"
 ENV CLUSTER_MODE=${CLUSTER_MODE}
 ENV TOP_LEVEL_PATH=/app

From e89042b2e3f924181c749f80d22e52d09d7482e8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 11 Sep 2024 18:06:05 +0100
Subject: [PATCH 018/101] Fixing some role typing issues, as well as fixing an
 issue with the validator not allowing the structure that roles are expected
 to have.

---
 packages/backend-core/src/security/roles.ts      |  6 +++---
 .../server/src/api/controllers/permission.ts     |  6 ++++--
 packages/server/src/api/controllers/role.ts      | 16 ++++++++++------
 .../server/src/api/routes/utils/validators.ts    | 10 ++++++++--
 packages/server/src/sdk/app/permissions/index.ts |  2 +-
 packages/types/src/documents/app/role.ts         |  3 ++-
 packages/types/src/documents/global/user.ts      |  5 ++---
 7 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index a64be6b319..50850219a3 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -44,7 +44,7 @@ export class Role implements RoleDoc {
   permissionId: string
   inherits?: string
   version?: string
-  permissions = {}
+  permissions: Record<string, PermissionLevel[]> = {}
 
   constructor(id: string, name: string, permissionId: string) {
     this._id = id
@@ -244,9 +244,9 @@ export async function getUserRoleHierarchy(
 // some templates/older apps will use a simple string instead of array for roles
 // convert the string to an array using the theory that write is higher than read
 export function checkForRoleResourceArray(
-  rolePerms: { [key: string]: string[] },
+  rolePerms: Record<string, PermissionLevel[]>,
   resourceId: string
-) {
+): Record<string, PermissionLevel[]> {
   if (rolePerms && !Array.isArray(rolePerms[resourceId])) {
     const permLevel = rolePerms[resourceId] as any
     rolePerms[resourceId] = [permLevel]
diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index cdfa6d8b1c..9444dfa251 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -75,7 +75,9 @@ async function updatePermissionOnRole(
   // resource from another role and then adding to the new role
   for (let role of dbRoles) {
     let updated = false
-    const rolePermissions = role.permissions ? role.permissions : {}
+    const rolePermissions: Record<string, PermissionLevel[]> = role.permissions
+      ? role.permissions
+      : {}
     // make sure its an array, also handle migrating
     if (
       !rolePermissions[resourceId] ||
@@ -83,7 +85,7 @@ async function updatePermissionOnRole(
     ) {
       rolePermissions[resourceId] =
         typeof rolePermissions[resourceId] === "string"
-          ? [rolePermissions[resourceId] as unknown as string]
+          ? [rolePermissions[resourceId] as unknown as PermissionLevel]
           : []
     }
     // handle the removal/updating the role which has this permission first
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 3398c8102c..605b462842 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -17,7 +17,7 @@ import {
   SaveRoleResponse,
   UserCtx,
   UserMetadata,
-  UserRoles,
+  DocumentType,
 } from "@budibase/types"
 import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
@@ -80,17 +80,21 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     _id = dbCore.prefixRoleID(_id)
   }
 
-  let dbRole
-  if (!isCreate) {
-    dbRole = await db.get<UserRoles>(_id)
+  let dbRole: Role | undefined
+  if (!isCreate && _id?.startsWith(DocumentType.ROLE)) {
+    dbRole = await db.get<Role>(_id)
   }
   if (dbRole && dbRole.name !== name && isNewVersion) {
     ctx.throw(400, "Cannot change custom role name")
   }
 
   const role = new roles.Role(_id, name, permissionId).addInheritance(inherits)
-  if (ctx.request.body._rev) {
-    role._rev = ctx.request.body._rev
+  if (dbRole?.permissions && !role.permissions) {
+    role.permissions = dbRole.permissions
+  }
+  const foundRev = ctx.request.body._rev || dbRole?._rev
+  if (foundRev) {
+    role._rev = foundRev
   }
   const result = await db.put(role)
   if (isCreate) {
diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index 5e2a585b4a..f0192b380b 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -200,7 +200,7 @@ export function webhookValidator() {
 
 export function roleValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-
+  const permissionString = Joi.string().valid(...permLevelArray)
   return auth.joiValidator.body(
     Joi.object({
       _id: OPTIONAL_STRING,
@@ -213,7 +213,13 @@ export function roleValidator() {
         .valid(...Object.values(permissions.BuiltinPermissionID))
         .required(),
       permissions: Joi.object()
-        .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+        .pattern(
+          /.*/,
+          Joi.alternatives().try(
+            Joi.array().items(permissionString),
+            permissionString
+          )
+        )
         .optional(),
       inherits: OPTIONAL_STRING,
     }).unknown(true)
diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts
index 18a376aaf0..dd4085d69e 100644
--- a/packages/server/src/sdk/app/permissions/index.ts
+++ b/packages/server/src/sdk/app/permissions/index.ts
@@ -90,7 +90,7 @@ export async function getResourcePerms(
       const rolePerms = allowsExplicitPerm
         ? roles.checkForRoleResourceArray(role.permissions || {}, resourceId)
         : {}
-      if (rolePerms[resourceId]?.indexOf(level) > -1) {
+      if (rolePerms[resourceId]?.indexOf(level as PermissionLevel) > -1) {
         permissions[level] = {
           role: roles.getExternalRoleID(role._id!, role.version),
           type: PermissionSource.EXPLICIT,
diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts
index f32ba810b0..7ccfb2e7e9 100644
--- a/packages/types/src/documents/app/role.ts
+++ b/packages/types/src/documents/app/role.ts
@@ -1,9 +1,10 @@
 import { Document } from "../document"
+import { PermissionLevel } from "../../sdk"
 
 export interface Role extends Document {
   permissionId: string
   inherits?: string
-  permissions: { [key: string]: string[] }
+  permissions: Record<string, PermissionLevel[]>
   version?: string
   name: string
 }
diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts
index af5c11374d..99e01fedf0 100644
--- a/packages/types/src/documents/global/user.ts
+++ b/packages/types/src/documents/global/user.ts
@@ -74,9 +74,8 @@ export enum UserStatus {
   INACTIVE = "inactive",
 }
 
-export interface UserRoles {
-  [key: string]: string
-}
+// specifies a map of app ID to role ID
+export type UserRoles = Record<string, string>
 
 // UTILITY TYPES
 

From 49a4e252c6cdf9b7b69dc11a8a3c7461e392c3fe Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 12 Sep 2024 09:38:11 +0100
Subject: [PATCH 019/101] Allow an account holder to login to account-portal if
 the tenant has been deleted. (#14547)

* Add email to session for populate user

* Add email param

* use param object

* Type fix for tests

* Fix test

* Temporarily remove account-portal tests
---
 .github/workflows/budibase_ci.yml             |  4 +--
 packages/backend-core/src/cache/user.ts       | 23 +++++++++----
 .../src/middleware/authenticated.ts           | 33 ++++++++++++++-----
 .../src/tests/utilities/TestConfiguration.ts  | 12 ++++---
 packages/server/src/threads/query.ts          |  4 ++-
 packages/server/src/utilities/global.ts       |  4 ++-
 .../rowProcessor/bbReferenceProcessor.ts      |  8 +++--
 .../src/utilities/rowProcessor/index.ts       |  4 ++-
 .../tests/bbReferenceProcessor.spec.ts        | 16 ++++++---
 packages/types/src/sdk/auth.ts                |  1 +
 packages/worker/src/sdk/auth/auth.ts          |  7 +++-
 .../worker/src/tests/TestConfiguration.ts     | 11 +++++--
 12 files changed, 94 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 4c5cc94d2b..044e29b445 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -117,9 +117,9 @@ jobs:
       - name: Test
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore @budibase/account-portal-server --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore @budibase/account-portal-server
           fi
 
   test-worker:
diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts
index d319c5dcb6..9de09a431d 100644
--- a/packages/backend-core/src/cache/user.ts
+++ b/packages/backend-core/src/cache/user.ts
@@ -63,14 +63,25 @@ async function populateUsersFromDB(
  * If not present fallback to loading the user directly and re-caching.
  * @param userId the id of the user to get
  * @param tenantId the tenant of the user to get
+ * @param email the email of the user to populate from account if needed
  * @param populateUser function to provide the user for re-caching. default to couch db
  * @returns
  */
-export async function getUser(
-  userId: string,
-  tenantId?: string,
-  populateUser?: (userId: string, tenantId: string) => Promise<User>
-) {
+export async function getUser({
+  userId,
+  tenantId,
+  email,
+  populateUser,
+}: {
+  userId: string
+  email?: string
+  tenantId?: string
+  populateUser?: (
+    userId: string,
+    tenantId: string,
+    email?: string
+  ) => Promise<User>
+}) {
   if (!populateUser) {
     populateUser = populateFromDB
   }
@@ -85,7 +96,7 @@ export async function getUser(
   // try cache
   let user: User = await client.get(userId)
   if (!user) {
-    user = await populateUser(userId, tenantId)
+    user = await populateUser(userId, tenantId, email)
     await client.store(userId, user, EXPIRY_SECONDS)
   }
   if (user && !user.tenantId && tenantId) {
diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts
index 51cd4ec2af..85aa2293ef 100644
--- a/packages/backend-core/src/middleware/authenticated.ts
+++ b/packages/backend-core/src/middleware/authenticated.ts
@@ -43,7 +43,11 @@ function finalise(ctx: any, opts: FinaliseOpts = {}) {
 
 async function checkApiKey(
   apiKey: string,
-  populateUser?: (userId: string, tenantId: string) => Promise<User>
+  populateUser?: (
+    userId: string,
+    tenantId: string,
+    email?: string
+  ) => Promise<User>
 ) {
   // check both the primary and the fallback internal api keys
   // this allows for rotation
@@ -70,7 +74,11 @@ async function checkApiKey(
     if (userId) {
       return {
         valid: true,
-        user: await getUser(userId, tenantId, populateUser),
+        user: await getUser({
+          userId,
+          tenantId,
+          populateUser,
+        }),
       }
     } else {
       throw new InvalidAPIKeyError()
@@ -123,13 +131,18 @@ export default function (
           // getting session handles error checking (if session exists etc)
           session = await getSession(userId, sessionId)
           if (opts && opts.populateUser) {
-            user = await getUser(
+            user = await getUser({
               userId,
-              session.tenantId,
-              opts.populateUser(ctx)
-            )
+              tenantId: session.tenantId,
+              email: session.email,
+              populateUser: opts.populateUser(ctx),
+            })
           } else {
-            user = await getUser(userId, session.tenantId)
+            user = await getUser({
+              userId,
+              tenantId: session.tenantId,
+              email: session.email,
+            })
           }
           // @ts-ignore
           user.csrfToken = session.csrfToken
@@ -148,7 +161,11 @@ export default function (
       }
       // this is an internal request, no user made it
       if (!authenticated && apiKey) {
-        const populateUser = opts.populateUser ? opts.populateUser(ctx) : null
+        const populateUser: (
+          userId: string,
+          tenantId: string,
+          email?: string
+        ) => Promise<User> = opts.populateUser ? opts.populateUser(ctx) : null
         const { valid, user: foundUser } = await checkApiKey(
           apiKey,
           populateUser
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index d5ed2ce906..214c9498d6 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -333,6 +333,7 @@ export default class TestConfiguration {
       sessionId: this.sessionIdForUser(_id),
       tenantId: this.getTenantId(),
       csrfToken: this.csrfToken,
+      email,
     })
     const resp = await db.put(user)
     await cache.user.invalidateUser(_id)
@@ -396,16 +397,17 @@ export default class TestConfiguration {
       }
       // make sure the user exists in the global DB
       if (roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) {
-        await this.globalUser({
+        const user = await this.globalUser({
           _id: userId,
           builder: { global: builder },
           roles: { [appId]: roleId || roles.BUILTIN_ROLE_IDS.BASIC },
         })
+        await sessions.createASession(userId, {
+          sessionId: this.sessionIdForUser(userId),
+          tenantId: this.getTenantId(),
+          email: user.email,
+        })
       }
-      await sessions.createASession(userId, {
-        sessionId: this.sessionIdForUser(userId),
-        tenantId: this.getTenantId(),
-      })
       // have to fake this
       const authObj = {
         userId,
diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts
index ba451a3325..c7e28d3bf4 100644
--- a/packages/server/src/threads/query.ts
+++ b/packages/server/src/threads/query.ts
@@ -247,7 +247,9 @@ class QueryRunner {
     if (!resp.err) {
       const globalUserId = getGlobalIDFromUserMetadataID(_id)
       await auth.updateUserOAuth(globalUserId, resp)
-      this.ctx.user = await cache.user.getUser(globalUserId)
+      this.ctx.user = await cache.user.getUser({
+        userId: globalUserId,
+      })
     } else {
       // In this event the user may have oAuth issues that
       // could require re-authenticating with their provider.
diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts
index bbb84c1882..c1fcf35634 100644
--- a/packages/server/src/utilities/global.ts
+++ b/packages/server/src/utilities/global.ts
@@ -77,7 +77,9 @@ export async function getCachedSelf(
 ): Promise<ContextUser> {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
-  const user = await cache.user.getUser(ctx.user?._id!)
+  const user = await cache.user.getUser({
+    userId: ctx.user?._id!,
+  })
   return processUser(user, { appId })
 }
 
diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
index 874113f6f1..b02ea2ff60 100644
--- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
+++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
@@ -35,7 +35,9 @@ export async function processInputBBReference(
       }
 
       try {
-        await cache.user.getUser(id)
+        await cache.user.getUser({
+          userId: id,
+        })
         return id
       } catch (e: any) {
         if (e.statusCode === 404) {
@@ -125,7 +127,9 @@ export async function processOutputBBReference(
     case BBReferenceFieldSubType.USER: {
       let user
       try {
-        user = await cache.user.getUser(value as string)
+        user = await cache.user.getUser({
+          userId: value as string,
+        })
       } catch (err: any) {
         if (err.statusCode !== 404) {
           throw err
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index c2a63fd674..f6cf44d6d6 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -110,7 +110,9 @@ async function processDefaultValues(table: Table, row: Row) {
 
   const identity = context.getIdentity()
   if (identity?._id && identity.type === IdentityType.USER) {
-    const user = await cache.user.getUser(identity._id)
+    const user = await cache.user.getUser({
+      userId: identity._id,
+    })
     delete user.password
 
     ctx["Current User"] = user
diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
index c5a7b7fc84..28c16dad71 100644
--- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
+++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
@@ -74,7 +74,9 @@ describe("bbReferenceProcessor", () => {
 
         expect(result).toEqual(userId)
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
 
       it("throws an error given an invalid id", async () => {
@@ -98,7 +100,9 @@ describe("bbReferenceProcessor", () => {
 
         expect(result).toEqual(userId)
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
 
       it("empty strings will return null", async () => {
@@ -243,7 +247,9 @@ describe("bbReferenceProcessor", () => {
           lastName: user.lastName,
         })
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
 
       it("returns undefined given an unexisting user", async () => {
@@ -255,7 +261,9 @@ describe("bbReferenceProcessor", () => {
 
         expect(result).toBeUndefined()
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
     })
   })
diff --git a/packages/types/src/sdk/auth.ts b/packages/types/src/sdk/auth.ts
index edcaf197e2..5bc75e8377 100644
--- a/packages/types/src/sdk/auth.ts
+++ b/packages/types/src/sdk/auth.ts
@@ -10,6 +10,7 @@ export interface AuthToken {
 export interface CreateSession {
   sessionId: string
   tenantId: string
+  email: string
   csrfToken?: string
   hosting?: Hosting
 }
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index 3f24de440a..078ac441e9 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -19,12 +19,17 @@ import { EmailTemplatePurpose } from "../../constants"
 export async function loginUser(user: User) {
   const sessionId = coreUtils.newid()
   const tenantId = tenancy.getTenantId()
-  await sessions.createASession(user._id!, { sessionId, tenantId })
+  await sessions.createASession(user._id!, {
+    sessionId,
+    tenantId,
+    email: user.email,
+  })
   return jwt.sign(
     {
       userId: user._id,
       sessionId,
       tenantId,
+      email: user.email,
     },
     coreEnv.JWT_SECRET!
   )
diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts
index c1c8aa8b19..c2cf005308 100644
--- a/packages/worker/src/tests/TestConfiguration.ts
+++ b/packages/worker/src/tests/TestConfiguration.ts
@@ -170,19 +170,26 @@ class TestConfiguration {
   async _createSession({
     userId,
     tenantId,
+    email,
   }: {
     userId: string
     tenantId: string
+    email: string
   }) {
     await sessions.createASession(userId!, {
       sessionId: "sessionid",
-      tenantId: tenantId,
+      tenantId,
       csrfToken: CSRF_TOKEN,
+      email,
     })
   }
 
   async createSession(user: User) {
-    return this._createSession({ userId: user._id!, tenantId: user.tenantId })
+    return this._createSession({
+      userId: user._id!,
+      tenantId: user.tenantId,
+      email: user.email,
+    })
   }
 
   cookieHeader(cookies: any) {

From 9ac2ddc14b8c6642d4c16538445aaf03c563b85d Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 12 Sep 2024 10:32:27 +0000
Subject: [PATCH 020/101] Bump version to 2.32.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 4a7cfc95cc..2569096f47 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.0",
+  "version": "2.32.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 021a706001a805f37ce02f1425a66695e691181d Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 12 Sep 2024 13:52:04 +0100
Subject: [PATCH 021/101] updating pro reference

---
 yarn.lock | 281 ++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 179 insertions(+), 102 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 146fa74d5b..3727e08d0a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2053,44 +2053,6 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.31.8":
-  version "0.0.0"
-  dependencies:
-    "@budibase/nano" "10.1.5"
-    "@budibase/pouchdb-replication-stream" "1.2.11"
-    "@budibase/shared-core" "0.0.0"
-    "@budibase/types" "0.0.0"
-    aws-cloudfront-sign "3.0.2"
-    aws-sdk "2.1030.0"
-    bcrypt "5.1.0"
-    bcryptjs "2.4.3"
-    bull "4.10.1"
-    correlation-id "4.0.0"
-    dd-trace "5.2.0"
-    dotenv "16.0.1"
-    ioredis "5.3.2"
-    joi "17.6.0"
-    jsonwebtoken "9.0.2"
-    knex "2.4.2"
-    koa-passport "^6.0.0"
-    koa-pino-logger "4.0.0"
-    lodash "4.17.21"
-    node-fetch "2.6.7"
-    passport-google-oauth "2.0.0"
-    passport-local "1.0.0"
-    passport-oauth2-refresh "^2.1.0"
-    pino "8.11.0"
-    pino-http "8.3.3"
-    posthog-node "4.0.1"
-    pouchdb "7.3.0"
-    pouchdb-find "7.2.2"
-    redlock "4.2.0"
-    rotating-file-stream "3.1.0"
-    sanitize-s3-objectkey "0.0.1"
-    semver "^7.5.4"
-    tar-fs "2.1.1"
-    uuid "^8.3.2"
-
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2133,45 +2095,6 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@npm:@budibase/pro@latest":
-  version "2.31.8"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.31.8.tgz#92b27f99f815f5d20bf58bfae916760b14a036da"
-  integrity sha512-nmNKVoMdUVqEIq6xqoBq0gVBCLkoPMszmn0Zu0SJ/Dc2SpsXhPz9S3n9xXfAA+FHUg9LgUAS+eKPCKPWZXtDHQ==
-  dependencies:
-    "@budibase/backend-core" "2.31.8"
-    "@budibase/shared-core" "2.31.8"
-    "@budibase/string-templates" "2.31.8"
-    "@budibase/types" "2.31.8"
-    "@koa/router" "8.0.8"
-    bull "4.10.1"
-    dd-trace "5.2.0"
-    joi "17.6.0"
-    jsonwebtoken "9.0.2"
-    lru-cache "^7.14.1"
-    memorystream "^0.3.1"
-    node-fetch "2.6.7"
-    scim-patch "^0.8.1"
-    scim2-parse-filter "^0.2.8"
-
-"@budibase/shared-core@2.31.8":
-  version "0.0.0"
-  dependencies:
-    "@budibase/types" "0.0.0"
-    cron-validate "1.4.5"
-
-"@budibase/string-templates@2.31.8":
-  version "0.0.0"
-  dependencies:
-    "@budibase/handlebars-helpers" "^0.13.2"
-    dayjs "^1.10.8"
-    handlebars "^4.7.8"
-    lodash.clonedeep "^4.5.0"
-
-"@budibase/types@2.31.8":
-  version "0.0.0"
-  dependencies:
-    scim-patch "^0.8.1"
-
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -6117,6 +6040,11 @@
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
   integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
 
+"@types/qs@^6.9.15":
+  version "6.9.15"
+  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.15.tgz#adde8a060ec9c305a82de1babc1056e73bd64dce"
+  integrity sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==
+
 "@types/range-parser@*":
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
@@ -7525,7 +7453,30 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
+axios@0.24.0:
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
+  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
+  dependencies:
+    follow-redirects "^1.14.4"
+
+axios@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
+  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
+axios@^0.21.1:
+  version "0.21.4"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
+  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
+  dependencies:
+    follow-redirects "^1.14.0"
+
+axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11635,6 +11586,11 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
+follow-redirects@^1.14.0, follow-redirects@^1.14.4:
+  version "1.15.9"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
+  integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==
+
 follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
@@ -12728,7 +12684,12 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1:
+  version "3.8.1"
+  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
+  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
+
+http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13739,11 +13700,6 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
-isobject@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
-  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
-
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -16303,7 +16259,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@1.10.1, msgpackr@^1.5.2:
+msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16497,13 +16453,25 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
+node-fetch@2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
+node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.6.9, node-fetch@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -17132,6 +17100,21 @@ openai@^4.52.1:
     node-fetch "^2.6.7"
     web-streams-polyfill "^3.2.1"
 
+openai@^4.59.0:
+  version "4.59.0"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08"
+  integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==
+  dependencies:
+    "@types/node" "^18.11.18"
+    "@types/node-fetch" "^2.6.4"
+    "@types/qs" "^6.9.15"
+    abort-controller "^3.0.0"
+    agentkeepalive "^4.2.1"
+    form-data-encoder "1.7.2"
+    formdata-node "^4.3.2"
+    node-fetch "^2.6.7"
+    qs "^6.10.3"
+
 openapi-response-validator@^9.2.0:
   version "9.3.1"
   resolved "https://registry.yarnpkg.com/openapi-response-validator/-/openapi-response-validator-9.3.1.tgz#54284d8be608ef53283cbe7448accce8106b1c56"
@@ -17654,7 +17637,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
+passport@^0.4.0:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
+  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
+  dependencies:
+    passport-strategy "1.x.x"
+    pause "0.0.1"
+
+passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -17798,11 +17789,21 @@ periscopic@^3.1.0:
     estree-walker "^3.0.0"
     is-reference "^3.0.0"
 
+pg-cloudflare@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
+  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
+
 pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
   integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
 
+pg-connection-string@^2.6.4:
+  version "2.6.4"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d"
+  integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==
+
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
@@ -17813,11 +17814,21 @@ pg-pool@^3.6.0:
   resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e"
   integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==
 
+pg-pool@^3.6.2:
+  version "3.6.2"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2"
+  integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==
+
 pg-protocol@*, pg-protocol@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
   integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
 
+pg-protocol@^1.6.1:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3"
+  integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==
+
 pg-types@^2.1.0, pg-types@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
@@ -17842,6 +17853,19 @@ pg@8.10.0:
     pg-types "^2.1.0"
     pgpass "1.x"
 
+pg@^8.12.0:
+  version "8.12.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79"
+  integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==
+  dependencies:
+    pg-connection-string "^2.6.4"
+    pg-pool "^3.6.2"
+    pg-protocol "^1.6.1"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+  optionalDependencies:
+    pg-cloudflare "^1.1.1"
+
 pgpass@1.x:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
@@ -18927,7 +18951,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.33:
+psl@^1.1.28, psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -19999,6 +20023,11 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
+sax@>=0.1.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
+  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
+
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -20071,13 +20100,33 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1:
+"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
+  version "5.7.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
+  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
+
+semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
+semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
+  version "6.3.1"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
+  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
+
+semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
+semver@~2.3.1:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
+  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
+
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -21640,7 +21689,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
+"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21650,6 +21699,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
+tough-cookie@~2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
+  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
+  dependencies:
+    psl "^1.1.28"
+    punycode "^2.1.1"
+
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -22178,14 +22235,6 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
-unset-value@2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
-  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
-  dependencies:
-    has-value "^2.0.2"
-    isobject "^4.0.0"
-
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -22965,10 +23014,33 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
-  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
+xml2js@0.1.x:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
+  dependencies:
+    sax ">=0.1.1"
+
+xml2js@0.4.19:
+  version "0.4.19"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
+  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~9.0.1"
+
+xml2js@0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
+  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~11.0.0"
+
+xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.4.23"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
+  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -22978,6 +23050,11 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
+xmlbuilder@~9.0.1:
+  version "9.0.7"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
+  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
+
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"

From 62ff1503645745fb519d5201c931cd78e2474f53 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 12 Sep 2024 13:53:51 +0100
Subject: [PATCH 022/101] update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ec1d2bda75..922431260e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c
+Subproject commit 922431260e90d558a1ca55398475412e75088057

From 7414cc0c09ccffad4dae5af02bb8b48f68c806b3 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 12 Sep 2024 13:03:52 +0000
Subject: [PATCH 023/101] Bump version to 2.32.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 2569096f47..30a02755c5 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.1",
+  "version": "2.32.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From d8ff33441e9298712b4061dce355af78feef8352 Mon Sep 17 00:00:00 2001
From: Pascal Reichmuth <pascal.reichmuth@gmail.com>
Date: Thu, 12 Sep 2024 15:51:00 +0200
Subject: [PATCH 024/101] Update openai.ts

---
 packages/server/src/automations/steps/openai.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 1c148b2e73..e4c77c3b85 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -15,6 +15,8 @@ enum Model {
   GPT_35_TURBO = "gpt-3.5-turbo",
   // will only work with api keys that have access to the GPT4 API
   GPT_4 = "gpt-4",
+  GPT_4O = "gpt-4o",
+  GPT_4O_MINI = "gpt-4o-mini"
 }
 
 export const definition: AutomationStepDefinition = {

From 0798f97066f943ca571295a036551e56210f558c Mon Sep 17 00:00:00 2001
From: Pascal Reichmuth <pascal.reichmuth@gmail.com>
Date: Thu, 12 Sep 2024 15:53:35 +0200
Subject: [PATCH 025/101] Update StepInputsOutputs.ts

---
 .../types/src/documents/app/automation/StepInputsOutputs.ts     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/types/src/documents/app/automation/StepInputsOutputs.ts b/packages/types/src/documents/app/automation/StepInputsOutputs.ts
index 983a67daf5..fab672e04b 100644
--- a/packages/types/src/documents/app/automation/StepInputsOutputs.ts
+++ b/packages/types/src/documents/app/automation/StepInputsOutputs.ts
@@ -140,6 +140,8 @@ enum Model {
   GPT_35_TURBO = "gpt-3.5-turbo",
   // will only work with api keys that have access to the GPT4 API
   GPT_4 = "gpt-4",
+  GPT_4O = "gpt-4o",
+  GPT_4O_MINI = "gpt-4o-mini"
 }
 
 export type OpenAIStepOutputs = Omit<BaseAutomationOutputs, "response"> & {

From a831a4bf077b9bc95f824058abebbf8e3d9fdaa9 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Thu, 12 Sep 2024 14:58:53 +0100
Subject: [PATCH 026/101] tests to cover automation naming

---
 .../tests/scenarios/looping.spec.ts           |  24 ++
 .../tests/scenarios/scenarios.spec.ts         | 309 ++++++++++--------
 .../tests/utilities/AutomationTestBuilder.ts  |  54 +--
 .../documents/app/automation/automation.ts    |   1 +
 4 files changed, 233 insertions(+), 155 deletions(-)

diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts
index 9bc382a187..f229610542 100644
--- a/packages/server/src/automations/tests/scenarios/looping.spec.ts
+++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts
@@ -242,4 +242,28 @@ describe("Loop automations", () => {
     expect(results.steps[1].outputs.message).toContain("- 3")
     expect(results.steps[3].outputs.message).toContain("- 3")
   })
+
+  it("should use automation names to loop with", async () => {
+    const builder = createAutomationBuilder({
+      name: "Test Trigger with Loop and Create Row",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .loop(
+        {
+          option: LoopStepType.ARRAY,
+          binding: [1, 2, 3],
+        },
+        "FirstLoopStep"
+      )
+      .serverLog({ text: "Message {{loop.currentItem}}" }, "FirstLoopLog")
+      .serverLog(
+        { text: "{{steps.FirstLoopLog.iterations}}" },
+        "FirstLoopIterationLog"
+      )
+      .run()
+
+    expect(results.steps[1].outputs.message).toContain("- 3")
+  })
 })
diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts
index 40d6094525..398ef20100 100644
--- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts
+++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts
@@ -49,151 +49,188 @@ describe("Automation Scenarios", () => {
         },
       })
     })
-  })
 
-  it("should trigger an automation which querys the database", async () => {
-    const table = await config.createTable()
-    const row = {
-      name: "Test Row",
-      description: "original description",
-      tableId: table._id,
-    }
-    await config.createRow(row)
-    await config.createRow(row)
-    const builder = createAutomationBuilder({
-      name: "Test Row Save and Create",
+    it("should trigger an automation which querys the database", async () => {
+      const table = await config.createTable()
+      const row = {
+        name: "Test Row",
+        description: "original description",
+        tableId: table._id,
+      }
+      await config.createRow(row)
+      await config.createRow(row)
+      const builder = createAutomationBuilder({
+        name: "Test Row Save and Create",
+      })
+
+      const results = await builder
+        .appAction({ fields: {} })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(1)
+      expect(results.steps[0].outputs.rows).toHaveLength(2)
     })
 
-    const results = await builder
-      .appAction({ fields: {} })
-      .queryRows({
-        tableId: table._id!,
+    it("should trigger an automation which querys the database then deletes a row", async () => {
+      const table = await config.createTable()
+      const row = {
+        name: "DFN",
+        description: "original description",
+        tableId: table._id,
+      }
+      await config.createRow(row)
+      await config.createRow(row)
+      const builder = createAutomationBuilder({
+        name: "Test Row Save and Create",
       })
-      .run()
 
-    expect(results.steps).toHaveLength(1)
-    expect(results.steps[0].outputs.rows).toHaveLength(2)
-  })
+      const results = await builder
+        .appAction({ fields: {} })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .deleteRow({
+          tableId: table._id!,
+          id: "{{ steps.1.rows.0._id }}",
+        })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .run()
 
-  it("should trigger an automation which querys the database then deletes a row", async () => {
-    const table = await config.createTable()
-    const row = {
-      name: "DFN",
-      description: "original description",
-      tableId: table._id,
-    }
-    await config.createRow(row)
-    await config.createRow(row)
-    const builder = createAutomationBuilder({
-      name: "Test Row Save and Create",
+      expect(results.steps).toHaveLength(3)
+      expect(results.steps[1].outputs.success).toBeTruthy()
+      expect(results.steps[2].outputs.rows).toHaveLength(1)
     })
 
-    const results = await builder
-      .appAction({ fields: {} })
-      .queryRows({
-        tableId: table._id!,
-      })
-      .deleteRow({
-        tableId: table._id!,
-        id: "{{ steps.1.rows.0._id }}",
-      })
-      .queryRows({
-        tableId: table._id!,
-      })
-      .run()
-
-    expect(results.steps).toHaveLength(3)
-    expect(results.steps[1].outputs.success).toBeTruthy()
-    expect(results.steps[2].outputs.rows).toHaveLength(1)
-  })
-
-  it("should query an external database for some data then insert than into an internal table", async () => {
-    const { datasource, client } = await setup.setupTestDatasource(
-      config,
-      DatabaseName.MYSQL
-    )
-
-    const newTable = await config.createTable({
-      name: "table",
-      type: "table",
-      schema: {
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-          constraints: {
-            presence: true,
-          },
-        },
-        age: {
-          name: "age",
-          type: FieldType.NUMBER,
-          constraints: {
-            presence: true,
-          },
-        },
-      },
-    })
-
-    const tableName = await setup.createTestTable(client, {
-      name: { type: "string" },
-      age: { type: "number" },
-    })
-
-    const rows = [
-      { name: "Joe", age: 20 },
-      { name: "Bob", age: 25 },
-      { name: "Paul", age: 30 },
-    ]
-
-    await setup.insertTestData(client, tableName, rows)
-
-    const query = await setup.saveTestQuery(
-      config,
-      client,
-      tableName,
-      datasource
-    )
-
-    const builder = createAutomationBuilder({
-      name: "Test external query and save",
-    })
-
-    const results = await builder
-      .appAction({
-        fields: {},
-      })
-      .executeQuery({
-        query: {
-          queryId: query._id!,
-        },
-      })
-      .loop({
-        option: LoopStepType.ARRAY,
-        binding: "{{ steps.1.response }}",
-      })
-      .createRow({
-        row: {
-          name: "{{ loop.currentItem.name }}",
-          age: "{{ loop.currentItem.age }}",
-          tableId: newTable._id!,
-        },
-      })
-      .queryRows({
-        tableId: newTable._id!,
-      })
-      .run()
-
-    expect(results.steps).toHaveLength(3)
-
-    expect(results.steps[1].outputs.iterations).toBe(3)
-    expect(results.steps[1].outputs.items).toHaveLength(3)
-
-    expect(results.steps[2].outputs.rows).toHaveLength(3)
-
-    rows.forEach(expectedRow => {
-      expect(results.steps[2].outputs.rows).toEqual(
-        expect.arrayContaining([expect.objectContaining(expectedRow)])
+    it("should query an external database for some data then insert than into an internal table", async () => {
+      const { datasource, client } = await setup.setupTestDatasource(
+        config,
+        DatabaseName.MYSQL
       )
+
+      const newTable = await config.createTable({
+        name: "table",
+        type: "table",
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+            constraints: {
+              presence: true,
+            },
+          },
+          age: {
+            name: "age",
+            type: FieldType.NUMBER,
+            constraints: {
+              presence: true,
+            },
+          },
+        },
+      })
+
+      const tableName = await setup.createTestTable(client, {
+        name: { type: "string" },
+        age: { type: "number" },
+      })
+
+      const rows = [
+        { name: "Joe", age: 20 },
+        { name: "Bob", age: 25 },
+        { name: "Paul", age: 30 },
+      ]
+
+      await setup.insertTestData(client, tableName, rows)
+
+      const query = await setup.saveTestQuery(
+        config,
+        client,
+        tableName,
+        datasource
+      )
+
+      const builder = createAutomationBuilder({
+        name: "Test external query and save",
+      })
+
+      const results = await builder
+        .appAction({
+          fields: {},
+        })
+        .executeQuery({
+          query: {
+            queryId: query._id!,
+          },
+        })
+        .loop({
+          option: LoopStepType.ARRAY,
+          binding: "{{ steps.1.response }}",
+        })
+        .createRow({
+          row: {
+            name: "{{ loop.currentItem.name }}",
+            age: "{{ loop.currentItem.age }}",
+            tableId: newTable._id!,
+          },
+        })
+        .queryRows({
+          tableId: newTable._id!,
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(3)
+
+      expect(results.steps[1].outputs.iterations).toBe(3)
+      expect(results.steps[1].outputs.items).toHaveLength(3)
+
+      expect(results.steps[2].outputs.rows).toHaveLength(3)
+
+      rows.forEach(expectedRow => {
+        expect(results.steps[2].outputs.rows).toEqual(
+          expect.arrayContaining([expect.objectContaining(expectedRow)])
+        )
+      })
+    })
+  })
+
+  describe("Name Based Automations", () => {
+    it("should fetch and delete a rpw using automation naming", async () => {
+      const table = await config.createTable()
+      const row = {
+        name: "DFN",
+        description: "original description",
+        tableId: table._id,
+      }
+      await config.createRow(row)
+      await config.createRow(row)
+      const builder = createAutomationBuilder({
+        name: "Test Query and Delete Row",
+      })
+
+      const results = await builder
+        .appAction({ fields: {} })
+        .queryRows(
+          {
+            tableId: table._id!,
+          },
+          "InitialQueryStep"
+        )
+        .deleteRow({
+          tableId: table._id!,
+          id: "{{ steps.InitialQueryStep.rows.0._id }}",
+        })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(3)
+      expect(results.steps[1].outputs.success).toBeTruthy()
+      expect(results.steps[2].outputs.rows).toHaveLength(1)
     })
   })
 })
diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
index f477efabe4..34c25d7004 100644
--- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
+++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
@@ -57,21 +57,27 @@ type BranchConfig = {
 
 class BaseStepBuilder {
   protected steps: AutomationStep[] = []
+  protected stepNames: { [key: string]: string } = {}
 
   protected step<TStep extends AutomationActionStepId>(
     stepId: TStep,
     stepSchema: Omit<AutomationStep, "id" | "stepId" | "inputs">,
-    inputs: AutomationStepInputs<TStep>
+    inputs: AutomationStepInputs<TStep>,
+    stepName?: string
   ): this {
+    const id = uuidv4()
     this.steps.push({
       ...stepSchema,
       inputs: inputs as any,
-      id: uuidv4(),
+      id,
       stepId,
+      name: stepName || stepSchema.name,
     })
+    if (stepName) {
+      this.stepNames[id] = stepName
+    }
     return this
   }
-
   protected addBranchStep(branchConfig: BranchConfig): void {
     const branchStepInputs: BranchStepInputs = {
       branches: [] as Branch[],
@@ -99,66 +105,74 @@ class BaseStepBuilder {
   }
 
   // STEPS
-  createRow(inputs: CreateRowStepInputs): this {
+  createRow(inputs: CreateRowStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.CREATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.CREATE_ROW,
-      inputs
+      inputs,
+      stepName
     )
   }
 
-  updateRow(inputs: UpdateRowStepInputs): this {
+  updateRow(inputs: UpdateRowStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.UPDATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW,
-      inputs
+      inputs,
+      stepName
     )
   }
 
-  deleteRow(inputs: DeleteRowStepInputs): this {
+  deleteRow(inputs: DeleteRowStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.DELETE_ROW,
       BUILTIN_ACTION_DEFINITIONS.DELETE_ROW,
-      inputs
+      inputs,
+      stepName
     )
   }
 
-  sendSmtpEmail(inputs: SmtpEmailStepInputs): this {
+  sendSmtpEmail(inputs: SmtpEmailStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.SEND_EMAIL_SMTP,
       BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP,
-      inputs
+      inputs,
+      stepName
     )
   }
 
-  executeQuery(inputs: ExecuteQueryStepInputs): this {
+  executeQuery(inputs: ExecuteQueryStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.EXECUTE_QUERY,
       BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY,
-      inputs
+      inputs,
+      stepName
     )
   }
 
-  queryRows(inputs: QueryRowsStepInputs): this {
+  queryRows(inputs: QueryRowsStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.QUERY_ROWS,
       BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS,
-      inputs
+      inputs,
+      stepName
     )
   }
-  loop(inputs: LoopStepInputs): this {
+  loop(inputs: LoopStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.LOOP,
       BUILTIN_ACTION_DEFINITIONS.LOOP,
-      inputs
+      inputs,
+      stepName
     )
   }
 
-  serverLog(input: ServerLogStepInputs): this {
+  serverLog(input: ServerLogStepInputs, stepName?: string): this {
     return this.step(
       AutomationActionStepId.SERVER_LOG,
       BUILTIN_ACTION_DEFINITIONS.SERVER_LOG,
-      input
+      input,
+      stepName
     )
   }
 }
@@ -186,6 +200,7 @@ class AutomationBuilder extends BaseStepBuilder {
       definition: {
         steps: [],
         trigger: {} as AutomationTrigger,
+        stepNames: {},
       },
       type: "automation",
       appId: options.appId ?? setup.getConfig().getAppId(),
@@ -268,6 +283,7 @@ class AutomationBuilder extends BaseStepBuilder {
 
   build(): Automation {
     this.automationConfig.definition.steps = this.steps
+    this.automationConfig.definition.stepNames = this.stepNames
     return this.automationConfig
   }
 
diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts
index 72f8a1aa7c..78d22c787f 100644
--- a/packages/types/src/documents/app/automation/automation.ts
+++ b/packages/types/src/documents/app/automation/automation.ts
@@ -124,6 +124,7 @@ export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
     trigger: AutomationTrigger
+    stepNames?: Record<string, string>
   }
   screenId?: string
   uiTree?: any

From e4664d23d7a48ad2d1bbfa259cb374343b6e8df7 Mon Sep 17 00:00:00 2001
From: Pascal Reichmuth <pascal.reichmuth@gmail.com>
Date: Thu, 12 Sep 2024 16:22:49 +0200
Subject: [PATCH 027/101] Update openai.ts

---
 packages/server/src/automations/steps/openai.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index e4c77c3b85..d02ba56b70 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -16,7 +16,7 @@ enum Model {
   // will only work with api keys that have access to the GPT4 API
   GPT_4 = "gpt-4",
   GPT_4O = "gpt-4o",
-  GPT_4O_MINI = "gpt-4o-mini"
+  GPT_4O_MINI = "gpt-4o-mini",
 }
 
 export const definition: AutomationStepDefinition = {

From 2700dfc63db08a68b90e34c5f82cf1f641f5d32c Mon Sep 17 00:00:00 2001
From: Pascal Reichmuth <pascal.reichmuth@gmail.com>
Date: Thu, 12 Sep 2024 16:23:05 +0200
Subject: [PATCH 028/101] Update StepInputsOutputs.ts

---
 .../types/src/documents/app/automation/StepInputsOutputs.ts     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/documents/app/automation/StepInputsOutputs.ts b/packages/types/src/documents/app/automation/StepInputsOutputs.ts
index fab672e04b..9ba7b6cc1d 100644
--- a/packages/types/src/documents/app/automation/StepInputsOutputs.ts
+++ b/packages/types/src/documents/app/automation/StepInputsOutputs.ts
@@ -141,7 +141,7 @@ enum Model {
   // will only work with api keys that have access to the GPT4 API
   GPT_4 = "gpt-4",
   GPT_4O = "gpt-4o",
-  GPT_4O_MINI = "gpt-4o-mini"
+  GPT_4O_MINI = "gpt-4o-mini",
 }
 
 export type OpenAIStepOutputs = Omit<BaseAutomationOutputs, "response"> & {

From 4952747ae955b333a5c89b5c6ab413a44134627d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 12 Sep 2024 15:40:37 +0100
Subject: [PATCH 029/101] Taking working from new-rbac-ui branch and separating
 it into its own PR, so that other work can be based on this from master.

---
 packages/backend-core/src/security/roles.ts   | 51 ++++++++++-------
 .../server/src/api/controllers/permission.ts  | 15 -----
 packages/server/src/api/controllers/role.ts   | 11 +++-
 .../src/api/routes/tests/permissions.spec.ts  | 51 -----------------
 .../server/src/sdk/app/permissions/index.ts   | 57 ++-----------------
 .../app/permissions/tests/permissions.spec.ts | 53 -----------------
 packages/shared-core/src/constants/colors.ts  |  8 +++
 packages/shared-core/src/constants/index.ts   |  1 +
 packages/types/src/api/web/app/permission.ts  |  1 -
 packages/types/src/api/web/role.ts            |  3 +-
 packages/types/src/documents/app/role.ts      |  7 +++
 11 files changed, 61 insertions(+), 197 deletions(-)
 delete mode 100644 packages/server/src/sdk/app/permissions/tests/permissions.spec.ts
 create mode 100644 packages/shared-core/src/constants/colors.ts

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 50850219a3..a7210ec2b8 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -7,8 +7,9 @@ import {
   doWithDB,
 } from "../db"
 import { getAppDB } from "../context"
-import { Screen, Role as RoleDoc } from "@budibase/types"
+import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types"
 import cloneDeep from "lodash/fp/cloneDeep"
+import { RoleColor } from "@budibase/shared-core"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",
@@ -45,10 +46,12 @@ export class Role implements RoleDoc {
   inherits?: string
   version?: string
   permissions: Record<string, PermissionLevel[]> = {}
+  uiMetadata?: RoleUIMetadata
 
-  constructor(id: string, name: string, permissionId: string) {
+  constructor(id: string, permissionId: string, uiMetadata?: RoleUIMetadata) {
     this._id = id
-    this.name = name
+    this.name = uiMetadata?.displayName || id
+    this.uiMetadata = uiMetadata
     this.permissionId = permissionId
     // version for managing the ID - removing the role_ when responding
     this.version = RoleIDVersion.NAME
@@ -61,23 +64,31 @@ export class Role implements RoleDoc {
 }
 
 const BUILTIN_ROLES = {
-  ADMIN: new Role(
-    BUILTIN_IDS.ADMIN,
-    "Admin",
-    BuiltinPermissionID.ADMIN
-  ).addInheritance(BUILTIN_IDS.POWER),
-  POWER: new Role(
-    BUILTIN_IDS.POWER,
-    "Power",
-    BuiltinPermissionID.POWER
-  ).addInheritance(BUILTIN_IDS.BASIC),
-  BASIC: new Role(
-    BUILTIN_IDS.BASIC,
-    "Basic",
-    BuiltinPermissionID.WRITE
-  ).addInheritance(BUILTIN_IDS.PUBLIC),
-  PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public", BuiltinPermissionID.PUBLIC),
-  BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder", BuiltinPermissionID.ADMIN),
+  ADMIN: new Role(BUILTIN_IDS.ADMIN, BuiltinPermissionID.ADMIN, {
+    displayName: "App admin",
+    description: "Can do everything",
+    color: RoleColor.ADMIN,
+  }).addInheritance(BUILTIN_IDS.POWER),
+  POWER: new Role(BUILTIN_IDS.POWER, BuiltinPermissionID.POWER, {
+    displayName: "App power user",
+    description: "An app user with more access",
+    color: RoleColor.POWER,
+  }).addInheritance(BUILTIN_IDS.BASIC),
+  BASIC: new Role(BUILTIN_IDS.BASIC, BuiltinPermissionID.WRITE, {
+    displayName: "App user",
+    description: "Any logged in user",
+    color: RoleColor.BASIC,
+  }).addInheritance(BUILTIN_IDS.PUBLIC),
+  PUBLIC: new Role(BUILTIN_IDS.PUBLIC, BuiltinPermissionID.PUBLIC, {
+    displayName: "Public user",
+    description: "Accessible to anyone",
+    color: RoleColor.PUBLIC,
+  }),
+  BUILDER: new Role(BUILTIN_IDS.BUILDER, BuiltinPermissionID.ADMIN, {
+    displayName: "Builder user",
+    description: "Users that can edit this app",
+    color: RoleColor.BUILDER,
+  }),
 }
 
 export function getBuiltinRoles(): { [key: string]: RoleDoc } {
diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index 9444dfa251..0629ebc967 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -45,18 +45,6 @@ async function updatePermissionOnRole(
   }: { roleId: string; resourceId: string; level: PermissionLevel },
   updateType: PermissionUpdateType
 ) {
-  const allowedAction = await sdk.permissions.resourceActionAllowed({
-    resourceId,
-    level,
-  })
-
-  if (!allowedAction.allowed) {
-    throw new HTTPError(
-      `You are not allowed to '${allowedAction.level}' the resource type '${allowedAction.resourceType}'`,
-      403
-    )
-  }
-
   const db = context.getAppDB()
   const remove = updateType === PermissionUpdateType.REMOVE
   const isABuiltin = roles.isBuiltin(roleId)
@@ -184,9 +172,6 @@ export async function getResourcePerms(
       },
       {} as Record<string, ResourcePermissionInfo>
     ),
-    requiresPlanToModify: (
-      await sdk.permissions.allowsExplicitPermissions(resourceId)
-    ).minPlan,
   }
 }
 
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index 605b462842..ee1c223952 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -19,7 +19,7 @@ import {
   UserMetadata,
   DocumentType,
 } from "@budibase/types"
-import { sdk as sharedSdk } from "@budibase/shared-core"
+import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
 const UpdateRolesOptions = {
@@ -62,7 +62,8 @@ export async function find(ctx: UserCtx<void, FindRoleResponse>) {
 
 export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   const db = context.getAppDB()
-  let { _id, name, inherits, permissionId, version } = ctx.request.body
+  let { _id, name, inherits, permissionId, version, uiMetadata } =
+    ctx.request.body
   let isCreate = false
   const isNewVersion = version === roles.RoleIDVersion.NAME
 
@@ -88,7 +89,11 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     ctx.throw(400, "Cannot change custom role name")
   }
 
-  const role = new roles.Role(_id, name, permissionId).addInheritance(inherits)
+  const role = new roles.Role(_id, permissionId, {
+    displayName: uiMetadata?.displayName || name,
+    description: uiMetadata?.description || "Custom role",
+    color: uiMetadata?.color || RoleColor.DEFAULT_CUSTOM,
+  }).addInheritance(inherits)
   if (dbRole?.permissions && !role.permissions) {
     role.permissions = dbRole.permissions
   }
diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 838e1aca0b..43df63cd33 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -1,8 +1,4 @@
 const mockedSdk = sdk.permissions as jest.Mocked<typeof sdk.permissions>
-jest.mock("../../../sdk/app/permissions", () => ({
-  ...jest.requireActual("../../../sdk/app/permissions"),
-  resourceActionAllowed: jest.fn(),
-}))
 
 import sdk from "../../../sdk"
 
@@ -40,7 +36,6 @@ describe("/permission", () => {
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-    mockedSdk.resourceActionAllowed.mockResolvedValue({ allowed: true })
 
     table = (await config.createTable()) as typeof table
     row = await config.createRow()
@@ -112,29 +107,6 @@ describe("/permission", () => {
       expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID)
       expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID)
     })
-
-    it("throw forbidden if the action is not allowed for the resource", async () => {
-      mockedSdk.resourceActionAllowed.mockResolvedValue({
-        allowed: false,
-        resourceType: DocumentType.DATASOURCE,
-        level: PermissionLevel.READ,
-      })
-
-      await config.api.permission.add(
-        {
-          roleId: STD_ROLE_ID,
-          resourceId: table._id,
-          level: PermissionLevel.EXECUTE,
-        },
-        {
-          status: 403,
-          body: {
-            message:
-              "You are not allowed to 'read' the resource type 'datasource'",
-          },
-        }
-      )
-    })
   })
 
   describe("remove", () => {
@@ -148,29 +120,6 @@ describe("/permission", () => {
       const permsRes = await config.api.permission.get(table._id)
       expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
     })
-
-    it("throw forbidden if the action is not allowed for the resource", async () => {
-      mockedSdk.resourceActionAllowed.mockResolvedValue({
-        allowed: false,
-        resourceType: DocumentType.DATASOURCE,
-        level: PermissionLevel.READ,
-      })
-
-      await config.api.permission.revoke(
-        {
-          roleId: STD_ROLE_ID,
-          resourceId: table._id,
-          level: PermissionLevel.EXECUTE,
-        },
-        {
-          status: 403,
-          body: {
-            message:
-              "You are not allowed to 'read' the resource type 'datasource'",
-          },
-        }
-      )
-    })
   })
 
   describe("check public user allowed", () => {
diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts
index dd4085d69e..a6e81652ee 100644
--- a/packages/server/src/sdk/app/permissions/index.ts
+++ b/packages/server/src/sdk/app/permissions/index.ts
@@ -1,10 +1,7 @@
 import { db, roles } from "@budibase/backend-core"
-import { features } from "@budibase/pro"
 import {
-  DocumentType,
   PermissionLevel,
   PermissionSource,
-  PlanType,
   VirtualDocumentType,
 } from "@budibase/types"
 import { extractViewInfoFromID, isViewID } from "../../../db/utils"
@@ -15,36 +12,6 @@ import {
 import sdk from "../../../sdk"
 import { isV2 } from "../views"
 
-type ResourceActionAllowedResult =
-  | { allowed: true }
-  | {
-      allowed: false
-      level: PermissionLevel
-      resourceType: DocumentType | VirtualDocumentType
-    }
-
-export async function resourceActionAllowed({
-  resourceId,
-  level,
-}: {
-  resourceId: string
-  level: PermissionLevel
-}): Promise<ResourceActionAllowedResult> {
-  if (!isViewID(resourceId)) {
-    return { allowed: true }
-  }
-
-  if (await features.isViewPermissionEnabled()) {
-    return { allowed: true }
-  }
-
-  return {
-    allowed: false,
-    level,
-    resourceType: VirtualDocumentType.VIEW,
-  }
-}
-
 type ResourcePermissions = Record<
   string,
   { role: string; type: PermissionSource }
@@ -58,20 +25,6 @@ export async function getInheritablePermissions(
   }
 }
 
-export async function allowsExplicitPermissions(resourceId: string) {
-  if (isViewID(resourceId)) {
-    const allowed = await features.isViewPermissionEnabled()
-    const minPlan = !allowed ? PlanType.PREMIUM_PLUS : undefined
-
-    return {
-      allowed,
-      minPlan,
-    }
-  }
-
-  return { allowed: true }
-}
-
 export async function getResourcePerms(
   resourceId: string
 ): Promise<ResourcePermissions> {
@@ -81,15 +34,13 @@ export async function getResourcePerms(
 
   const permsToInherit = await getInheritablePermissions(resourceId)
 
-  const allowsExplicitPerm = (await allowsExplicitPermissions(resourceId))
-    .allowed
-
   for (let level of CURRENTLY_SUPPORTED_LEVELS) {
     // update the various roleIds in the resource permissions
     for (let role of rolesList) {
-      const rolePerms = allowsExplicitPerm
-        ? roles.checkForRoleResourceArray(role.permissions || {}, resourceId)
-        : {}
+      const rolePerms = roles.checkForRoleResourceArray(
+        role.permissions || {},
+        resourceId
+      )
       if (rolePerms[resourceId]?.indexOf(level as PermissionLevel) > -1) {
         permissions[level] = {
           role: roles.getExternalRoleID(role._id!, role.version),
diff --git a/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts b/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts
deleted file mode 100644
index 4c233e68fa..0000000000
--- a/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { PermissionLevel } from "@budibase/types"
-import { mocks, structures } from "@budibase/backend-core/tests"
-import { resourceActionAllowed } from ".."
-import { generateViewID } from "../../../../db/utils"
-import { initProMocks } from "../../../../tests/utilities/mocks/pro"
-
-initProMocks()
-
-describe("permissions sdk", () => {
-  beforeEach(() => {
-    mocks.licenses.useCloudFree()
-  })
-
-  describe("resourceActionAllowed", () => {
-    it("non view resources actions are always allowed", async () => {
-      const resourceId = structures.users.user()._id!
-
-      const result = await resourceActionAllowed({
-        resourceId,
-        level: PermissionLevel.READ,
-      })
-
-      expect(result).toEqual({ allowed: true })
-    })
-
-    it("view resources actions allowed if the feature flag is enabled", async () => {
-      mocks.licenses.useViewPermissions()
-      const resourceId = generateViewID(structures.generator.guid())
-
-      const result = await resourceActionAllowed({
-        resourceId,
-        level: PermissionLevel.READ,
-      })
-
-      expect(result).toEqual({ allowed: true })
-    })
-
-    it("view resources actions allowed if the feature flag is disabled", async () => {
-      const resourceId = generateViewID(structures.generator.guid())
-
-      const result = await resourceActionAllowed({
-        resourceId,
-        level: PermissionLevel.READ,
-      })
-
-      expect(result).toEqual({
-        allowed: false,
-        level: "read",
-        resourceType: "view",
-      })
-    })
-  })
-})
diff --git a/packages/shared-core/src/constants/colors.ts b/packages/shared-core/src/constants/colors.ts
new file mode 100644
index 0000000000..0abff46c3a
--- /dev/null
+++ b/packages/shared-core/src/constants/colors.ts
@@ -0,0 +1,8 @@
+export enum RoleColor {
+  ADMIN = "var(--spectrum-global-color-static-red-400)",
+  POWER = "var(--spectrum-global-color-static-orange-400)",
+  BASIC = "var(--spectrum-global-color-static-green-400)",
+  PUBLIC = "var(--spectrum-global-color-static-blue-400)",
+  BUILDER = "var(--spectrum-global-color-static-magenta-600)",
+  DEFAULT_CUSTOM = "var(--spectrum-global-color-static-magenta-400)",
+}
diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts
index 78984aafa4..5a42fc5677 100644
--- a/packages/shared-core/src/constants/index.ts
+++ b/packages/shared-core/src/constants/index.ts
@@ -1,6 +1,7 @@
 export * from "./api"
 export * from "./fields"
 export * from "./rows"
+export * from "./colors"
 
 export const OperatorOptions = {
   Equals: {
diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts
index 88ff4e9d2f..719be4f78e 100644
--- a/packages/types/src/api/web/app/permission.ts
+++ b/packages/types/src/api/web/app/permission.ts
@@ -8,7 +8,6 @@ export interface ResourcePermissionInfo {
 
 export interface GetResourcePermsResponse {
   permissions: Record<string, ResourcePermissionInfo>
-  requiresPlanToModify?: PlanType
 }
 
 export interface GetDependantResourcesResponse {
diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts
index c37dee60e0..644222b8f9 100644
--- a/packages/types/src/api/web/role.ts
+++ b/packages/types/src/api/web/role.ts
@@ -1,4 +1,4 @@
-import { Role } from "../../documents"
+import { Role, RoleUIMetadata } from "../../documents"
 
 export interface SaveRoleRequest {
   _id?: string
@@ -7,6 +7,7 @@ export interface SaveRoleRequest {
   inherits: string
   permissionId: string
   version: string
+  uiMetadata?: RoleUIMetadata
 }
 
 export interface SaveRoleResponse extends Role {}
diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts
index 7ccfb2e7e9..6557b7e19d 100644
--- a/packages/types/src/documents/app/role.ts
+++ b/packages/types/src/documents/app/role.ts
@@ -1,10 +1,17 @@
 import { Document } from "../document"
 import { PermissionLevel } from "../../sdk"
 
+export interface RoleUIMetadata {
+  displayName?: string
+  color?: string
+  description?: string
+}
+
 export interface Role extends Document {
   permissionId: string
   inherits?: string
   permissions: Record<string, PermissionLevel[]>
   version?: string
   name: string
+  uiMetadata?: RoleUIMetadata
 }

From 4d428f33f647504d3d16d6e3b6285eaf4993a1de Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 12 Sep 2024 15:45:41 +0100
Subject: [PATCH 030/101] Adding validator for uiMetadata.

---
 packages/server/src/api/routes/utils/validators.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts
index f0192b380b..b589d44b31 100644
--- a/packages/server/src/api/routes/utils/validators.ts
+++ b/packages/server/src/api/routes/utils/validators.ts
@@ -208,6 +208,11 @@ export function roleValidator() {
       name: Joi.string()
         .regex(/^[a-zA-Z0-9_]*$/)
         .required(),
+      uiMetadata: Joi.object({
+        displayName: OPTIONAL_STRING,
+        color: OPTIONAL_STRING,
+        description: OPTIONAL_STRING,
+      }).optional(),
       // this is the base permission ID (for now a built in)
       permissionId: Joi.string()
         .valid(...Object.values(permissions.BuiltinPermissionID))

From d47424fb93f7cf281698c9f460eec3656e361d06 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 12 Sep 2024 15:48:52 +0100
Subject: [PATCH 031/101] Re-add account-portal tests into Budibase CI (#14569)

---
 .github/workflows/budibase_ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 044e29b445..4c5cc94d2b 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -117,9 +117,9 @@ jobs:
       - name: Test
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore @budibase/account-portal-server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore @budibase/account-portal-server
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server
           fi
 
   test-worker:

From 8144aed935c6077aac1fa140afdfa6a7f9bf9fab Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 12 Sep 2024 15:53:44 +0100
Subject: [PATCH 032/101] Linting.

---
 packages/server/src/api/controllers/permission.ts   |  2 +-
 .../server/src/api/routes/tests/permissions.spec.ts | 13 +------------
 packages/types/src/api/web/app/permission.ts        |  2 +-
 3 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts
index 0629ebc967..66a3254348 100644
--- a/packages/server/src/api/controllers/permission.ts
+++ b/packages/server/src/api/controllers/permission.ts
@@ -1,4 +1,4 @@
-import { permissions, roles, context, HTTPError } from "@budibase/backend-core"
+import { permissions, roles, context } from "@budibase/backend-core"
 import {
   UserCtx,
   Database,
diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 43df63cd33..73c0952bf1 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -1,16 +1,5 @@
-const mockedSdk = sdk.permissions as jest.Mocked<typeof sdk.permissions>
-
-import sdk from "../../../sdk"
-
 import { roles } from "@budibase/backend-core"
-import {
-  Document,
-  DocumentType,
-  PermissionLevel,
-  Row,
-  Table,
-  ViewV2,
-} from "@budibase/types"
+import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts
index 719be4f78e..bead2a4279 100644
--- a/packages/types/src/api/web/app/permission.ts
+++ b/packages/types/src/api/web/app/permission.ts
@@ -1,4 +1,4 @@
-import { PermissionLevel, PlanType } from "../../../sdk"
+import { PermissionLevel } from "../../../sdk"
 
 export interface ResourcePermissionInfo {
   role: string

From 3a68b1ae30ccdb2f95aa16dd92af50f0af7710b5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 12 Sep 2024 16:16:40 +0100
Subject: [PATCH 033/101] Fixing test case - removing licensing.

---
 .../src/api/routes/tests/permissions.spec.ts  | 22 +------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index 73c0952bf1..0f059998ae 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -144,27 +144,7 @@ describe("/permission", () => {
       await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
     })
 
-    it("should ignore the view permissions if the flag is not on", async () => {
-      await config.api.permission.add({
-        roleId: STD_ROLE_ID,
-        resourceId: view.id,
-        level: PermissionLevel.READ,
-      })
-      await config.api.permission.revoke({
-        roleId: STD_ROLE_ID,
-        resourceId: table._id,
-        level: PermissionLevel.READ,
-      })
-      // replicate changes before checking permissions
-      await config.publish()
-
-      await config.api.viewV2.publicSearch(view.id, undefined, {
-        status: 401,
-      })
-    })
-
-    it("should use the view permissions if the flag is on", async () => {
-      mocks.licenses.useViewPermissions()
+    it("should use the view permissions", async () => {
       await config.api.permission.add({
         roleId: STD_ROLE_ID,
         resourceId: view.id,

From ef3f726346fb4dfba1f78c60b18e7108804c957c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 12 Sep 2024 16:23:00 +0100
Subject: [PATCH 034/101] Fixing test cases.

---
 packages/backend-core/tests/core/utilities/mocks/licenses.ts | 4 ----
 packages/server/src/api/routes/tests/rowAction.spec.ts       | 4 ----
 packages/server/src/api/routes/tests/viewV2.spec.ts          | 1 -
 packages/types/src/sdk/licensing/feature.ts                  | 1 +
 4 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
index ec7d2af794..2d8e81d125 100644
--- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
@@ -102,10 +102,6 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
-export const useViewPermissions = () => {
-  return useFeature(Feature.VIEW_PERMISSIONS)
-}
-
 export const useViewReadonlyColumns = () => {
   return useFeature(Feature.VIEW_READONLY_COLUMNS)
 }
diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts
index 3f4447c50d..ef7d2afbba 100644
--- a/packages/server/src/api/routes/tests/rowAction.spec.ts
+++ b/packages/server/src/api/routes/tests/rowAction.spec.ts
@@ -763,10 +763,6 @@ describe("/rowsActions", () => {
     })
 
     describe("role permission checks", () => {
-      beforeAll(() => {
-        mocks.licenses.useViewPermissions()
-      })
-
       afterAll(() => {
         mocks.licenses.useCloudFree()
       })
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 3ca28f31aa..f86291e9cd 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -2297,7 +2297,6 @@ describe.each([
 
     describe("permissions", () => {
       beforeEach(async () => {
-        mocks.licenses.useViewPermissions()
         await Promise.all(
           Array.from({ length: 10 }, () => config.api.row.save(table._id!, {}))
         )
diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts
index 9d09f1d14f..286faaa218 100644
--- a/packages/types/src/sdk/licensing/feature.ts
+++ b/packages/types/src/sdk/licensing/feature.ts
@@ -13,6 +13,7 @@ export enum Feature {
   APP_BUILDERS = "appBuilders",
   OFFLINE = "offline",
   EXPANDED_PUBLIC_API = "expandedPublicApi",
+  // deprecated - no longer licensed
   VIEW_PERMISSIONS = "viewPermissions",
   VIEW_READONLY_COLUMNS = "viewReadonlyColumns",
   BUDIBASE_AI = "budibaseAI",

From b644dd47b9fc20bcde16e3a52e1ffefd2248157c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 12 Sep 2024 16:27:18 +0100
Subject: [PATCH 035/101] build fix.

---
 packages/worker/src/api/routes/global/tests/roles.spec.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/worker/src/api/routes/global/tests/roles.spec.ts b/packages/worker/src/api/routes/global/tests/roles.spec.ts
index 0c3a1c60a9..35060d65fb 100644
--- a/packages/worker/src/api/routes/global/tests/roles.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts
@@ -35,8 +35,8 @@ describe("/api/global/roles", () => {
 
   const role = new roles.Role(
     db.generateRoleID(ROLE_NAME),
-    roles.BUILTIN_ROLE_IDS.BASIC,
-    permissions.BuiltinPermissionID.READ_ONLY
+    permissions.BuiltinPermissionID.READ_ONLY,
+    { displayName: roles.BUILTIN_ROLE_IDS.BASIC }
   )
 
   beforeAll(async () => {

From 683745d3fd8e0f3d41092ee2fe9d13dcda62930e Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 12 Sep 2024 17:14:16 +0100
Subject: [PATCH 036/101] Update account-portal ref (#14571)

* Update account-portal ref

* update account-portal ref

---------

Co-authored-by: Michael Drury <me@michaeldrury.co.uk>
---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 7899d07904..905773d708 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 7899d07904d89d48954dd500da7b5dec32b781dd
+Subproject commit 905773d70854a43c6ef2461c7a49671bff56fedc

From 57cf5421015b731dfbe78cfb5956c16d28054103 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 12 Sep 2024 17:32:42 +0100
Subject: [PATCH 037/101] updating JS dependencies to get rid of vulns

---
 package.json                 |   5 +-
 packages/server/package.json |   2 +-
 yarn.lock                    | 254 ++++++++---------------------------
 3 files changed, 61 insertions(+), 200 deletions(-)

diff --git a/package.json b/package.json
index 582e35180e..fc7e202e3d 100644
--- a/package.json
+++ b/package.json
@@ -117,7 +117,10 @@
     "axios": "1.6.3",
     "xml2js": "0.6.2",
     "unset-value": "2.0.1",
-    "passport": "0.6.0"
+    "passport": "0.6.0",
+    "fast-xml-parser": "4.4.1",
+    "@azure/identity": "4.2.1",
+    "kind-of": "6.0.3"
   },
   "engines": {
     "node": ">=20.0.0 <21.0.0"
diff --git a/packages/server/package.json b/packages/server/package.json
index df0ece7bb6..94bd09b7b4 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -63,7 +63,7 @@
     "@koa/router": "8.0.8",
     "@socket.io/redis-adapter": "^8.2.1",
     "@types/xml2js": "^0.4.14",
-    "airtable": "0.10.1",
+    "airtable": "^0.12.2",
     "arangojs": "7.2.0",
     "archiver": "7.0.1",
     "aws-sdk": "2.1030.0",
diff --git a/yarn.lock b/yarn.lock
index 3727e08d0a..09d2cd9b7d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -759,20 +759,20 @@
     "@azure/abort-controller" "^1.0.0"
     tslib "^2.2.0"
 
-"@azure/identity@^3.4.1":
-  version "3.4.2"
-  resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de"
-  integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA==
+"@azure/identity@4.2.1", "@azure/identity@^3.4.1":
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c"
+  integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q==
   dependencies:
     "@azure/abort-controller" "^1.0.0"
     "@azure/core-auth" "^1.5.0"
     "@azure/core-client" "^1.4.0"
     "@azure/core-rest-pipeline" "^1.1.0"
     "@azure/core-tracing" "^1.0.0"
-    "@azure/core-util" "^1.6.1"
+    "@azure/core-util" "^1.3.0"
     "@azure/logger" "^1.0.0"
-    "@azure/msal-browser" "^3.5.0"
-    "@azure/msal-node" "^2.5.1"
+    "@azure/msal-browser" "^3.11.1"
+    "@azure/msal-node" "^2.9.2"
     events "^3.0.0"
     jws "^4.0.0"
     open "^8.0.0"
@@ -803,24 +803,24 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/msal-browser@^3.5.0":
-  version "3.18.0"
-  resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.18.0.tgz#dabbde2c53195a2e0ec8404f61f337c82c159b71"
-  integrity sha512-jvK5bDUWbpOaJt2Io/rjcaOVcUzkqkrCme/WntdV1SMUc67AiTcEdKuY6G/nMQ7N5Cfsk9SfpugflQwDku53yg==
+"@azure/msal-browser@^3.11.1":
+  version "3.23.0"
+  resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b"
+  integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw==
   dependencies:
-    "@azure/msal-common" "14.13.0"
+    "@azure/msal-common" "14.14.2"
 
-"@azure/msal-common@14.13.0":
-  version "14.13.0"
-  resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.13.0.tgz#7377b4909a46d19ea91dadd24af7705e6aa947af"
-  integrity sha512-b4M/tqRzJ4jGU91BiwCsLTqChveUEyFK3qY2wGfZ0zBswIBZjAxopx5CYt5wzZFKuN15HqRDYXQbztttuIC3nA==
+"@azure/msal-common@14.14.2":
+  version "14.14.2"
+  resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4"
+  integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog==
 
-"@azure/msal-node@^2.5.1":
-  version "2.10.0"
-  resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.10.0.tgz#0b893ab05dbef5c963aba080c88a0330393c4973"
-  integrity sha512-JxsSE0464a8IA/+q5EHKmchwNyUFJHtCH00tSXsLaOddwLjG6yVvTH6lGgPcWMhO7YWUXj/XVgVgeE9kZtsPUQ==
+"@azure/msal-node@^2.9.2":
+  version "2.13.1"
+  resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47"
+  integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw==
   dependencies:
-    "@azure/msal-common" "14.13.0"
+    "@azure/msal-common" "14.14.2"
     jsonwebtoken "^9.0.0"
     uuid "^8.3.0"
 
@@ -6040,11 +6040,6 @@
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
   integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
 
-"@types/qs@^6.9.15":
-  version "6.9.15"
-  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.15.tgz#adde8a060ec9c305a82de1babc1056e73bd64dce"
-  integrity sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==
-
 "@types/range-parser@*":
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
@@ -6895,16 +6890,16 @@ aggregate-error@^3.0.0:
     clean-stack "^2.0.0"
     indent-string "^4.0.0"
 
-airtable@0.10.1:
-  version "0.10.1"
-  resolved "https://registry.yarnpkg.com/airtable/-/airtable-0.10.1.tgz#0b311002bb44b39f19bf7c4bd2d47d75c733bf87"
-  integrity sha512-obFW+R3ly2mKtCj0D/xto0ggUvYwdM0RJT3VJ9wvgqoxDkzqg2mNtkuTNfYjF6wWQA0GvoHG9guqzgBBqFjItw==
+airtable@^0.12.2:
+  version "0.12.2"
+  resolved "https://registry.yarnpkg.com/airtable/-/airtable-0.12.2.tgz#e53e66db86744f9bc684faa58881d6c9c12f0e6f"
+  integrity sha512-HS3VytUBTKj8A0vPl7DDr5p/w3IOGv6RXL0fv7eczOWAtj9Xe8ri4TAiZRXoOyo+Z/COADCj+oARFenbxhmkIg==
   dependencies:
     "@types/node" ">=8.0.0 <15"
     abort-controller "^3.0.0"
     abortcontroller-polyfill "^1.4.0"
-    lodash "^4.17.19"
-    node-fetch "^2.6.1"
+    lodash "^4.17.21"
+    node-fetch "^2.6.7"
 
 ajv-formats@^2.0.2:
   version "2.1.1"
@@ -7453,30 +7448,7 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0:
-  version "0.24.0"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
-  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
-  dependencies:
-    follow-redirects "^1.14.4"
-
-axios@1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
-  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
-  dependencies:
-    follow-redirects "^1.15.0"
-    form-data "^4.0.0"
-    proxy-from-env "^1.1.0"
-
-axios@^0.21.1:
-  version "0.21.4"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
-  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
-  dependencies:
-    follow-redirects "^1.14.0"
-
-axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
+axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11318,27 +11290,13 @@ fast-url-parser@^1.1.3:
   dependencies:
     punycode "^1.3.2"
 
-fast-xml-parser@4.2.5:
-  version "4.2.5"
-  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f"
-  integrity "sha1-pnR6CSlqbLNPKuY0AZvxc487Qh8= sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g=="
-  dependencies:
-    strnum "^1.0.5"
-
-fast-xml-parser@^4.1.3:
+fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
   version "4.4.1"
   resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f"
   integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==
   dependencies:
     strnum "^1.0.5"
 
-fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
-  version "4.4.0"
-  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz#341cc98de71e9ba9e651a67f41f1752d1441a501"
-  integrity sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==
-  dependencies:
-    strnum "^1.0.5"
-
 fastq@^1.6.0:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c"
@@ -11586,11 +11544,6 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
-follow-redirects@^1.14.0, follow-redirects@^1.14.4:
-  version "1.15.9"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
-  integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==
-
 follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
@@ -12684,12 +12637,7 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1:
-  version "3.8.1"
-  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
-  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
-
-http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13214,7 +13162,7 @@ is-boolean-object@^1.1.0:
     call-bind "^1.0.2"
     has-tostringtag "^1.0.0"
 
-is-buffer@^1.1.5, is-buffer@~1.1.6:
+is-buffer@~1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
   integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
@@ -13700,6 +13648,11 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
+isobject@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
+  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
+
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -14609,14 +14562,7 @@ kill-port@^1.6.1:
     get-them-args "1.3.2"
     shell-exec "1.0.2"
 
-kind-of@^3.0.2, kind-of@^3.1.0:
-  version "3.2.2"
-  resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
-  integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
-  dependencies:
-    is-buffer "^1.1.5"
-
-kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
+kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
   version "6.0.3"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd"
   integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==
@@ -15441,7 +15387,7 @@ lodash.xor@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
   integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ==
 
-lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
+lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -16259,7 +16205,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@^1.5.2:
+msgpackr@1.10.1, msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16453,25 +16399,13 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
-  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
-
-node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
+node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-fetch@^2.6.9, node-fetch@^2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
-  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -17100,21 +17034,6 @@ openai@^4.52.1:
     node-fetch "^2.6.7"
     web-streams-polyfill "^3.2.1"
 
-openai@^4.59.0:
-  version "4.59.0"
-  resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08"
-  integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==
-  dependencies:
-    "@types/node" "^18.11.18"
-    "@types/node-fetch" "^2.6.4"
-    "@types/qs" "^6.9.15"
-    abort-controller "^3.0.0"
-    agentkeepalive "^4.2.1"
-    form-data-encoder "1.7.2"
-    formdata-node "^4.3.2"
-    node-fetch "^2.6.7"
-    qs "^6.10.3"
-
 openapi-response-validator@^9.2.0:
   version "9.3.1"
   resolved "https://registry.yarnpkg.com/openapi-response-validator/-/openapi-response-validator-9.3.1.tgz#54284d8be608ef53283cbe7448accce8106b1c56"
@@ -17637,15 +17556,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@^0.4.0:
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
-  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
-  dependencies:
-    passport-strategy "1.x.x"
-    pause "0.0.1"
-
-passport@^0.6.0:
+passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -18951,7 +18862,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.28, psl@^1.1.33:
+psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -20023,11 +19934,6 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
-sax@>=0.1.1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
-  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
-
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -20100,33 +20006,13 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
-  version "5.7.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
-  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
-
-semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
+"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
-  version "6.3.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
-  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
-
-semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
-  version "7.6.3"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
-  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
-
-semver@~2.3.1:
-  version "2.3.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
-  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
-
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -21689,7 +21575,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
+tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21699,14 +21585,6 @@ touch@^3.1.0:
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
-tough-cookie@~2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
-  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
-  dependencies:
-    psl "^1.1.28"
-    punycode "^2.1.1"
-
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -22235,6 +22113,14 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
+unset-value@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
+  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
+  dependencies:
+    has-value "^2.0.2"
+    isobject "^4.0.0"
+
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -23014,33 +22900,10 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x:
-  version "0.1.14"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
-  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
-  dependencies:
-    sax ">=0.1.1"
-
-xml2js@0.4.19:
-  version "0.4.19"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
-  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~9.0.1"
-
-xml2js@0.5.0:
-  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
-  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~11.0.0"
-
-xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.4.23"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
-  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
+xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
+  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -23050,11 +22913,6 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
-xmlbuilder@~9.0.1:
-  version "9.0.7"
-  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
-  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
-
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"

From 01a1379335985efa38da2cf0d691985625282e61 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 12 Sep 2024 17:43:13 +0100
Subject: [PATCH 038/101] pin airtable dependency

---
 packages/server/package.json | 2 +-
 yarn.lock                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 94bd09b7b4..6dfd528963 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -63,7 +63,7 @@
     "@koa/router": "8.0.8",
     "@socket.io/redis-adapter": "^8.2.1",
     "@types/xml2js": "^0.4.14",
-    "airtable": "^0.12.2",
+    "airtable": "0.12.2",
     "arangojs": "7.2.0",
     "archiver": "7.0.1",
     "aws-sdk": "2.1030.0",
diff --git a/yarn.lock b/yarn.lock
index 09d2cd9b7d..110cbd7a15 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6890,7 +6890,7 @@ aggregate-error@^3.0.0:
     clean-stack "^2.0.0"
     indent-string "^4.0.0"
 
-airtable@^0.12.2:
+airtable@0.12.2:
   version "0.12.2"
   resolved "https://registry.yarnpkg.com/airtable/-/airtable-0.12.2.tgz#e53e66db86744f9bc684faa58881d6c9c12f0e6f"
   integrity sha512-HS3VytUBTKj8A0vPl7DDr5p/w3IOGv6RXL0fv7eczOWAtj9Xe8ri4TAiZRXoOyo+Z/COADCj+oARFenbxhmkIg==

From dabe401753c980ab4a1a038658a9c4ebb6604ddd Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 12 Sep 2024 17:07:58 +0000
Subject: [PATCH 039/101] Bump version to 2.32.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 30a02755c5..070c0f5315 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.2",
+  "version": "2.32.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 7d6cce20e972b3fa9ef1edb988c0afba53f3119b Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Thu, 12 Sep 2024 21:02:50 +0100
Subject: [PATCH 040/101] fix getAvailableBindings again, aka my personal hell

---
 .../SetupPanel/AutomationBlockSetup.svelte           | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index 1e2e409243..be7286a79e 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -700,14 +700,12 @@
         allSteps[idx]?.stepId === ActionStepID.LOOP &&
         allSteps.some(x => x.blockToLoop === block.id)
       let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {}
-      if (wasLoopBlock) {
-        break
+      if (allSteps[idx]?.name.includes("Looping")) {
+        isLoopBlock = true
+        loopBlockCount++
       }
       let bindingName =
-        automation.stepNames?.[allSteps[idx - loopBlockCount].id] ||
-        !isLoopBlock
-          ? allSteps[idx]?.name
-          : allSteps[idx - 1]?.name
+        automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name
 
       if (isLoopBlock) {
         schema = {
@@ -756,7 +754,7 @@
 
       if (wasLoopBlock) {
         loopBlockCount++
-        continue
+        schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
       }
       Object.entries(schema).forEach(([name, value]) =>
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)

From aaa7c9162f6063bac88dec0d93e8556015c08ea6 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 13 Sep 2024 11:59:18 +0100
Subject: [PATCH 041/101] account Portal

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index cd64894684..723eff3807 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit cd648946847aade0ac1f53a1498596670c8a0aee
+Subproject commit 723eff3807fa24fa30de69f9f556910e190a29d1

From 7e767e408955687f8841acb6fb68e9b34e44efd7 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 13 Sep 2024 12:25:39 +0100
Subject: [PATCH 042/101] some pr comments

---
 .../tests/scenarios/looping.spec.ts           |  9 +++--
 .../tests/scenarios/scenarios.spec.ts         |  2 +-
 .../tests/utilities/AutomationTestBuilder.ts  | 38 +++++++++++--------
 .../documents/app/automation/automation.ts    |  1 +
 4 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts
index f229610542..6dde05ddb8 100644
--- a/packages/server/src/automations/tests/scenarios/looping.spec.ts
+++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts
@@ -255,12 +255,15 @@ describe("Loop automations", () => {
           option: LoopStepType.ARRAY,
           binding: [1, 2, 3],
         },
-        "FirstLoopStep"
+        { stepName: "FirstLoopStep" }
+      )
+      .serverLog(
+        { text: "Message {{loop.currentItem}}" },
+        { stepName: "FirstLoopLog" }
       )
-      .serverLog({ text: "Message {{loop.currentItem}}" }, "FirstLoopLog")
       .serverLog(
         { text: "{{steps.FirstLoopLog.iterations}}" },
-        "FirstLoopIterationLog"
+        { stepName: "FirstLoopIterationLog" }
       )
       .run()
 
diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts
index 398ef20100..49c05984fe 100644
--- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts
+++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts
@@ -217,7 +217,7 @@ describe("Automation Scenarios", () => {
           {
             tableId: table._id!,
           },
-          "InitialQueryStep"
+          { stepName: "InitialQueryStep" }
         )
         .deleteRow({
           tableId: table._id!,
diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
index 34c25d7004..7528ea0f2e 100644
--- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
+++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
@@ -105,74 +105,80 @@ class BaseStepBuilder {
   }
 
   // STEPS
-  createRow(inputs: CreateRowStepInputs, stepName?: string): this {
+  createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.CREATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.CREATE_ROW,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
 
-  updateRow(inputs: UpdateRowStepInputs, stepName?: string): this {
+  updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.UPDATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
 
-  deleteRow(inputs: DeleteRowStepInputs, stepName?: string): this {
+  deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.DELETE_ROW,
       BUILTIN_ACTION_DEFINITIONS.DELETE_ROW,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
 
-  sendSmtpEmail(inputs: SmtpEmailStepInputs, stepName?: string): this {
+  sendSmtpEmail(
+    inputs: SmtpEmailStepInputs,
+    opts?: { stepName?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.SEND_EMAIL_SMTP,
       BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
 
-  executeQuery(inputs: ExecuteQueryStepInputs, stepName?: string): this {
+  executeQuery(
+    inputs: ExecuteQueryStepInputs,
+    opts?: { stepName?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.EXECUTE_QUERY,
       BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
 
-  queryRows(inputs: QueryRowsStepInputs, stepName?: string): this {
+  queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.QUERY_ROWS,
       BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
-  loop(inputs: LoopStepInputs, stepName?: string): this {
+  loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.LOOP,
       BUILTIN_ACTION_DEFINITIONS.LOOP,
       inputs,
-      stepName
+      opts?.stepName
     )
   }
 
-  serverLog(input: ServerLogStepInputs, stepName?: string): this {
+  serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.SERVER_LOG,
       BUILTIN_ACTION_DEFINITIONS.SERVER_LOG,
       input,
-      stepName
+      opts?.stepName
     )
   }
 }
diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts
index 78d22c787f..effe99a328 100644
--- a/packages/types/src/documents/app/automation/automation.ts
+++ b/packages/types/src/documents/app/automation/automation.ts
@@ -124,6 +124,7 @@ export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
     trigger: AutomationTrigger
+    // stepNames is used to lookup step names from their correspnding step ID.
     stepNames?: Record<string, string>
   }
   screenId?: string

From 663c0f20af3b4057d18db996020f7efd37e53eb5 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 13 Sep 2024 14:14:36 +0100
Subject: [PATCH 043/101] Don't use display names as role names, and restore
 usual names for built in roles

---
 packages/backend-core/src/security/roles.ts   | 84 +++++++++++++------
 packages/server/src/api/controllers/role.ts   |  2 +-
 .../src/api/routes/global/tests/roles.spec.ts |  1 +
 3 files changed, 59 insertions(+), 28 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index a7210ec2b8..65339832cf 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -48,9 +48,14 @@ export class Role implements RoleDoc {
   permissions: Record<string, PermissionLevel[]> = {}
   uiMetadata?: RoleUIMetadata
 
-  constructor(id: string, permissionId: string, uiMetadata?: RoleUIMetadata) {
+  constructor(
+    id: string,
+    name: string,
+    permissionId: string,
+    uiMetadata?: RoleUIMetadata
+  ) {
     this._id = id
-    this.name = uiMetadata?.displayName || id
+    this.name = name
     this.uiMetadata = uiMetadata
     this.permissionId = permissionId
     // version for managing the ID - removing the role_ when responding
@@ -64,31 +69,56 @@ export class Role implements RoleDoc {
 }
 
 const BUILTIN_ROLES = {
-  ADMIN: new Role(BUILTIN_IDS.ADMIN, BuiltinPermissionID.ADMIN, {
-    displayName: "App admin",
-    description: "Can do everything",
-    color: RoleColor.ADMIN,
-  }).addInheritance(BUILTIN_IDS.POWER),
-  POWER: new Role(BUILTIN_IDS.POWER, BuiltinPermissionID.POWER, {
-    displayName: "App power user",
-    description: "An app user with more access",
-    color: RoleColor.POWER,
-  }).addInheritance(BUILTIN_IDS.BASIC),
-  BASIC: new Role(BUILTIN_IDS.BASIC, BuiltinPermissionID.WRITE, {
-    displayName: "App user",
-    description: "Any logged in user",
-    color: RoleColor.BASIC,
-  }).addInheritance(BUILTIN_IDS.PUBLIC),
-  PUBLIC: new Role(BUILTIN_IDS.PUBLIC, BuiltinPermissionID.PUBLIC, {
-    displayName: "Public user",
-    description: "Accessible to anyone",
-    color: RoleColor.PUBLIC,
-  }),
-  BUILDER: new Role(BUILTIN_IDS.BUILDER, BuiltinPermissionID.ADMIN, {
-    displayName: "Builder user",
-    description: "Users that can edit this app",
-    color: RoleColor.BUILDER,
-  }),
+  ADMIN: new Role(
+    BUILTIN_IDS.ADMIN,
+    BUILTIN_IDS.ADMIN,
+    BuiltinPermissionID.ADMIN,
+    {
+      displayName: "App admin",
+      description: "Can do everything",
+      color: RoleColor.ADMIN,
+    }
+  ).addInheritance(BUILTIN_IDS.POWER),
+  POWER: new Role(
+    BUILTIN_IDS.POWER,
+    BUILTIN_IDS.POWER,
+    BuiltinPermissionID.POWER,
+    {
+      displayName: "App power user",
+      description: "An app user with more access",
+      color: RoleColor.POWER,
+    }
+  ).addInheritance(BUILTIN_IDS.BASIC),
+  BASIC: new Role(
+    BUILTIN_IDS.BASIC,
+    BUILTIN_IDS.BASIC,
+    BuiltinPermissionID.WRITE,
+    {
+      displayName: "App user",
+      description: "Any logged in user",
+      color: RoleColor.BASIC,
+    }
+  ).addInheritance(BUILTIN_IDS.PUBLIC),
+  PUBLIC: new Role(
+    BUILTIN_IDS.PUBLIC,
+    BUILTIN_IDS.PUBLIC,
+    BuiltinPermissionID.PUBLIC,
+    {
+      displayName: "Public user",
+      description: "Accessible to anyone",
+      color: RoleColor.PUBLIC,
+    }
+  ),
+  BUILDER: new Role(
+    BUILTIN_IDS.BUILDER,
+    BUILTIN_IDS.BUILDER,
+    BuiltinPermissionID.ADMIN,
+    {
+      displayName: "Builder user",
+      description: "Users that can edit this app",
+      color: RoleColor.BUILDER,
+    }
+  ),
 }
 
 export function getBuiltinRoles(): { [key: string]: RoleDoc } {
diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index ee1c223952..b6b9ac1a29 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -89,7 +89,7 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     ctx.throw(400, "Cannot change custom role name")
   }
 
-  const role = new roles.Role(_id, permissionId, {
+  const role = new roles.Role(_id, name, permissionId, {
     displayName: uiMetadata?.displayName || name,
     description: uiMetadata?.description || "Custom role",
     color: uiMetadata?.color || RoleColor.DEFAULT_CUSTOM,
diff --git a/packages/worker/src/api/routes/global/tests/roles.spec.ts b/packages/worker/src/api/routes/global/tests/roles.spec.ts
index 35060d65fb..11de06328e 100644
--- a/packages/worker/src/api/routes/global/tests/roles.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts
@@ -35,6 +35,7 @@ describe("/api/global/roles", () => {
 
   const role = new roles.Role(
     db.generateRoleID(ROLE_NAME),
+    ROLE_NAME,
     permissions.BuiltinPermissionID.READ_ONLY,
     { displayName: roles.BUILTIN_ROLE_IDS.BASIC }
   )

From 04daa423cfe326aba255de37bd4bb5b6d042fc0e Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 13 Sep 2024 16:03:37 +0100
Subject: [PATCH 044/101] small fix for edge case

---
 .../automation/SetupPanel/AutomationBlockSetup.svelte  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index be7286a79e..eb85cb19ba 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -62,6 +62,7 @@
   } from "@budibase/types"
   import { FIELDS } from "constants/backend"
   import PropField from "./PropField.svelte"
+  import { name } from "helpers/validation/yup/app"
 
   export let block
   export let testData
@@ -638,8 +639,7 @@
       }
       /* End special cases for generating custom schemas based on triggers */
 
-      let hasUserDefinedName =
-        automation.stepNames?.[allSteps[idx - loopBlockCount]]?.id
+      let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
       if (isLoopBlock) {
         runtimeName = `loop.${name}`
       } else if (block.name.startsWith("JS")) {
@@ -756,10 +756,12 @@
         loopBlockCount++
         schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
       }
-      Object.entries(schema).forEach(([name, value]) =>
+      Object.entries(schema).forEach(([name, value]) => {
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)
-      )
+      })
+      console.log(bindings)
     }
+    console.log(bindings)
     return bindings
   }
 

From 59549b74b5bc615317cb4994f876808e285c67ef Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 13 Sep 2024 16:04:13 +0100
Subject: [PATCH 045/101] js scripting edge case

---
 .../automation/SetupPanel/AutomationBlockSetup.svelte           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index eb85cb19ba..fe9710164b 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -644,7 +644,7 @@
         runtimeName = `loop.${name}`
       } else if (block.name.startsWith("JS")) {
         runtimeName = hasUserDefinedName
-          ? `stepsByName.${bindingName}.${name}`
+          ? `stepsByName[${bindingName}].${name}`
           : `steps[${idx - loopBlockCount}].${name}`
       } else {
         runtimeName = hasUserDefinedName

From 70b9d085162a9b8394a450bd0b80e46a44e2d4e7 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 13 Sep 2024 16:10:38 +0100
Subject: [PATCH 046/101] lint

---
 .../automation/SetupPanel/AutomationBlockSetup.svelte          | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index fe9710164b..af67ae8d22 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -62,7 +62,6 @@
   } from "@budibase/types"
   import { FIELDS } from "constants/backend"
   import PropField from "./PropField.svelte"
-  import { name } from "helpers/validation/yup/app"
 
   export let block
   export let testData
@@ -759,9 +758,7 @@
       Object.entries(schema).forEach(([name, value]) => {
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)
       })
-      console.log(bindings)
     }
-    console.log(bindings)
     return bindings
   }
 

From 8222e1df3fd546c926d414144f7ad1fc98ffaec2 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 13 Sep 2024 16:33:01 +0100
Subject: [PATCH 047/101] refs

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 723eff3807..25c6ed45c8 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 723eff3807fa24fa30de69f9f556910e190a29d1
+Subproject commit 25c6ed45c826f27b1ee7a7d1460dc1a386c6c471

From 2a69cb220fefe3b2ff3e2bda6aa647ca8d484d15 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 13 Sep 2024 18:54:47 +0100
Subject: [PATCH 048/101] Fixing an issue where table doesn't exist anymore for
 row trigger, which can cause the service to crash.

---
 packages/server/src/automations/triggers.ts   | 45 +++++++++++++------
 packages/server/src/sdk/app/tables/getters.ts |  9 ++++
 packages/types/src/sdk/automations/index.ts   |  6 +++
 3 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts
index 18b033cdcf..110ccfa37a 100644
--- a/packages/server/src/automations/triggers.ts
+++ b/packages/server/src/automations/triggers.ts
@@ -18,6 +18,7 @@ import {
   SearchFilters,
   AutomationStoppedReason,
   AutomationStatus,
+  AutomationRowEvent,
 } from "@budibase/types"
 import { executeInThread } from "../threads/automation"
 import { dataFilters, sdk } from "@budibase/shared-core"
@@ -28,6 +29,7 @@ const JOB_OPTS = {
   removeOnFail: true,
 }
 import * as automationUtils from "../automations/automationUtils"
+import { doesTableExist } from "../sdk/app/tables/getters"
 
 async function getAllAutomations() {
   const db = context.getAppDB()
@@ -38,25 +40,35 @@ async function getAllAutomations() {
 }
 
 async function queueRelevantRowAutomations(
-  event: { appId: string; row: Row; oldRow: Row },
-  eventType: string
+  event: AutomationRowEvent,
+  eventType: AutomationEventType
 ) {
+  const tableId = event.row.tableId
   if (event.appId == null) {
     throw `No appId specified for ${eventType} - check event emitters.`
   }
 
+  // make sure table exists and is valid before proceeding
+  if (!tableId || !(await doesTableExist(tableId))) {
+    return
+  }
+
   await context.doInAppContext(event.appId, async () => {
     let automations = await getAllAutomations()
 
     // filter down to the correct event type and enabled automations
+    // make sure it is the correct table ID as well
     automations = automations.filter(automation => {
       const trigger = automation.definition.trigger
-      return trigger && trigger.event === eventType && !automation.disabled
+      return (
+        trigger &&
+        trigger.event === eventType &&
+        !automation.disabled &&
+        trigger?.inputs?.tableId === event.row.tableId
+      )
     })
 
     for (const automation of automations) {
-      const automationDef = automation.definition
-      const automationTrigger = automationDef?.trigger
       // don't queue events which are for dev apps, only way to test automations is
       // running tests on them, in production the test flag will never
       // be checked due to lazy evaluation (first always false)
@@ -72,11 +84,7 @@ async function queueRelevantRowAutomations(
         row: event.row,
         oldRow: event.oldRow,
       })
-      if (
-        automationTrigger?.inputs &&
-        automationTrigger.inputs.tableId === event.row.tableId &&
-        shouldTrigger
-      ) {
+      if (shouldTrigger) {
         try {
           await automationQueue.add({ automation, event }, JOB_OPTS)
         } catch (e) {
@@ -87,6 +95,17 @@ async function queueRelevantRowAutomations(
   })
 }
 
+async function queueRowAutomations(
+  event: AutomationRowEvent,
+  type: AutomationEventType
+) {
+  try {
+    await queueRelevantRowAutomations(event, type)
+  } catch (err: any) {
+    logging.logWarn("Unable to process row event", err)
+  }
+}
+
 emitter.on(
   AutomationEventType.ROW_SAVE,
   async function (event: UpdatedRowEventEmitter) {
@@ -94,7 +113,7 @@ emitter.on(
     if (!event || !event.row || !event.row.tableId) {
       return
     }
-    await queueRelevantRowAutomations(event, AutomationEventType.ROW_SAVE)
+    await queueRowAutomations(event, AutomationEventType.ROW_SAVE)
   }
 )
 
@@ -103,7 +122,7 @@ emitter.on(AutomationEventType.ROW_UPDATE, async function (event) {
   if (!event || !event.row || !event.row.tableId) {
     return
   }
-  await queueRelevantRowAutomations(event, AutomationEventType.ROW_UPDATE)
+  await queueRowAutomations(event, AutomationEventType.ROW_UPDATE)
 })
 
 emitter.on(AutomationEventType.ROW_DELETE, async function (event) {
@@ -111,7 +130,7 @@ emitter.on(AutomationEventType.ROW_DELETE, async function (event) {
   if (!event || !event.row || !event.row.tableId) {
     return
   }
-  await queueRelevantRowAutomations(event, AutomationEventType.ROW_DELETE)
+  await queueRowAutomations(event, AutomationEventType.ROW_DELETE)
 })
 
 function rowPassesFilters(row: Row, filters: SearchFilters) {
diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts
index 27e9962a1a..5ff000fe12 100644
--- a/packages/server/src/sdk/app/tables/getters.ts
+++ b/packages/server/src/sdk/app/tables/getters.ts
@@ -101,6 +101,15 @@ export async function getTable(tableId: string): Promise<Table> {
   return await processTable(output)
 }
 
+export async function doesTableExist(tableId: string): Promise<boolean> {
+  try {
+    const table = await getTable(tableId)
+    return !!table
+  } catch (err) {
+    return false
+  }
+}
+
 export async function getAllTables() {
   const [internal, external] = await Promise.all([
     getAllInternalTables(),
diff --git a/packages/types/src/sdk/automations/index.ts b/packages/types/src/sdk/automations/index.ts
index 5ea22148a5..d04f126c32 100644
--- a/packages/types/src/sdk/automations/index.ts
+++ b/packages/types/src/sdk/automations/index.ts
@@ -15,4 +15,10 @@ export interface AutomationData {
   automation: Automation
 }
 
+export interface AutomationRowEvent {
+  appId: string
+  row: Row
+  oldRow: Row
+}
+
 export type AutomationJob = Job<AutomationData>

From 285d264d5fc26b2a8d12784c32792ce21b329b4a Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 13 Sep 2024 18:14:08 +0000
Subject: [PATCH 049/101] Bump version to 2.32.4

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 070c0f5315..a9c50ea4b5 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.3",
+  "version": "2.32.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From c85bc88bf9099d2e4f9082e5bb2baff00fedeaa9 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 16 Sep 2024 11:54:04 +0100
Subject: [PATCH 050/101] draft of openai impl

---
 packages/backend-core/src/configs/configs.ts  |  9 ++++
 .../server/src/automations/steps/openai.ts    | 51 ++++++++++++++-----
 2 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 0d189e3f7d..00ccee8c4e 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -1,4 +1,6 @@
 import {
+  AIConfig,
+  AIInnerConfig,
   Config,
   ConfigType,
   GoogleConfig,
@@ -254,3 +256,10 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
   const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
   return config?.config
 }
+
+// AI
+export async function getAIConfig(): Promise<AIInnerConfig | undefined> {
+  const config = await getConfig<AIConfig>(ConfigType.AI)
+  return config?.config
+}
+
diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 1c148b2e73..d8017d0ceb 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -10,6 +10,7 @@ import {
 } from "@budibase/types"
 import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
+import * as pro from "@budibase/pro"
 
 enum Model {
   GPT_35_TURBO = "gpt-3.5-turbo",
@@ -60,6 +61,23 @@ export const definition: AutomationStepDefinition = {
   },
 }
 
+async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) {
+  const openai = new OpenAI({
+    apiKey: env.OPENAI_API_KEY,
+  })
+
+  const completion = await openai.chat.completions.create({
+    model: inputs.model,
+    messages: [
+      {
+        role: "user",
+        content: inputs.prompt,
+      },
+    ],
+  })
+  return completion?.choices[0]?.message?.content
+}
+
 export async function run({
   inputs,
 }: {
@@ -81,20 +99,27 @@ export async function run({
   }
 
   try {
-    const openai = new OpenAI({
-      apiKey: env.OPENAI_API_KEY,
-    })
+    let response
+    const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
+    const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
 
-    const completion = await openai.chat.completions.create({
-      model: inputs.model,
-      messages: [
-        {
-          role: "user",
-          content: inputs.prompt,
-        },
-      ],
-    })
-    const response = completion?.choices[0]?.message?.content
+    if (budibaseAIEnabled || customConfigsEnabled) {
+      // Enterprise has custom configs
+      // if custom configs are enabled full stop
+      // Don't use their budibase AI credits, unless it uses the budibase AI configuration
+      // TODO: grab the config from the database (maybe wrap this in the pro AI module)
+      // TODO: pass it into the model to execute the prompt
+
+      // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config
+      // Make sure it uses their credits
+      // Should be handled in the LLM wrapper in pro
+      const llm = new pro.ai.LLMWrapper()
+      await llm.init()
+      response = await llm.run(inputs.prompt)
+    } else {
+      // fallback to the default that uses the environment variable for backwards compat
+      response = await legacyOpenAIPrompt(inputs)
+    }
 
     return {
       response,

From eaad70d0316bb5078702c97238799e5241592fd9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 11:59:06 +0100
Subject: [PATCH 051/101] Get table deletion working.

---
 .../src/api/controllers/table/external.ts     |  1 +
 .../integrations/tests/googlesheets.spec.ts   |  7 ++--
 .../integrations/tests/utils/googlesheets.ts  | 34 +++++++++++++++----
 3 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index c3356919c8..94c16f3090 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -74,6 +74,7 @@ export async function destroy(ctx: UserCtx) {
     builderSocket?.emitDatasourceUpdate(ctx, datasource)
     return table
   } catch (err: any) {
+    throw err
     if (err instanceof Error) {
       ctx.throw(400, err.message)
     } else {
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 3e7e0cddc4..9a3cee3881 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -429,13 +429,12 @@ describe("Google Sheets Integration", () => {
       })
     })
 
-    it.skip("can delete a table", async () => {
+    it("can delete a table", async () => {
       await config.api.table.destroy(table._id!, table._rev!)
-      expect(mock.cell("A1")).toEqual(null)
-      expect(mock.cell("B1")).toEqual(null)
+      expect(mock.sheet(table.name)).toBeUndefined()
     })
 
-    it.skip("can delete a row", async () => {
+    it.only("can delete a row", async () => {
       const rows = await config.api.row.fetch(table._id!)
       expect(rows.length).toEqual(2)
 
diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts
index 90f3b3652c..4b17c25b01 100644
--- a/packages/server/src/integrations/tests/utils/googlesheets.ts
+++ b/packages/server/src/integrations/tests/utils/googlesheets.ts
@@ -104,11 +104,17 @@ interface DeleteRangeRequest {
   shiftDimension: WorksheetDimension
 }
 
+// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#DeleteSheetRequest
+interface DeleteSheetRequest {
+  sheetId: number
+}
+
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request
 interface BatchUpdateRequest {
   requests: {
     addSheet?: AddSheetRequest
     deleteRange?: DeleteRangeRequest
+    deleteSheet?: DeleteSheetRequest
   }[]
   includeSpreadsheetInResponse: boolean
   responseRanges: string[]
@@ -445,6 +451,11 @@ export class GoogleSheetsMock {
       }
       if (request.deleteRange) {
         this.handleDeleteRange(request.deleteRange)
+        response.replies.push({})
+      }
+      if (request.deleteSheet) {
+        this.handleDeleteSheet(request.deleteSheet)
+        response.replies.push({})
       }
     }
 
@@ -489,11 +500,16 @@ export class GoogleSheetsMock {
       throw new Error("Only row-based deletes are supported")
     }
 
-    this.iterateRange(range, (cell, value) => {
+    this.iterateRange(range, cell => {
       cell.userEnteredValue = this.createValue(null)
     })
   }
 
+  private handleDeleteSheet(request: DeleteSheetRequest) {
+    const { sheetId } = request
+    this.spreadsheet.sheets.splice(sheetId, 1)
+  }
+
   private handleGetSpreadsheet(): Spreadsheet {
     return this.spreadsheet
   }
@@ -514,17 +530,14 @@ export class GoogleSheetsMock {
     return response
   }
 
-  private iterateRange(
-    range: Required<GridRange>,
-    cb: (cell: CellData) => void
-  ) {
+  private iterateRange(range: GridRange, cb: (cell: CellData) => void) {
     const {
       sheetId,
       startRowIndex,
       endRowIndex,
       startColumnIndex,
       endColumnIndex,
-    } = range
+    } = this.ensureGridRange(range)
 
     for (let row = startRowIndex; row <= endRowIndex; row++) {
       for (let col = startColumnIndex; col <= endColumnIndex; col++) {
@@ -754,7 +767,7 @@ export class GoogleSheetsMock {
     return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex)
   }
 
-  cell(cell: string): Value | undefined {
+  public cell(cell: string): Value | undefined {
     const cellData = this.cellData(cell)
     if (!cellData) {
       return undefined
@@ -762,6 +775,13 @@ export class GoogleSheetsMock {
     return this.cellValue(cellData)
   }
 
+  public sheet(name: string | number): Sheet | undefined {
+    if (typeof name === "number") {
+      return this.getSheetById(name)
+    }
+    return this.getSheetByName(name)
+  }
+
   private getCellNumericIndexes(
     sheet: Sheet | number,
     row: number,

From ddf70415626fad3c0a75baab3e151bb4bbff5848 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 12:06:27 +0100
Subject: [PATCH 052/101] Get row deletion working.

---
 .../server/src/integrations/tests/googlesheets.spec.ts    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 9a3cee3881..5cb67ae816 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -430,15 +430,19 @@ describe("Google Sheets Integration", () => {
     })
 
     it("can delete a table", async () => {
+      expect(mock.sheet(table.name)).toBeDefined()
       await config.api.table.destroy(table._id!, table._rev!)
       expect(mock.sheet(table.name)).toBeUndefined()
     })
 
-    it.only("can delete a row", async () => {
+    it("can delete a row", async () => {
       const rows = await config.api.row.fetch(table._id!)
       expect(rows.length).toEqual(2)
 
-      for (const row of rows) {
+      // Because row IDs in Google Sheets are sequential and determined by the
+      // actual row in the sheet, deleting a row will shift the row IDs down by
+      // one. This is why we reverse the rows before deleting them.
+      for (const row of rows.reverse()) {
         await config.api.row.delete(table._id!, { _id: row._id! })
       }
 

From ea6e2a472ed482a4d7b329f43b32fc39f6eec6e8 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 12:09:04 +0100
Subject: [PATCH 053/101] Remove unused throw.

---
 packages/server/src/api/controllers/table/external.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 94c16f3090..c3356919c8 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -74,7 +74,6 @@ export async function destroy(ctx: UserCtx) {
     builderSocket?.emitDatasourceUpdate(ctx, datasource)
     return table
   } catch (err: any) {
-    throw err
     if (err instanceof Error) {
       ctx.throw(400, err.message)
     } else {

From d6c8ae8ec44dfa8540339b4b6e3dff0534c0da29 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 12:09:50 +0100
Subject: [PATCH 054/101] Remove unused imports.

---
 packages/server/src/integrations/tests/googlesheets.spec.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 5cb67ae816..079e418f3b 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -5,13 +5,11 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import {
   Datasource,
   FieldType,
-  Row,
   SourceName,
   Table,
   TableSourceType,
 } from "@budibase/types"
 import { GoogleSheetsMock } from "./utils/googlesheets"
-import exp from "constants"
 
 describe("Google Sheets Integration", () => {
   const config = new TestConfiguration()

From 9d6fc54a992e2ee0fe389a519d6d158d88f5995d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 16 Sep 2024 16:12:07 +0100
Subject: [PATCH 055/101] Adding function parameter limit control for different
 SQL DBs, every DB has different limits with Postgres being the lowest at 100.
 We need to fix for wide tables which are related.

---
 packages/backend-core/src/sql/sql.ts | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 2b5243f856..b0e7b30067 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core"
 import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
-const MAX_SQS_RELATIONSHIP_FIELDS = 63
 
 function getBaseLimit() {
   const envLimit = environment.SQL_MAX_ROWS
@@ -877,6 +876,22 @@ class InternalBuilder {
     return `'${unaliased}'${separator}${tableField}`
   }
 
+  maxFunctionParameters() {
+    // functions like say json_build_object() in SQL have a limit as to how many can be performed
+    // before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as
+    // for JSON column building as part of relationships. We also have a default limit to avoid very complex
+    // functions being built - it is likely this is not necessary or the best way to do it.
+    switch (this.client) {
+      case SqlClient.SQL_LITE:
+        return 127
+      case SqlClient.POSTGRES:
+        return 100
+      // other DBs don't have a limit, but set some sort of limit
+      default:
+        return 200
+    }
+  }
+
   addJsonRelationships(
     query: Knex.QueryBuilder,
     fromTable: string,
@@ -908,12 +923,10 @@ class InternalBuilder {
       let relationshipFields = fields.filter(
         field => field.split(".")[0] === toAlias
       )
-      if (this.client === SqlClient.SQL_LITE) {
-        relationshipFields = relationshipFields.slice(
-          0,
-          MAX_SQS_RELATIONSHIP_FIELDS
-        )
-      }
+      relationshipFields = relationshipFields.slice(
+        0,
+        Math.floor(this.maxFunctionParameters() / 2)
+      )
       const fieldList: string = relationshipFields
         .map(field => this.buildJsonField(field))
         .join(",")

From 26ad987072e13b4956dca68ee44c681aa8e2aee3 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 16:15:09 +0100
Subject: [PATCH 056/101] Fix Google Sheets pagination.

---
 .../server/src/integrations/googlesheets.ts   | 15 +++++---
 .../integrations/tests/googlesheets.spec.ts   | 37 +++++++++++++++++++
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 0d766ac1ef..dd9bef84ab 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -551,11 +551,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
       await this.connect()
       const hasFilters = dataFilters.hasFilters(query.filters)
       const limit = query.paginate?.limit || 100
-      const page: number =
-        typeof query.paginate?.page === "number"
-          ? query.paginate.page
-          : parseInt(query.paginate?.page || "1")
-      const offset = (page - 1) * limit
+      let offset = query.paginate?.offset || 0
+
+      let page = query.paginate?.page
+      if (typeof page === "string") {
+        page = parseInt(page)
+      }
+      if (page !== undefined) {
+        offset = page * limit
+      }
+
       const sheet = this.client.sheetsByTitle[query.sheet]
       let rows: GoogleSpreadsheetRow[] = []
       if (query.paginate && !hasFilters) {
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 079e418f3b..a1fabc2c04 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import {
   Datasource,
   FieldType,
+  Row,
   SourceName,
   Table,
   TableSourceType,
@@ -208,6 +209,42 @@ describe("Google Sheets Integration", () => {
       expect(row2.name).toEqual("Test Contact 2")
       expect(row2.description).toEqual("original description 2")
     })
+
+    it("can paginate correctly", async () => {
+      await config.api.row.bulkImport(table._id!, {
+        rows: Array.from({ length: 248 }, (_, i) => ({
+          name: `${i}`,
+          description: "",
+        })),
+      })
+
+      let resp = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {},
+        paginate: true,
+        limit: 10,
+      })
+      let rows = resp.rows
+
+      while (resp.hasNextPage) {
+        resp = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query: {},
+          paginate: true,
+          limit: 10,
+          bookmark: resp.bookmark,
+        })
+        rows = rows.concat(resp.rows)
+        if (rows.length > 250) {
+          throw new Error("Too many rows returned")
+        }
+      }
+
+      expect(rows.length).toEqual(250)
+      expect(rows.map(row => row.name)).toEqual(
+        expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`))
+      )
+    })
   })
 
   describe("update", () => {

From 9f21dc88b016ff631f132f1239381ee8c88d0a62 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 16:18:57 +0100
Subject: [PATCH 057/101] Fix lint.

---
 packages/server/src/integrations/tests/googlesheets.spec.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index a1fabc2c04..62d56bb2c2 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -5,7 +5,6 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import {
   Datasource,
   FieldType,
-  Row,
   SourceName,
   Table,
   TableSourceType,

From 27f6fa7de46ba2f5c482cdcc20b4a866181606e1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 16:36:17 +0100
Subject: [PATCH 058/101] Add a test for row exports on Google Sheets.

---
 .../src/integrations/tests/googlesheets.spec.ts   | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 62d56bb2c2..cef4decfa7 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -10,6 +10,7 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import { GoogleSheetsMock } from "./utils/googlesheets"
+import rows from "src/sdk/app/rows"
 
 describe("Google Sheets Integration", () => {
   const config = new TestConfiguration()
@@ -244,6 +245,20 @@ describe("Google Sheets Integration", () => {
         expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`))
       )
     })
+
+    it("can export rows", async () => {
+      const resp = await config.api.row.exportRows(table._id!, {})
+      const parsed = JSON.parse(resp)
+      expect(parsed.length).toEqual(2)
+      expect(parsed[0]).toMatchObject({
+        name: "Test Contact 1",
+        description: "original description 1",
+      })
+      expect(parsed[1]).toMatchObject({
+        name: "Test Contact 2",
+        description: "original description 2",
+      })
+    })
   })
 
   describe("update", () => {

From 68a710699d6ab8fa978d9ee46c0020814b966845 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 16 Sep 2024 18:09:01 +0100
Subject: [PATCH 059/101] Getting external DBs to correctly handle when too
 many fields.

---
 packages/backend-core/src/sql/sql.ts          | 15 +++++--
 .../api/controllers/row/ExternalRequest.ts    | 13 +++---
 .../src/api/routes/tests/search.spec.ts       | 42 +++++++++++++++++++
 3 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index b0e7b30067..ff32026814 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -899,7 +899,7 @@ class InternalBuilder {
   ): Knex.QueryBuilder {
     const sqlClient = this.client
     const knex = this.knex
-    const { resource, tableAliases: aliases, endpoint } = this.query
+    const { resource, tableAliases: aliases, endpoint, meta } = this.query
     const fields = resource?.fields || []
     for (let relationship of relationships) {
       const {
@@ -914,15 +914,22 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
+      const relatedTable = meta.tables?.[toTable]
       const toAlias = aliases?.[toTable] || toTable,
         fromAlias = aliases?.[fromTable] || fromTable
       let toTableWithSchema = this.tableNameWithSchema(toTable, {
         alias: toAlias,
         schema: endpoint.schema,
       })
-      let relationshipFields = fields.filter(
-        field => field.split(".")[0] === toAlias
-      )
+      const requiredFields = [
+        ...(relatedTable?.primary || []),
+        relatedTable?.primaryDisplay,
+      ]
+      let relationshipFields = fields
+        .filter(field => field.split(".")[0] === toAlias)
+        // sort the required fields to first in the list
+        .sort(field => (requiredFields.includes(field) ? 1 : -1))
+
       relationshipFields = relationshipFields.slice(
         0,
         Math.floor(this.maxFunctionParameters() / 2)
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index ac2d1e8c39..9c9bd0b284 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -12,6 +12,7 @@ import {
   OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
+  QueryJson,
   RelationshipFieldMetadata,
   Row,
   SearchFilters,
@@ -161,7 +162,6 @@ export class ExternalRequest<T extends Operation> {
   private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
-  private tableList: Table[]
 
   constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
@@ -170,7 +170,6 @@ export class ExternalRequest<T extends Operation> {
     if (datasource && datasource.entities) {
       this.tables = datasource.entities
     }
-    this.tableList = Object.values(this.tables)
   }
 
   private prepareFilters(
@@ -301,7 +300,6 @@ export class ExternalRequest<T extends Operation> {
         throw "No tables found, fetch tables before query."
       }
       this.tables = this.datasource.entities
-      this.tableList = Object.values(this.tables)
     }
     return { tables: this.tables, datasource: this.datasource }
   }
@@ -463,7 +461,7 @@ export class ExternalRequest<T extends Operation> {
         breakExternalTableId(relatedTableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
-      if (!lookupField || !row[lookupField]) {
+      if (!lookupField || !row?.[lookupField] == null) {
         continue
       }
       const endpoint = getEndpoint(relatedTableId, Operation.READ)
@@ -631,7 +629,8 @@ export class ExternalRequest<T extends Operation> {
       const { datasource: ds } = await this.retrieveMetadata(datasourceId)
       datasource = ds
     }
-    const table = this.tables[tableName]
+    const tables = this.tables
+    const table = tables[tableName]
     let isSql = isSQL(datasource)
     if (!table) {
       throw new Error(
@@ -686,7 +685,7 @@ export class ExternalRequest<T extends Operation> {
     ) {
       throw "Deletion must be filtered"
     }
-    let json = {
+    let json: QueryJson = {
       endpoint: {
         datasourceId: datasourceId!,
         entityId: tableName,
@@ -715,7 +714,7 @@ export class ExternalRequest<T extends Operation> {
       },
       meta: {
         table,
-        id: config.id,
+        tables: tables,
       },
     }
 
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index c9e7f4c3c5..0b0802bab2 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -3080,4 +3080,46 @@ describe.each([
         }).toHaveLength(4)
       })
     })
+
+  isSql &&
+    describe("max related columns", () => {
+      let relatedRows: Row[]
+
+      beforeAll(async () => {
+        const relatedSchema: TableSchema = {}
+        const row: Row = {}
+        for (let i = 0; i < 100; i++) {
+          const name = `column${i}`
+          relatedSchema[name] = { name, type: FieldType.NUMBER }
+          row[name] = i
+        }
+        const relatedTable = await createTable(relatedSchema)
+        table = await createTable({
+          name: { name: "name", type: FieldType.STRING },
+          related1: {
+            type: FieldType.LINK,
+            name: "related1",
+            fieldName: "main1",
+            tableId: relatedTable._id!,
+            relationshipType: RelationshipType.MANY_TO_MANY,
+          },
+        })
+        relatedRows = await Promise.all([
+          config.api.row.save(relatedTable._id!, row),
+        ])
+        await config.api.row.save(table._id!, {
+          name: "foo",
+          related1: [relatedRows[0]._id],
+        })
+      })
+
+      it("retrieve the row with relationships", async () => {
+        await expectQuery({}).toContainExactly([
+          {
+            name: "foo",
+            related1: [{ _id: relatedRows[0]._id }],
+          },
+        ])
+      })
+    })
 })

From 63c0d9afb8cf6eb21a5b84734aa2c5b97534b883 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 16 Sep 2024 18:27:53 +0100
Subject: [PATCH 060/101] Sorting the field list to make sure we have the
 important fields at the top (if known).

---
 packages/backend-core/src/sql/sql.ts | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index ff32026814..55f71d76b0 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -55,6 +55,20 @@ function getRelationshipLimit() {
   return envLimit || 500
 }
 
+function prioritisedArraySort(toSort: string[], priorities: string[]) {
+  return toSort.sort((a, b) => {
+    const aPriority = priorities.find(field => field && a.endsWith(field))
+    const bPriority = priorities.find(field => field && b.endsWith(field))
+    if (aPriority && !bPriority) {
+      return -1
+    }
+    if (!aPriority && bPriority) {
+      return 1
+    }
+    return a.localeCompare(b)
+  })
+}
+
 function getTableName(table?: Table): string | undefined {
   // SQS uses the table ID rather than the table name
   if (
@@ -924,11 +938,12 @@ class InternalBuilder {
       const requiredFields = [
         ...(relatedTable?.primary || []),
         relatedTable?.primaryDisplay,
-      ]
-      let relationshipFields = fields
-        .filter(field => field.split(".")[0] === toAlias)
-        // sort the required fields to first in the list
-        .sort(field => (requiredFields.includes(field) ? 1 : -1))
+      ].filter(field => field) as string[]
+      // sort the required fields to first in the list, so they don't get sliced out
+      let relationshipFields = prioritisedArraySort(
+        fields.filter(field => field.split(".")[0] === toAlias),
+        requiredFields
+      )
 
       relationshipFields = relationshipFields.slice(
         0,

From ec400dee6f73b510d5b96378abb66f7877cda5cb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 16 Sep 2024 19:20:58 +0100
Subject: [PATCH 061/101] Fixing test cases.

---
 packages/server/src/integrations/tests/sql.spec.ts      | 4 ++--
 packages/server/src/integrations/tests/sqlAlias.spec.ts | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index cf5b88b0fd..5a505fbb40 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -162,7 +162,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
       bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
+      sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
     })
   })
 
@@ -170,7 +170,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateRelationshipJson())
     expect(query).toEqual({
       bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
+      sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
     })
   })
 
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 9548499c65..fc5af4238c 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -63,7 +63,7 @@ describe("Captures of real examples", () => {
         bindings: [primaryLimit, relationshipLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
+            `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")`
           )
         ),
       })
@@ -95,7 +95,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
-                 select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) 
+                 select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) 
                  from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
                  from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
@@ -113,7 +113,7 @@ describe("Captures of real examples", () => {
         bindings: [...filters, relationshipLimit, relationshipLimit],
         sql: multiline(
           `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
-               select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) 
+               select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
                from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
                where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
         ),

From 614132eb0462a071cbbcc6f0c867e589e1502108 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 16 Sep 2024 20:40:02 +0000
Subject: [PATCH 062/101] Bump version to 2.32.5

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a9c50ea4b5..c710d888c7 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.4",
+  "version": "2.32.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 5e245d11e5963774b8c16f478624c41e031abcb7 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Tue, 17 Sep 2024 09:20:23 +0100
Subject: [PATCH 063/101] Bump account portal to latest

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 25c6ed45c8..905773d708 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 25c6ed45c826f27b1ee7a7d1460dc1a386c6c471
+Subproject commit 905773d70854a43c6ef2461c7a49671bff56fedc

From 224be10b6c21e7d69c9c1d3dc63cabca75236f0e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 17 Sep 2024 16:08:59 +0100
Subject: [PATCH 064/101] support for overriding models

---
 packages/backend-core/src/configs/configs.ts  |   6 +-
 packages/pro                                  |   2 +-
 packages/server/package.json                  |   2 +-
 .../server/src/automations/steps/openai.ts    |   2 +-
 .../src/api/controllers/global/configs.ts     |  34 +--
 yarn.lock                                     | 223 ++++++++++++++----
 6 files changed, 189 insertions(+), 80 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 00ccee8c4e..379c1de7a4 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -258,8 +258,8 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
 }
 
 // AI
-export async function getAIConfig(): Promise<AIInnerConfig | undefined> {
-  const config = await getConfig<AIConfig>(ConfigType.AI)
-  return config?.config
+
+export async function getAIConfig(): Promise<AIConfig | undefined> {
+  return getConfig<AIConfig>(ConfigType.AI)
 }
 
diff --git a/packages/pro b/packages/pro
index 922431260e..ff141defc6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 922431260e90d558a1ca55398475412e75088057
+Subproject commit ff141defc6eb744f7edac788eaaaa046423cfa7f
diff --git a/packages/server/package.json b/packages/server/package.json
index 6dfd528963..41ce10c135 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -101,7 +101,7 @@
     "mysql2": "3.9.8",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
-    "openai": "^4.52.1",
+    "openai": "4.59.0",
     "openapi-types": "9.3.1",
     "oracledb": "6.5.1",
     "pg": "8.10.0",
diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 1dffc37d1f..13b0008654 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -115,7 +115,7 @@ export async function run({
       // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config
       // Make sure it uses their credits
       // Should be handled in the LLM wrapper in pro
-      const llm = new pro.ai.LLMWrapper()
+      const llm = new pro.ai.LargeLanguageModel(inputs.model)
       await llm.init()
       response = await llm.run(inputs.prompt)
     } else {
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 70b2279f6c..d7a18713a2 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -334,32 +334,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
   )
 }
 
-async function enrichAIConfig(aiConfig: AIConfig) {
-  // Strip out the API Keys from the response so they don't show in the UI
-  for (const key in aiConfig.config) {
-    if (aiConfig.config[key].apiKey) {
-      aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT
-    }
-  }
-
-  // Return the Budibase AI data source as part of the response if licensing allows
-  const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
-  const defaultConfigExists = Object.keys(aiConfig.config).some(
-    key => aiConfig.config[key].isDefault
-  )
-  if (budibaseAIEnabled) {
-    aiConfig.config["budibase_ai"] = {
-      provider: "OpenAI",
-      active: true,
-      isDefault: !defaultConfigExists,
-      defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "",
-      name: "Budibase AI",
-    }
-  }
-
-  return aiConfig
-}
-
 export async function find(ctx: UserCtx) {
   try {
     // Find the config with the most granular scope based on context
@@ -372,7 +346,13 @@ export async function find(ctx: UserCtx) {
       }
 
       if (type === ConfigType.AI) {
-        await enrichAIConfig(scopedConfig)
+        await pro.ai.getAIConfig(scopedConfig)
+        // Strip out the API Keys from the response so they don't show in the UI
+        for (const key in scopedConfig.config) {
+          if (scopedConfig.config[key].apiKey) {
+            scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT
+          }
+        }
       }
       ctx.body = scopedConfig
     } else {
diff --git a/yarn.lock b/yarn.lock
index 110cbd7a15..69c3978aaf 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -33,6 +33,19 @@
     "@jridgewell/gen-mapping" "^0.3.5"
     "@jridgewell/trace-mapping" "^0.3.24"
 
+"@anthropic-ai/sdk@^0.27.3":
+  version "0.27.3"
+  resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475"
+  integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==
+  dependencies:
+    "@types/node" "^18.11.18"
+    "@types/node-fetch" "^2.6.4"
+    abort-controller "^3.0.0"
+    agentkeepalive "^4.2.1"
+    form-data-encoder "1.7.2"
+    formdata-node "^4.3.2"
+    node-fetch "^2.6.7"
+
 "@apidevtools/json-schema-ref-parser@^9.0.6":
   version "9.1.2"
   resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8"
@@ -759,20 +772,20 @@
     "@azure/abort-controller" "^1.0.0"
     tslib "^2.2.0"
 
-"@azure/identity@4.2.1", "@azure/identity@^3.4.1":
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c"
-  integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q==
+"@azure/identity@^3.4.1":
+  version "3.4.2"
+  resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de"
+  integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA==
   dependencies:
     "@azure/abort-controller" "^1.0.0"
     "@azure/core-auth" "^1.5.0"
     "@azure/core-client" "^1.4.0"
     "@azure/core-rest-pipeline" "^1.1.0"
     "@azure/core-tracing" "^1.0.0"
-    "@azure/core-util" "^1.3.0"
+    "@azure/core-util" "^1.6.1"
     "@azure/logger" "^1.0.0"
-    "@azure/msal-browser" "^3.11.1"
-    "@azure/msal-node" "^2.9.2"
+    "@azure/msal-browser" "^3.5.0"
+    "@azure/msal-node" "^2.5.1"
     events "^3.0.0"
     jws "^4.0.0"
     open "^8.0.0"
@@ -803,7 +816,7 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/msal-browser@^3.11.1":
+"@azure/msal-browser@^3.5.0":
   version "3.23.0"
   resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b"
   integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw==
@@ -815,7 +828,7 @@
   resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4"
   integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog==
 
-"@azure/msal-node@^2.9.2":
+"@azure/msal-node@^2.5.1":
   version "2.13.1"
   resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47"
   integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw==
@@ -6040,6 +6053,11 @@
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
   integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
 
+"@types/qs@^6.9.15":
+  version "6.9.16"
+  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794"
+  integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==
+
 "@types/range-parser@*":
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
@@ -7448,7 +7466,30 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
+axios@0.24.0:
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
+  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
+  dependencies:
+    follow-redirects "^1.14.4"
+
+axios@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
+  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
+axios@^0.21.1:
+  version "0.21.4"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
+  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
+  dependencies:
+    follow-redirects "^1.14.0"
+
+axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11290,7 +11331,14 @@ fast-url-parser@^1.1.3:
   dependencies:
     punycode "^1.3.2"
 
-fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
+fast-xml-parser@4.2.5:
+  version "4.2.5"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f"
+  integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==
+  dependencies:
+    strnum "^1.0.5"
+
+fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
   version "4.4.1"
   resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f"
   integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==
@@ -11544,6 +11592,11 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
+follow-redirects@^1.14.0, follow-redirects@^1.14.4:
+  version "1.15.9"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
+  integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==
+
 follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
@@ -12637,7 +12690,12 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1:
+  version "3.8.1"
+  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
+  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
+
+http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13162,7 +13220,7 @@ is-boolean-object@^1.1.0:
     call-bind "^1.0.2"
     has-tostringtag "^1.0.0"
 
-is-buffer@~1.1.6:
+is-buffer@^1.1.5, is-buffer@~1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
   integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
@@ -13648,11 +13706,6 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
-isobject@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
-  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
-
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -14562,7 +14615,14 @@ kill-port@^1.6.1:
     get-them-args "1.3.2"
     shell-exec "1.0.2"
 
-kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
+kind-of@^3.0.2, kind-of@^3.1.0:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
+  integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
+  dependencies:
+    is-buffer "^1.1.5"
+
+kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
   version "6.0.3"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd"
   integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==
@@ -16205,7 +16265,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@1.10.1, msgpackr@^1.5.2:
+msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16399,13 +16459,25 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
+node-fetch@2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
+node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.6.9, node-fetch@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -17020,19 +17092,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
-openai@^4.52.1:
-  version "4.52.1"
-  resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65"
-  integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg==
+openai@4.59.0:
+  version "4.59.0"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08"
+  integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==
   dependencies:
     "@types/node" "^18.11.18"
     "@types/node-fetch" "^2.6.4"
+    "@types/qs" "^6.9.15"
     abort-controller "^3.0.0"
     agentkeepalive "^4.2.1"
     form-data-encoder "1.7.2"
     formdata-node "^4.3.2"
     node-fetch "^2.6.7"
-    web-streams-polyfill "^3.2.1"
+    qs "^6.10.3"
 
 openapi-response-validator@^9.2.0:
   version "9.3.1"
@@ -17556,7 +17629,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
+passport@^0.4.0:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
+  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
+  dependencies:
+    passport-strategy "1.x.x"
+    pause "0.0.1"
+
+passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -18862,7 +18943,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.33:
+psl@^1.1.28, psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -19934,6 +20015,11 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
+sax@>=0.1.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
+  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
+
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -20006,13 +20092,33 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1:
+"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
+  version "5.7.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
+  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
+
+semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
+semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
+  version "6.3.1"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
+  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
+
+semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
+semver@~2.3.1:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
+  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
+
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -21575,7 +21681,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
+"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21585,6 +21691,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
+tough-cookie@~2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
+  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
+  dependencies:
+    psl "^1.1.28"
+    punycode "^2.1.1"
+
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -22113,14 +22227,6 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
-unset-value@2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
-  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
-  dependencies:
-    has-value "^2.0.2"
-    isobject "^4.0.0"
-
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -22522,11 +22628,6 @@ web-streams-polyfill@4.0.0-beta.3:
   resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
   integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
 
-web-streams-polyfill@^3.2.1:
-  version "3.3.3"
-  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b"
-  integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==
-
 web-vitals@^4.0.1:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7"
@@ -22900,10 +23001,33 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
-  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
+xml2js@0.1.x:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
+  dependencies:
+    sax ">=0.1.1"
+
+xml2js@0.4.19:
+  version "0.4.19"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
+  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~9.0.1"
+
+xml2js@0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
+  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~11.0.0"
+
+xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.4.23"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
+  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -22913,6 +23037,11 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
+xmlbuilder@~9.0.1:
+  version "9.0.7"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
+  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
+
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"

From e40f397c25a96e9cdab5f8ad134637a850c72a0e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 17 Sep 2024 16:29:44 +0100
Subject: [PATCH 065/101] move check for OpenAI config variable to legacy path

---
 packages/pro                                  |  2 +-
 .../server/src/automations/steps/openai.ts    | 30 ++++++++-----------
 .../src/api/controllers/global/configs.ts     |  2 +-
 3 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/packages/pro b/packages/pro
index ff141defc6..72e9846146 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ff141defc6eb744f7edac788eaaaa046423cfa7f
+Subproject commit 72e9846146b46efc7cbdc69daea17761cb45f1d3
diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 13b0008654..79b6664674 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -63,6 +63,11 @@ export const definition: AutomationStepDefinition = {
   },
 }
 
+/**
+ * Maintains backward compatibility with automation steps created before the introduction
+ * of custom configurations and Budibase AI
+ * @param inputs - automation inputs from the OpenAI automation step.
+ */
 async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) {
   const openai = new OpenAI({
     apiKey: env.OPENAI_API_KEY,
@@ -85,14 +90,6 @@ export async function run({
 }: {
   inputs: OpenAIStepInputs
 }): Promise<OpenAIStepOutputs> {
-  if (!env.OPENAI_API_KEY) {
-    return {
-      success: false,
-      response:
-        "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
-    }
-  }
-
   if (inputs.prompt == null) {
     return {
       success: false,
@@ -106,20 +103,19 @@ export async function run({
     const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
 
     if (budibaseAIEnabled || customConfigsEnabled) {
-      // Enterprise has custom configs
-      // if custom configs are enabled full stop
-      // Don't use their budibase AI credits, unless it uses the budibase AI configuration
-      // TODO: grab the config from the database (maybe wrap this in the pro AI module)
-      // TODO: pass it into the model to execute the prompt
-
-      // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config
-      // Make sure it uses their credits
-      // Should be handled in the LLM wrapper in pro
       const llm = new pro.ai.LargeLanguageModel(inputs.model)
       await llm.init()
       response = await llm.run(inputs.prompt)
     } else {
       // fallback to the default that uses the environment variable for backwards compat
+      if (!env.OPENAI_API_KEY) {
+        return {
+          success: false,
+          response:
+            "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
+        }
+      }
+
       response = await legacyOpenAIPrompt(inputs)
     }
 
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index d7a18713a2..f53868d7df 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -346,7 +346,7 @@ export async function find(ctx: UserCtx) {
       }
 
       if (type === ConfigType.AI) {
-        await pro.ai.getAIConfig(scopedConfig)
+        await pro.ai.enrichAIConfig(scopedConfig)
         // Strip out the API Keys from the response so they don't show in the UI
         for (const key in scopedConfig.config) {
           if (scopedConfig.config[key].apiKey) {

From fd3c8c4a119fbf28140c25c23cc804e669a599c7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 17 Sep 2024 16:29:09 +0100
Subject: [PATCH 066/101] Make view readonly columns available for free

---
 .../tests/core/utilities/mocks/licenses.ts        |  4 ----
 .../backend/DataTable/ViewV2DataTable.svelte      |  1 -
 packages/builder/src/stores/portal/licensing.js   |  5 -----
 .../grid/controls/ColumnsSettingButton.svelte     | 14 ++------------
 .../src/components/grid/layout/Grid.svelte        |  4 +---
 .../server/src/api/routes/tests/viewV2.spec.ts    | 15 ---------------
 packages/server/src/sdk/app/views/index.ts        |  7 -------
 7 files changed, 3 insertions(+), 47 deletions(-)

diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
index 2d8e81d125..bc9a3b635c 100644
--- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
@@ -102,10 +102,6 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
-export const useViewReadonlyColumns = () => {
-  return useFeature(Feature.VIEW_READONLY_COLUMNS)
-}
-
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {
diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index b56c5f6568..ed35c5d6a3 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -30,7 +30,6 @@
     showAvatars={false}
     on:updatedatasource={handleGridViewUpdate}
     isCloud={$admin.cloud}
-    allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
     canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
   >
     <svelte:fragment slot="filter">
diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js
index d48377207c..f0ce7046f2 100644
--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@@ -140,10 +140,6 @@ export const createLicensingStore = () => {
         Constants.Features.VIEW_PERMISSIONS
       )
 
-      const isViewReadonlyColumnsEnabled = license.features.includes(
-        Constants.Features.VIEW_READONLY_COLUMNS
-      )
-
       const budibaseAIEnabled = license.features.includes(
         Constants.Features.BUDIBASE_AI
       )
@@ -173,7 +169,6 @@ export const createLicensingStore = () => {
           triggerAutomationRunEnabled,
           isViewPermissionsEnabled,
           perAppBuildersEnabled,
-          isViewReadonlyColumnsEnabled,
         }
       })
     },
diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
index b4940c8903..dafc25a00b 100644
--- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
@@ -4,16 +4,13 @@
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
   import { FieldPermissions } from "../../../constants"
 
-  export let allowViewReadonlyColumns = false
-
   const { columns, datasource } = getContext("grid")
 
   let open = false
   let anchor
 
   $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
-  $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns"
-
+  $: text = anyRestricted ? `Columns: ${anyRestricted} restricted` : "Columns"
   $: permissions =
     $datasource.type === "viewV2"
       ? [
@@ -22,9 +19,6 @@
           FieldPermissions.HIDDEN,
         ]
       : [FieldPermissions.WRITABLE, FieldPermissions.HIDDEN]
-  $: disabledPermissions = allowViewReadonlyColumns
-    ? []
-    : [FieldPermissions.READONLY]
 </script>
 
 <div bind:this={anchor}>
@@ -41,9 +35,5 @@
 </div>
 
 <Popover bind:open {anchor} align="left">
-  <ColumnsSettingContent
-    columns={$columns}
-    {permissions}
-    {disabledPermissions}
-  />
+  <ColumnsSettingContent columns={$columns} {permissions} />
 </Popover>
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index f24ff0ae10..f2aeffb9f4 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -58,7 +58,6 @@
   export let buttons = null
   export let darkMode
   export let isCloud = null
-  export let allowViewReadonlyColumns = false
   export let rowConditions = null
 
   // Unique identifier for DOM nodes inside this instance
@@ -115,7 +114,6 @@
     buttons,
     darkMode,
     isCloud,
-    allowViewReadonlyColumns,
     rowConditions,
   })
 
@@ -157,7 +155,7 @@
       <div class="controls-left">
         <slot name="filter" />
         <SortButton />
-        <ColumnsSettingButton {allowViewReadonlyColumns} />
+        <ColumnsSettingButton />
         <SizeButton />
         <slot name="controls" />
       </div>
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index f86291e9cd..824b69a6bb 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -309,10 +309,6 @@ describe.each([
       })
 
       describe("readonly fields", () => {
-        beforeEach(() => {
-          mocks.licenses.useViewReadonlyColumns()
-        })
-
         it("readonly fields are persisted", async () => {
           const table = await config.api.table.save(
             saveTableRequest({
@@ -513,7 +509,6 @@ describe.each([
       })
 
       it("display fields can be readonly", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const table = await config.api.table.save(
           saveTableRequest({
             schema: {
@@ -588,7 +583,6 @@ describe.each([
       })
 
       it("can update all fields", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const tableId = table._id!
 
         const updatedData: Required<UpdateViewRequest> = {
@@ -803,8 +797,6 @@ describe.each([
       })
 
       it("cannot update views with readonly on on free license", async () => {
-        mocks.licenses.useViewReadonlyColumns()
-
         view = await config.api.viewV2.update({
           ...view,
           schema: {
@@ -826,8 +818,6 @@ describe.each([
       })
 
       it("can remove readonly config after license downgrade", async () => {
-        mocks.licenses.useViewReadonlyColumns()
-
         view = await config.api.viewV2.update({
           ...view,
           schema: {
@@ -1046,7 +1036,6 @@ describe.each([
       })
 
       it("should be able to fetch readonly config after downgrades", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const res = await config.api.viewV2.create({
           name: generator.name(),
           tableId: table._id!,
@@ -1112,8 +1101,6 @@ describe.each([
         })
 
         it("rejects if field is readonly in any view", async () => {
-          mocks.licenses.useViewReadonlyColumns()
-
           await config.api.viewV2.create({
             name: "view a",
             tableId: table._id!,
@@ -1538,7 +1525,6 @@ describe.each([
       })
 
       it("can't persist readonly columns", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
@@ -1607,7 +1593,6 @@ describe.each([
       })
 
       it("can't update readonly columns", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index d7e05abf2f..3ba91e152f 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -59,13 +59,6 @@ async function guardViewSchema(
     }
 
     if (viewSchema[field].readonly) {
-      if (
-        !(await features.isViewReadonlyColumnsEnabled()) &&
-        !(tableSchemaField as ViewFieldMetadata).readonly
-      ) {
-        throw new HTTPError(`Readonly fields are not enabled`, 400)
-      }
-
       if (!viewSchema[field].visible) {
         throw new HTTPError(
           `Field "${field}" must be visible if you want to make it readonly`,

From 4bd4cb57f80ffad3c05f93ad850501b6b68953e9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 17 Sep 2024 16:34:47 +0100
Subject: [PATCH 067/101] Undo accidental UI change

---
 .../src/components/grid/controls/ColumnsSettingButton.svelte    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
index dafc25a00b..2a3ca139fc 100644
--- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
@@ -10,7 +10,7 @@
   let anchor
 
   $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
-  $: text = anyRestricted ? `Columns: ${anyRestricted} restricted` : "Columns"
+  $: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns"
   $: permissions =
     $datasource.type === "viewV2"
       ? [

From 8e2d21c85fb85d38a217422d914d3bf86efd6589 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 17 Sep 2024 18:27:49 +0100
Subject: [PATCH 068/101] call right sdk method on ai module for enriching AI
 config

---
 packages/pro                                          | 2 +-
 packages/worker/src/api/controllers/global/configs.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 72e9846146..69633aa64b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 72e9846146b46efc7cbdc69daea17761cb45f1d3
+Subproject commit 69633aa64b962ac4bad01360a421fe941a15fb2c
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index f53868d7df..f01c7c8869 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -346,7 +346,7 @@ export async function find(ctx: UserCtx) {
       }
 
       if (type === ConfigType.AI) {
-        await pro.ai.enrichAIConfig(scopedConfig)
+        await pro.sdk.ai.enrichAIConfig(scopedConfig)
         // Strip out the API Keys from the response so they don't show in the UI
         for (const key in scopedConfig.config) {
           if (scopedConfig.config[key].apiKey) {

From d7334f100b263e358492e7f79456dd2f0ca72462 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 18 Sep 2024 01:21:13 +0000
Subject: [PATCH 069/101] Bump next from 14.1.1 to 14.2.10 in
 /examples/nextjs-api-sales

Bumps [next](https://github.com/vercel/next.js) from 14.1.1 to 14.2.10.
- [Release notes](https://github.com/vercel/next.js/releases)
- [Changelog](https://github.com/vercel/next.js/blob/canary/release.js)
- [Commits](https://github.com/vercel/next.js/compare/v14.1.1...v14.2.10)

---
updated-dependencies:
- dependency-name: next
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 examples/nextjs-api-sales/package.json |   2 +-
 examples/nextjs-api-sales/yarn.lock    | 124 +++++++++++++------------
 2 files changed, 66 insertions(+), 60 deletions(-)

diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json
index 7ecf264add..f1ef4843a1 100644
--- a/examples/nextjs-api-sales/package.json
+++ b/examples/nextjs-api-sales/package.json
@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "bulma": "^0.9.3",
-    "next": "14.1.1",
+    "next": "14.2.10",
     "node-fetch": "^3.2.10",
     "sass": "^1.52.3",
     "react": "17.0.2",
diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock
index a44956ba21..9acbdfdeb6 100644
--- a/examples/nextjs-api-sales/yarn.lock
+++ b/examples/nextjs-api-sales/yarn.lock
@@ -46,10 +46,10 @@
   resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
   integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
 
-"@next/env@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac"
-  integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA==
+"@next/env@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c"
+  integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw==
 
 "@next/eslint-plugin-next@12.1.0":
   version "12.1.0"
@@ -58,50 +58,50 @@
   dependencies:
     glob "7.1.7"
 
-"@next/swc-darwin-arm64@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64"
-  integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ==
+"@next/swc-darwin-arm64@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80"
+  integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ==
 
-"@next/swc-darwin-x64@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b"
-  integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw==
+"@next/swc-darwin-x64@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8"
+  integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA==
 
-"@next/swc-linux-arm64-gnu@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa"
-  integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg==
+"@next/swc-linux-arm64-gnu@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20"
+  integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA==
 
-"@next/swc-linux-arm64-musl@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a"
-  integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ==
+"@next/swc-linux-arm64-musl@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47"
+  integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ==
 
-"@next/swc-linux-x64-gnu@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528"
-  integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ==
+"@next/swc-linux-x64-gnu@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082"
+  integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg==
 
-"@next/swc-linux-x64-musl@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25"
-  integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og==
+"@next/swc-linux-x64-musl@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3"
+  integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA==
 
-"@next/swc-win32-arm64-msvc@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9"
-  integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A==
+"@next/swc-win32-arm64-msvc@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209"
+  integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ==
 
-"@next/swc-win32-ia32-msvc@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2"
-  integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw==
+"@next/swc-win32-ia32-msvc@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041"
+  integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg==
 
-"@next/swc-win32-x64-msvc@14.1.1":
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52"
-  integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A==
+"@next/swc-win32-x64-msvc@14.2.10":
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455"
+  integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ==
 
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
@@ -129,11 +129,17 @@
   resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323"
   integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A==
 
-"@swc/helpers@0.5.2":
-  version "0.5.2"
-  resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d"
-  integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==
+"@swc/counter@^0.1.3":
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9"
+  integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==
+
+"@swc/helpers@0.5.5":
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0"
+  integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==
   dependencies:
+    "@swc/counter" "^0.1.3"
     tslib "^2.4.0"
 
 "@types/json5@^0.0.29":
@@ -1245,28 +1251,28 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
-next@14.1.1:
-  version "14.1.1"
-  resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171"
-  integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww==
+next@14.2.10:
+  version "14.2.10"
+  resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6"
+  integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww==
   dependencies:
-    "@next/env" "14.1.1"
-    "@swc/helpers" "0.5.2"
+    "@next/env" "14.2.10"
+    "@swc/helpers" "0.5.5"
     busboy "1.6.0"
     caniuse-lite "^1.0.30001579"
     graceful-fs "^4.2.11"
     postcss "8.4.31"
     styled-jsx "5.1.1"
   optionalDependencies:
-    "@next/swc-darwin-arm64" "14.1.1"
-    "@next/swc-darwin-x64" "14.1.1"
-    "@next/swc-linux-arm64-gnu" "14.1.1"
-    "@next/swc-linux-arm64-musl" "14.1.1"
-    "@next/swc-linux-x64-gnu" "14.1.1"
-    "@next/swc-linux-x64-musl" "14.1.1"
-    "@next/swc-win32-arm64-msvc" "14.1.1"
-    "@next/swc-win32-ia32-msvc" "14.1.1"
-    "@next/swc-win32-x64-msvc" "14.1.1"
+    "@next/swc-darwin-arm64" "14.2.10"
+    "@next/swc-darwin-x64" "14.2.10"
+    "@next/swc-linux-arm64-gnu" "14.2.10"
+    "@next/swc-linux-arm64-musl" "14.2.10"
+    "@next/swc-linux-x64-gnu" "14.2.10"
+    "@next/swc-linux-x64-musl" "14.2.10"
+    "@next/swc-win32-arm64-msvc" "14.2.10"
+    "@next/swc-win32-ia32-msvc" "14.2.10"
+    "@next/swc-win32-x64-msvc" "14.2.10"
 
 node-domexception@^1.0.0:
   version "1.0.0"

From 2cf07a40f6467aaac25416509c1e6f528bb42c95 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 18 Sep 2024 09:44:33 +0100
Subject: [PATCH 070/101] fix for setting default configs

---
 .../src/pages/builder/portal/settings/ai/index.svelte        | 5 ++++-
 packages/pro                                                 | 2 +-
 packages/server/src/automations/steps/openai.ts              | 1 -
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 2ac1609e7c..b60ea24dbc 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -56,10 +56,13 @@
     } else {
       // We don't store the default BB AI config in the DB
       delete fullAIConfig.config.budibase_ai
+
       // unset the default value from other configs if default is set
       if (editingAIConfig.isDefault) {
         for (let key in fullAIConfig.config) {
-          fullAIConfig.config[key].isDefault = false
+          if (key !== id) {
+            fullAIConfig.config[key].isDefault = false
+          }
         }
       }
       // Add new or update existing custom AI Config
diff --git a/packages/pro b/packages/pro
index 69633aa64b..5daf17e325 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 69633aa64b962ac4bad01360a421fe941a15fb2c
+Subproject commit 5daf17e32595e539f1f4a92b59a2ea2854d9dbd4
diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 79b6664674..5eff12db47 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -115,7 +115,6 @@ export async function run({
             "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
         }
       }
-
       response = await legacyOpenAIPrompt(inputs)
     }
 

From cb8d0984b1fa484e4482dd7a2009842c71d293d0 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 18 Sep 2024 10:52:52 +0100
Subject: [PATCH 071/101] update Providers

---
 packages/types/src/documents/global/config.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 8d64b49ee9..33f7e10584 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -111,7 +111,7 @@ export interface SCIMInnerConfig {
 
 export interface SCIMConfig extends Config<SCIMInnerConfig> {}
 
-type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom"
+export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom"
 
 export interface AIInnerConfig {
   [key: string]: {

From 4325805ce361d3e04792785753eed6bde24061ac Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 11:54:24 +0100
Subject: [PATCH 072/101] Lint

---
 .../src/components/backend/DataTable/ViewV2DataTable.svelte     | 2 +-
 packages/server/src/sdk/app/views/index.ts                      | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index ed35c5d6a3..90e5e216f3 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2 } from "stores/builder"
-  import { admin, licensing } from "stores/portal"
+  import { admin } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index 3ba91e152f..269158e61e 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -5,13 +5,11 @@ import {
   Table,
   TableSchema,
   View,
-  ViewFieldMetadata,
   ViewV2,
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
 import { HTTPError } from "@budibase/backend-core"
-import { features } from "@budibase/pro"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,

From c8c4248b3cab659802c5f77251633573bc340061 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 11:55:40 +0100
Subject: [PATCH 073/101] Update tests

---
 .../src/api/routes/tests/viewV2.spec.ts       | 47 +------------------
 1 file changed, 2 insertions(+), 45 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 824b69a6bb..201c7db3d7 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -796,7 +796,7 @@ describe.each([
         )
       })
 
-      it("cannot update views with readonly on on free license", async () => {
+      it("can views with readonly on on free license", async () => {
         view = await config.api.viewV2.update({
           ...view,
           schema: {
@@ -810,53 +810,10 @@ describe.each([
 
         mocks.licenses.useCloudFree()
         await config.api.viewV2.update(view, {
-          status: 400,
-          body: {
-            message: "Readonly fields are not enabled",
-          },
+          status: 200,
         })
       })
 
-      it("can remove readonly config after license downgrade", async () => {
-        view = await config.api.viewV2.update({
-          ...view,
-          schema: {
-            id: { visible: true },
-            Price: {
-              visible: true,
-              readonly: true,
-            },
-            Category: {
-              visible: true,
-              readonly: true,
-            },
-          },
-        })
-        mocks.licenses.useCloudFree()
-        const res = await config.api.viewV2.update({
-          ...view,
-          schema: {
-            id: { visible: true },
-            Price: {
-              visible: true,
-              readonly: false,
-            },
-          },
-        })
-        expect(res).toEqual(
-          expect.objectContaining({
-            ...view,
-            schema: {
-              id: { visible: true },
-              Price: {
-                visible: true,
-                readonly: false,
-              },
-            },
-          })
-        )
-      })
-
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({

From fd9a2f87d12dcc6b5cafa8f640bfaa508bbb7b2d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 12:35:05 +0100
Subject: [PATCH 074/101] Update more tests

---
 packages/server/src/api/routes/tests/viewV2.spec.ts | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 201c7db3d7..65448a5bf3 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -432,7 +432,7 @@ describe.each([
           })
         })
 
-        it("readonly fields cannot be used on free license", async () => {
+        it("readonly fields can be used on free license", async () => {
           mocks.licenses.useCloudFree()
           const table = await config.api.table.save(
             saveTableRequest({
@@ -462,11 +462,7 @@ describe.each([
           }
 
           await config.api.viewV2.create(newView, {
-            status: 400,
-            body: {
-              message: "Readonly fields are not enabled",
-              status: 400,
-            },
+            status: 200,
           })
         })
       })

From a0350a75efe5fd8132975a56d84d18554fd16a9f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 13:17:30 +0100
Subject: [PATCH 075/101] Update tests again

---
 .../src/api/routes/tests/viewV2.spec.ts       | 20 +------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 65448a5bf3..c4a39ae8a9 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -462,7 +462,7 @@ describe.each([
           }
 
           await config.api.viewV2.create(newView, {
-            status: 200,
+            status: 201,
           })
         })
       })
@@ -792,24 +792,6 @@ describe.each([
         )
       })
 
-      it("can views with readonly on on free license", async () => {
-        view = await config.api.viewV2.update({
-          ...view,
-          schema: {
-            id: { visible: true },
-            Price: {
-              visible: true,
-              readonly: true,
-            },
-          },
-        })
-
-        mocks.licenses.useCloudFree()
-        await config.api.viewV2.update(view, {
-          status: 200,
-        })
-      })
-
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({

From 9cbaa02f5f305542453493d0e9520cf0ca7681bc Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 18 Sep 2024 14:01:54 +0100
Subject: [PATCH 076/101] add test for automation run step when LLM feature
 flags are on

---
 packages/pro                                  |  2 +-
 .../src/automations/tests/openai.spec.ts      | 35 ++++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/packages/pro b/packages/pro
index 5daf17e325..320f8ecf8a 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 5daf17e32595e539f1f4a92b59a2ea2854d9dbd4
+Subproject commit 320f8ecf8ae769995590ddc4e3679ef7c110bc11
diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts
index 3a5a57475a..0a64411a12 100644
--- a/packages/server/src/automations/tests/openai.spec.ts
+++ b/packages/server/src/automations/tests/openai.spec.ts
@@ -4,6 +4,7 @@ import {
   withEnv as withCoreEnv,
   setEnv as setCoreEnv,
 } from "@budibase/backend-core"
+import * as pro from "@budibase/pro"
 
 jest.mock("openai", () => ({
   OpenAI: jest.fn().mockImplementation(() => ({
@@ -23,6 +24,20 @@ jest.mock("openai", () => ({
   })),
 }))
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: jest.fn().mockImplementation(() => ({
+      init: jest.fn(),
+      run: jest.fn(),
+    })),
+  },
+  features: {
+    isAICustomConfigsEnabled: jest.fn(),
+    isBudibaseAIEnabled: jest.fn(),
+  }
+}))
+
 const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
 
 const OPENAI_PROMPT = "What is the meaning of life?"
@@ -41,6 +56,7 @@ describe("test the openai action", () => {
 
   afterEach(() => {
     resetEnv()
+    jest.clearAllMocks()
   })
 
   afterAll(_afterAll)
@@ -86,7 +102,7 @@ describe("test the openai action", () => {
     )
 
     const res = await runStep("OPENAI", {
-      prompt: OPENAI_PROMPT,
+      prompt: OPENAI_PROMPT,jj
     })
 
     expect(res.response).toEqual(
@@ -94,4 +110,21 @@ describe("test the openai action", () => {
     )
     expect(res.success).toBeFalsy()
   })
+
+  it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => {
+    jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true)
+    jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
+
+    const prompt = "What is the meaning of life?"
+    await runStep("OPENAI", {
+      model: "gpt-4o-mini",
+      prompt
+    })
+
+    expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini")
+
+    const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value
+    expect(llmInstance.init).toHaveBeenCalled()
+    expect(llmInstance.run).toHaveBeenCalledWith(prompt)
+  })
 })

From 69b2bf9bdb80aad9350ecb6ebf4b4f82555a7d5f Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 18 Sep 2024 14:21:10 +0100
Subject: [PATCH 077/101] lint

---
 packages/backend-core/src/configs/configs.ts         | 1 -
 packages/server/src/automations/tests/openai.spec.ts | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 379c1de7a4..6c2f2fe586 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -262,4 +262,3 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
 export async function getAIConfig(): Promise<AIConfig | undefined> {
   return getConfig<AIConfig>(ConfigType.AI)
 }
-
diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts
index 0a64411a12..498d15b93c 100644
--- a/packages/server/src/automations/tests/openai.spec.ts
+++ b/packages/server/src/automations/tests/openai.spec.ts
@@ -35,7 +35,7 @@ jest.mock("@budibase/pro", () => ({
   features: {
     isAICustomConfigsEnabled: jest.fn(),
     isBudibaseAIEnabled: jest.fn(),
-  }
+  },
 }))
 
 const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
@@ -102,7 +102,7 @@ describe("test the openai action", () => {
     )
 
     const res = await runStep("OPENAI", {
-      prompt: OPENAI_PROMPT,jj
+      prompt: OPENAI_PROMPT,
     })
 
     expect(res.response).toEqual(
@@ -118,7 +118,7 @@ describe("test the openai action", () => {
     const prompt = "What is the meaning of life?"
     await runStep("OPENAI", {
       model: "gpt-4o-mini",
-      prompt
+      prompt,
     })
 
     expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini")

From da122d2ac157bc26ba69bfdec32a184469033539 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 14:55:09 +0100
Subject: [PATCH 078/101] Add table ID to row deletion requests that only
 contain string IDs

---
 packages/server/src/api/controllers/row/index.ts |  2 +-
 packages/server/src/api/routes/tests/row.spec.ts | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index cd85f57982..2e5785157d 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
   const { tableId } = utils.getSourceId(ctx)
 
   const processedRows = request.rows.map(row => {
-    let processedRow: Row = typeof row == "string" ? { _id: row } : row
+    let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row
     return !processedRow._rev
       ? addRev(fixRow(processedRow, ctx.params), tableId)
       : fixRow(processedRow, ctx.params)
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9790703806..dc03a21d6d 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1138,6 +1138,18 @@ describe.each([
       await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
     })
 
+    it("should be able to delete a row with ID only", async () => {
+      const createdRow = await config.api.row.save(table._id!, {})
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.bulkDelete(table._id!, {
+        rows: [createdRow._id!],
+      })
+      expect(res[0]._id).toEqual(createdRow._id)
+      expect(res[0].tableId).toEqual(table._id!)
+      await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
+    })
+
     it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
       const createdRow = await config.api.row.save(table._id!, {})
       const createdRow2 = await config.api.row.save(table._id!, {})

From 8a399bfbdfd33a8b117ee08ad0612befeb05f24c Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 18 Sep 2024 17:42:55 +0100
Subject: [PATCH 079/101] set custom configs quotas to right number

---
 packages/pro                                          | 2 +-
 packages/worker/src/api/controllers/global/configs.ts | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 320f8ecf8a..4300b75bde 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 320f8ecf8ae769995590ddc4e3679ef7c110bc11
+Subproject commit 4300b75bde89e37bb0bace927a0e8d204367ae71
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index f01c7c8869..f3262dc0dd 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -253,6 +253,8 @@ export async function save(ctx: UserCtx<Config>) {
         if (existingConfig) {
           await verifyAIConfig(config, existingConfig)
         }
+        const numConfigs = Object.keys(config).length
+        await pro.quotas.updateCustomAIConfigCount(numConfigs)
         break
     }
   } catch (err: any) {

From 950bd45b2c1abd58a0bc46f1248dd7cdac76b2c0 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 18 Sep 2024 18:04:13 +0100
Subject: [PATCH 080/101] add ts ignore for mock

---
 packages/server/src/automations/tests/openai.spec.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts
index 498d15b93c..342288a6a1 100644
--- a/packages/server/src/automations/tests/openai.spec.ts
+++ b/packages/server/src/automations/tests/openai.spec.ts
@@ -123,6 +123,7 @@ describe("test the openai action", () => {
 
     expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini")
 
+    // @ts-ignore
     const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value
     expect(llmInstance.init).toHaveBeenCalled()
     expect(llmInstance.run).toHaveBeenCalledWith(prompt)

From f2f13bb84eda9645275598580443ee54c60a3194 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 19 Sep 2024 09:25:25 +0100
Subject: [PATCH 081/101] Fix conditions with empty values falsely evaluating
 to true

---
 packages/client/src/utils/conditions.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js
index a23d879e45..b68afa4ab3 100644
--- a/packages/client/src/utils/conditions.js
+++ b/packages/client/src/utils/conditions.js
@@ -1,4 +1,5 @@
 import { QueryUtils } from "@budibase/frontend-core"
+import { EmptyFilterOption } from "@budibase/types"
 
 export const getActiveConditions = conditions => {
   if (!conditions?.length) {
@@ -33,7 +34,8 @@ export const getActiveConditions = conditions => {
       value: condition.referenceValue,
     }
 
-    const query = QueryUtils.buildQuery([luceneCondition])
+    let query = QueryUtils.buildQuery([luceneCondition])
+    query.onEmptyFilter = EmptyFilterOption.RETURN_NONE
     const result = QueryUtils.runQuery([luceneCondition], query)
     return result.length > 0
   })

From fd353b2ecf6c292a4ba6bcd5e18ba5c40ddfc194 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 10:09:21 +0100
Subject: [PATCH 082/101] wip

---
 .../server/src/integrations/googlesheets.ts   | 63 ++++++++------
 .../integrations/tests/googlesheets.spec.ts   | 19 ++++-
 .../integrations/tests/utils/googlesheets.ts  | 82 ++++++++++++-------
 packages/shared-core/src/utils.ts             | 53 ++++++------
 4 files changed, 133 insertions(+), 84 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index dd9bef84ab..65606ad75c 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -25,7 +25,11 @@ import {
   checkExternalTables,
   finaliseExternalTables,
 } from "./utils"
-import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
+import {
+  GoogleSpreadsheet,
+  GoogleSpreadsheetRow,
+  GoogleSpreadsheetWorksheet,
+} from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"
@@ -330,15 +334,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
       return { tables: {}, errors: {} }
     }
     await this.connect()
+
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
-    await utils.parallelForeach(
-      sheets,
-      async sheet => {
-        // must fetch rows to determine schema
+
+    let promises: Promise<void>[] = []
+    for (const sheet of sheets) {
+      const f = async (sheet: GoogleSpreadsheetWorksheet) => {
         try {
-          await sheet.getRows()
+          await sheet.getRows({ limit: 1 })
         } catch (err) {
           // We expect this to always be an Error so if it's not, rethrow it to
           // make sure we don't fail quietly.
@@ -346,26 +351,36 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
             throw err
           }
 
-          if (err.message.startsWith("No values in the header row")) {
-            errors[sheet.title] = err.message
-          } else {
-            // If we get an error we don't expect, rethrow to avoid failing
-            // quietly.
-            throw err
+          if (
+            err.message.startsWith("No values in the header row") ||
+            err.message.startsWith("Header values are not yet loaded")
+          ) {
+            errors[
+              sheet.title
+            ] = `Failed to find a header row in sheet "${sheet.title}", is the first row blank?`
+            return
           }
-          return
-        }
 
-        const id = buildExternalTableId(datasourceId, sheet.title)
-        tables[sheet.title] = this.getTableSchema(
-          sheet.title,
-          sheet.headerValues,
-          datasourceId,
-          id
-        )
-      },
-      10
-    )
+          // If we get an error we don't expect, rethrow to avoid failing
+          // quietly.
+          throw err
+        }
+      }
+      promises.push(f(sheet))
+    }
+
+    await Promise.all(promises)
+
+    for (const sheet of sheets) {
+      const id = buildExternalTableId(datasourceId, sheet.title)
+      tables[sheet.title] = this.getTableSchema(
+        sheet.title,
+        sheet.headerValues,
+        datasourceId,
+        id
+      )
+    }
+
     let externalTables = finaliseExternalTables(tables, entities)
     errors = { ...errors, ...checkExternalTables(externalTables) }
     return { tables: externalTables, errors }
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index cef4decfa7..fe2da479c3 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -10,7 +10,6 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import { GoogleSheetsMock } from "./utils/googlesheets"
-import rows from "src/sdk/app/rows"
 
 describe("Google Sheets Integration", () => {
   const config = new TestConfiguration()
@@ -506,4 +505,22 @@ describe("Google Sheets Integration", () => {
       expect(emptyRows.length).toEqual(0)
     })
   })
+
+  describe("import spreadsheet", () => {
+    it.only("should fail to import a completely blank sheet", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource._id!,
+          tablesFilter: ["Sheet1"],
+        },
+        {
+          status: 400,
+          body: {
+            message: "",
+          },
+        }
+      )
+    })
+  })
 })
diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts
index 4b17c25b01..4747f5f9bf 100644
--- a/packages/server/src/integrations/tests/utils/googlesheets.ts
+++ b/packages/server/src/integrations/tests/utils/googlesheets.ts
@@ -22,6 +22,7 @@ import type {
   CellPadding,
   Color,
   GridRange,
+  DataSourceSheetProperties,
 } from "google-spreadsheet/src/lib/types/sheets-types"
 
 const BLACK: Color = { red: 0, green: 0, blue: 0 }
@@ -91,7 +92,7 @@ interface UpdateValuesResponse {
 
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest
 interface AddSheetRequest {
-  properties: WorksheetProperties
+  properties: Partial<WorksheetProperties>
 }
 
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse
@@ -236,6 +237,38 @@ export class GoogleSheetsMock {
     this.mockAPI()
   }
 
+  public cell(cell: string): Value | undefined {
+    const cellData = this.cellData(cell)
+    if (!cellData) {
+      return undefined
+    }
+    return this.cellValue(cellData)
+  }
+
+  public set(cell: string, value: Value): void {
+    const cellData = this.cellData(cell)
+    if (!cellData) {
+      throw new Error(`Cell ${cell} not found`)
+    }
+    cellData.userEnteredValue = this.createValue(value)
+  }
+
+  public sheet(name: string | number): Sheet | undefined {
+    if (typeof name === "number") {
+      return this.getSheetById(name)
+    }
+    return this.getSheetByName(name)
+  }
+
+  public createSheet(opts: Partial<WorksheetProperties>): Sheet {
+    const properties = this.defaultWorksheetProperties(opts)
+    if (this.getSheetByName(properties.title)) {
+      throw new Error(`Sheet ${properties.title} already exists`)
+    }
+    const resp = this.handleAddSheet({ properties })
+    return this.getSheetById(resp.properties.sheetId)!
+  }
+
   private route(
     method: "get" | "put" | "post",
     path: string | RegExp,
@@ -462,35 +495,39 @@ export class GoogleSheetsMock {
     return response
   }
 
-  private handleAddSheet(request: AddSheetRequest): AddSheetResponse {
-    const properties: Omit<WorksheetProperties, "dataSourceSheetProperties"> = {
+  private defaultWorksheetProperties(
+    opts: Partial<WorksheetProperties>
+  ): WorksheetProperties {
+    return {
       index: this.spreadsheet.sheets.length,
       hidden: false,
       rightToLeft: false,
       tabColor: BLACK,
       tabColorStyle: { rgbColor: BLACK },
       sheetType: "GRID",
-      title: request.properties.title,
+      title: "Sheet",
       sheetId: this.spreadsheet.sheets.length,
       gridProperties: {
         rowCount: 100,
         columnCount: 26,
-        frozenRowCount: 0,
-        frozenColumnCount: 0,
-        hideGridlines: false,
-        rowGroupControlAfter: false,
-        columnGroupControlAfter: false,
       },
+      dataSourceSheetProperties: {} as DataSourceSheetProperties,
+      ...opts,
     }
+  }
 
+  private handleAddSheet(request: AddSheetRequest): AddSheetResponse {
+    const properties = this.defaultWorksheetProperties(request.properties)
     this.spreadsheet.sheets.push({
-      properties: properties as WorksheetProperties,
-      data: [this.createEmptyGrid(100, 26)],
+      properties,
+      data: [
+        this.createEmptyGrid(
+          properties.gridProperties.rowCount,
+          properties.gridProperties.columnCount
+        ),
+      ],
     })
-
-    // dataSourceSheetProperties is only returned by the API if the sheet type is
-    // DATA_SOURCE, which we aren't using, so sadly we need to cast here.
-    return { properties: properties as WorksheetProperties }
+    return { properties }
   }
 
   private handleDeleteRange(request: DeleteRangeRequest) {
@@ -767,21 +804,6 @@ export class GoogleSheetsMock {
     return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex)
   }
 
-  public cell(cell: string): Value | undefined {
-    const cellData = this.cellData(cell)
-    if (!cellData) {
-      return undefined
-    }
-    return this.cellValue(cellData)
-  }
-
-  public sheet(name: string | number): Sheet | undefined {
-    if (typeof name === "number") {
-      return this.getSheetById(name)
-    }
-    return this.getSheetByName(name)
-  }
-
   private getCellNumericIndexes(
     sheet: Sheet | number,
     row: number,
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index b69a059745..31ce8778cd 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -7,43 +7,38 @@ export function unreachable(
   throw new Error(message)
 }
 
+interface PromiseWithId<T> {
+  promise: Promise<T>
+  id: number
+}
+
 export async function parallelForeach<T>(
   items: T[],
   task: (item: T) => Promise<void>,
   maxConcurrency: number
 ): Promise<void> {
-  const promises: Promise<void>[] = []
-  let index = 0
+  try {
+    let next = 0
+    let inProgress: PromiseWithId<number>[] = []
+    while (next < items.length) {
+      if (inProgress.length === maxConcurrency) {
+        const finished = await Promise.race(inProgress.map(t => t.promise))
+        inProgress = inProgress.filter(task => task.id !== finished)
+      }
 
-  const processItem = async (item: T) => {
-    try {
-      await task(item)
-    } finally {
-      processNext()
+      const promise = async (next: number) => {
+        await task(items[next])
+        return next
+      }
+
+      inProgress.push({ promise: promise(next), id: next })
+      next++
     }
+    await Promise.all(inProgress.map(t => t.promise))
+  } catch (e) {
+    console.error(e)
+    throw e
   }
-
-  const processNext = () => {
-    if (index >= items.length) {
-      // No more items to process
-      return
-    }
-
-    const item = items[index]
-    index++
-
-    const promise = processItem(item)
-    promises.push(promise)
-
-    if (promises.length >= maxConcurrency) {
-      Promise.race(promises).then(processNext)
-    } else {
-      processNext()
-    }
-  }
-  processNext()
-
-  await Promise.all(promises)
 }
 
 export function filterValueToLabel() {

From be5a4f5d9763122f38d5c083cd4ccd70944f5f0b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 10:11:24 +0100
Subject: [PATCH 083/101] Require a flag to be set to query PostHog for feature
 flags.

---
 packages/backend-core/src/environment.ts    | 1 +
 packages/backend-core/src/features/index.ts | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 6bef6efeb3..0ffe9966b1 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -143,6 +143,7 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
   POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
+  POSTHOG_FEATURE_FLAGS: process.env.POSTHOG_FEATURE_FLAGS,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
   CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index 5b6ea4ca92..fda95c6553 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -6,7 +6,12 @@ import tracer from "dd-trace"
 
 let posthog: PostHog | undefined
 export function init(opts?: PostHogOptions) {
-  if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) {
+  if (
+    env.POSTHOG_TOKEN &&
+    env.POSTHOG_API_HOST &&
+    !env.SELF_HOSTED &&
+    env.POSTHOG_FEATURE_FLAGS
+  ) {
     console.log("initializing posthog client...")
     posthog = new PostHog(env.POSTHOG_TOKEN, {
       host: env.POSTHOG_API_HOST,

From 6d7cffa43e269ffd1672e4e7de169cf45b2d69f4 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 10:15:18 +0100
Subject: [PATCH 084/101] Make flag name more accurate.

---
 packages/backend-core/src/environment.ts    | 2 +-
 packages/backend-core/src/features/index.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 0ffe9966b1..2ab8c550cc 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -143,7 +143,7 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
   POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
-  POSTHOG_FEATURE_FLAGS: process.env.POSTHOG_FEATURE_FLAGS,
+  POSTHOG_FEATURE_FLAGS_ENABLED: process.env.POSTHOG_FEATURE_FLAGS_ENABLED,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
   CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index fda95c6553..0765d09036 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -10,7 +10,7 @@ export function init(opts?: PostHogOptions) {
     env.POSTHOG_TOKEN &&
     env.POSTHOG_API_HOST &&
     !env.SELF_HOSTED &&
-    env.POSTHOG_FEATURE_FLAGS
+    env.POSTHOG_FEATURE_FLAGS_ENABLED
   ) {
     console.log("initializing posthog client...")
     posthog = new PostHog(env.POSTHOG_TOKEN, {

From e0fb44f63fbde2997d1c1c4765409ccf6f7fc3a2 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 19 Sep 2024 10:21:49 +0100
Subject: [PATCH 085/101] lint

---
 packages/backend-core/src/configs/configs.ts  |   1 -
 .../src/api/controllers/global/configs.ts     |   3 +-
 yarn.lock                                     | 266 ++++++++----------
 3 files changed, 115 insertions(+), 155 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 6c2f2fe586..e4f4a874a5 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -1,6 +1,5 @@
 import {
   AIConfig,
-  AIInnerConfig,
   Config,
   ConfigType,
   GoogleConfig,
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index f3262dc0dd..e6e80ff3a5 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -253,8 +253,7 @@ export async function save(ctx: UserCtx<Config>) {
         if (existingConfig) {
           await verifyAIConfig(config, existingConfig)
         }
-        const numConfigs = Object.keys(config).length
-        await pro.quotas.updateCustomAIConfigCount(numConfigs)
+        await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length)
         break
     }
   } catch (err: any) {
diff --git a/yarn.lock b/yarn.lock
index 69c3978aaf..84d64637e4 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -772,20 +772,20 @@
     "@azure/abort-controller" "^1.0.0"
     tslib "^2.2.0"
 
-"@azure/identity@^3.4.1":
-  version "3.4.2"
-  resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de"
-  integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA==
+"@azure/identity@4.2.1", "@azure/identity@^3.4.1":
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c"
+  integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q==
   dependencies:
     "@azure/abort-controller" "^1.0.0"
     "@azure/core-auth" "^1.5.0"
     "@azure/core-client" "^1.4.0"
     "@azure/core-rest-pipeline" "^1.1.0"
     "@azure/core-tracing" "^1.0.0"
-    "@azure/core-util" "^1.6.1"
+    "@azure/core-util" "^1.3.0"
     "@azure/logger" "^1.0.0"
-    "@azure/msal-browser" "^3.5.0"
-    "@azure/msal-node" "^2.5.1"
+    "@azure/msal-browser" "^3.11.1"
+    "@azure/msal-node" "^2.9.2"
     events "^3.0.0"
     jws "^4.0.0"
     open "^8.0.0"
@@ -816,7 +816,7 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/msal-browser@^3.5.0":
+"@azure/msal-browser@^3.11.1":
   version "3.23.0"
   resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b"
   integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw==
@@ -828,7 +828,7 @@
   resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4"
   integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog==
 
-"@azure/msal-node@^2.5.1":
+"@azure/msal-node@^2.9.2":
   version "2.13.1"
   resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47"
   integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw==
@@ -2066,6 +2066,44 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.32.5":
+  version "0.0.0"
+  dependencies:
+    "@budibase/nano" "10.1.5"
+    "@budibase/pouchdb-replication-stream" "1.2.11"
+    "@budibase/shared-core" "0.0.0"
+    "@budibase/types" "0.0.0"
+    aws-cloudfront-sign "3.0.2"
+    aws-sdk "2.1030.0"
+    bcrypt "5.1.0"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dd-trace "5.2.0"
+    dotenv "16.0.1"
+    ioredis "5.3.2"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    knex "2.4.2"
+    koa-passport "^6.0.0"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    pino "8.11.0"
+    pino-http "8.3.3"
+    posthog-node "4.0.1"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    rotating-file-stream "3.1.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "^7.5.4"
+    tar-fs "2.1.1"
+    uuid "^8.3.2"
+
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2108,6 +2146,45 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@npm:@budibase/pro@latest":
+  version "2.32.5"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea"
+  integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA==
+  dependencies:
+    "@budibase/backend-core" "2.32.5"
+    "@budibase/shared-core" "2.32.5"
+    "@budibase/string-templates" "2.32.5"
+    "@budibase/types" "2.32.5"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    dd-trace "5.2.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    lru-cache "^7.14.1"
+    memorystream "^0.3.1"
+    node-fetch "2.6.7"
+    scim-patch "^0.8.1"
+    scim2-parse-filter "^0.2.8"
+
+"@budibase/shared-core@2.32.5":
+  version "0.0.0"
+  dependencies:
+    "@budibase/types" "0.0.0"
+    cron-validate "1.4.5"
+
+"@budibase/string-templates@2.32.5":
+  version "0.0.0"
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.13.2"
+    dayjs "^1.10.8"
+    handlebars "^4.7.8"
+    lodash.clonedeep "^4.5.0"
+
+"@budibase/types@2.32.5":
+  version "0.0.0"
+  dependencies:
+    scim-patch "^0.8.1"
+
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -7466,30 +7543,7 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0:
-  version "0.24.0"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
-  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
-  dependencies:
-    follow-redirects "^1.14.4"
-
-axios@1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
-  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
-  dependencies:
-    follow-redirects "^1.15.0"
-    form-data "^4.0.0"
-    proxy-from-env "^1.1.0"
-
-axios@^0.21.1:
-  version "0.21.4"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
-  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
-  dependencies:
-    follow-redirects "^1.14.0"
-
-axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
+axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11331,14 +11385,7 @@ fast-url-parser@^1.1.3:
   dependencies:
     punycode "^1.3.2"
 
-fast-xml-parser@4.2.5:
-  version "4.2.5"
-  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f"
-  integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==
-  dependencies:
-    strnum "^1.0.5"
-
-fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
+fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
   version "4.4.1"
   resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f"
   integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==
@@ -11592,11 +11639,6 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
-follow-redirects@^1.14.0, follow-redirects@^1.14.4:
-  version "1.15.9"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
-  integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==
-
 follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
@@ -12690,12 +12732,7 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1:
-  version "3.8.1"
-  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
-  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
-
-http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13220,7 +13257,7 @@ is-boolean-object@^1.1.0:
     call-bind "^1.0.2"
     has-tostringtag "^1.0.0"
 
-is-buffer@^1.1.5, is-buffer@~1.1.6:
+is-buffer@~1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
   integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
@@ -13706,6 +13743,11 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
+isobject@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
+  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
+
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -14615,14 +14657,7 @@ kill-port@^1.6.1:
     get-them-args "1.3.2"
     shell-exec "1.0.2"
 
-kind-of@^3.0.2, kind-of@^3.1.0:
-  version "3.2.2"
-  resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
-  integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
-  dependencies:
-    is-buffer "^1.1.5"
-
-kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
+kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
   version "6.0.3"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd"
   integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==
@@ -16265,7 +16300,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@^1.5.2:
+msgpackr@1.10.1, msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16459,25 +16494,13 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
-  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
-
-node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
+node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-fetch@^2.6.9, node-fetch@^2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
-  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -17629,15 +17652,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@^0.4.0:
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
-  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
-  dependencies:
-    passport-strategy "1.x.x"
-    pause "0.0.1"
-
-passport@^0.6.0:
+passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -18943,7 +18958,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.28, psl@^1.1.33:
+psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -20015,11 +20030,6 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
-sax@>=0.1.1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
-  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
-
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -20092,33 +20102,13 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
-  version "5.7.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
-  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
-
-semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
+"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
-  version "6.3.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
-  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
-
-semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
-  version "7.6.3"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
-  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
-
-semver@~2.3.1:
-  version "2.3.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
-  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
-
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -21681,7 +21671,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
+tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21691,14 +21681,6 @@ touch@^3.1.0:
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
-tough-cookie@~2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
-  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
-  dependencies:
-    psl "^1.1.28"
-    punycode "^2.1.1"
-
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -22227,6 +22209,14 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
+unset-value@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
+  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
+  dependencies:
+    has-value "^2.0.2"
+    isobject "^4.0.0"
+
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -23001,33 +22991,10 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x:
-  version "0.1.14"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
-  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
-  dependencies:
-    sax ">=0.1.1"
-
-xml2js@0.4.19:
-  version "0.4.19"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
-  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~9.0.1"
-
-xml2js@0.5.0:
-  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
-  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~11.0.0"
-
-xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.4.23"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
-  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
+xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
+  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -23037,11 +23004,6 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
-xmlbuilder@~9.0.1:
-  version "9.0.7"
-  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
-  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
-
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"

From 062732b29d3514010c1b17743d74c0e673d414a8 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 19 Sep 2024 10:36:00 +0100
Subject: [PATCH 086/101] lint

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 4300b75bde..5929fbb3c6 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 4300b75bde89e37bb0bace927a0e8d204367ae71
+Subproject commit 5929fbb3c69b46c525faef4a5a805e0a0015bd8e

From 6a5f001977a4cfeda3f9d1842d2716165262df4a Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 19 Sep 2024 10:54:25 +0100
Subject: [PATCH 087/101] remove tests that are no longer relevant in worker

---
 .../controllers/global/tests/configs.spec.ts  | 49 -------------------
 1 file changed, 49 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 3ff6a5298c..ad2e2aa974 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -35,55 +35,6 @@ describe("Global configs controller", () => {
     })
   })
 
-  it("Should return the default BB AI config when the feature is turned on", async () => {
-    jest
-      .spyOn(pro.features, "isBudibaseAIEnabled")
-      .mockImplementation(() => Promise.resolve(true))
-    const data = structures.configs.ai()
-    await config.api.configs.saveConfig(data)
-    const response = await config.api.configs.getAIConfig()
-
-    expect(response.body.config).toEqual({
-      budibase_ai: {
-        provider: "OpenAI",
-        active: true,
-        isDefault: true,
-        name: "Budibase AI",
-        defaultModel: "",
-      },
-      ai: {
-        active: true,
-        apiKey: "--secret-value--",
-        baseUrl: "https://api.example.com",
-        defaultModel: "gpt4",
-        isDefault: false,
-        name: "Test",
-        provider: "OpenAI",
-      },
-    })
-  })
-
-  it("Should not not return the default Budibase AI config when on self host", async () => {
-    jest
-      .spyOn(pro.features, "isBudibaseAIEnabled")
-      .mockImplementation(() => Promise.resolve(false))
-    const data = structures.configs.ai()
-    await config.api.configs.saveConfig(data)
-    const response = await config.api.configs.getAIConfig()
-
-    expect(response.body.config).toEqual({
-      ai: {
-        active: true,
-        apiKey: "--secret-value--",
-        baseUrl: "https://api.example.com",
-        defaultModel: "gpt4",
-        isDefault: false,
-        name: "Test",
-        provider: "OpenAI",
-      },
-    })
-  })
-
   it("Should not update existing secrets when updating an existing AI Config", async () => {
     const data = structures.configs.ai()
     await config.api.configs.saveConfig(data)

From 74effbb55b51da624bee420c5ef054ee93fbd31a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 11:41:10 +0100
Subject: [PATCH 088/101] Fix tests.

---
 packages/backend-core/src/features/tests/features.spec.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts
index d092585cc6..01c9bfa3c6 100644
--- a/packages/backend-core/src/features/tests/features.spec.ts
+++ b/packages/backend-core/src/features/tests/features.spec.ts
@@ -148,6 +148,7 @@ describe("feature flags", () => {
       const env: Partial<typeof environment> = {
         TENANT_FEATURE_FLAGS: environmentFlags,
         SELF_HOSTED: false,
+        POSTHOG_FEATURE_FLAGS_ENABLED: "true",
       }
 
       if (posthogFlags) {

From 5d945b90ff8a650239eac13b5d9a2b7f54c6c973 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 19 Sep 2024 12:58:57 +0000
Subject: [PATCH 089/101] Bump version to 2.32.6

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index c710d888c7..d695869907 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.5",
+  "version": "2.32.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From aecd4f9e4d205411e89084574c03c670a4426827 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:48:50 +0100
Subject: [PATCH 090/101] Fetch schema tests.

---
 packages/server/package.json                  |  2 +-
 .../server/src/integrations/googlesheets.ts   | 17 ++--
 .../integrations/tests/googlesheets.spec.ts   | 83 +++++++++++++++++-
 packages/shared-core/src/utils.ts             | 40 ++++-----
 yarn.lock                                     | 85 ++++++++++++++++++-
 5 files changed, 187 insertions(+), 40 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 6dfd528963..0b36d49c2e 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -80,7 +80,7 @@
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
-    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
     "jimp": "0.22.12",
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 65606ad75c..ea06ca8d69 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -339,9 +339,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
     const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
 
-    let promises: Promise<void>[] = []
-    for (const sheet of sheets) {
-      const f = async (sheet: GoogleSpreadsheetWorksheet) => {
+    await utils.parallelForEach(
+      sheets,
+      async sheet => {
         try {
           await sheet.getRows({ limit: 1 })
         } catch (err) {
@@ -353,7 +353,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
 
           if (
             err.message.startsWith("No values in the header row") ||
-            err.message.startsWith("Header values are not yet loaded")
+            err.message.startsWith("Header values are not yet loaded") ||
+            err.message.startsWith("All your header cells are blank")
           ) {
             errors[
               sheet.title
@@ -365,11 +366,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
           // quietly.
           throw err
         }
-      }
-      promises.push(f(sheet))
-    }
-
-    await Promise.all(promises)
+      },
+      { maxConcurrency: 2 }
+    )
 
     for (const sheet of sheets) {
       const id = buildExternalTableId(datasourceId, sheet.title)
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index fe2da479c3..dcf4a61b50 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -506,8 +506,8 @@ describe("Google Sheets Integration", () => {
     })
   })
 
-  describe("import spreadsheet", () => {
-    it.only("should fail to import a completely blank sheet", async () => {
+  describe("fetch schema", () => {
+    it("should fail to import a completely blank sheet", async () => {
       mock.createSheet({ title: "Sheet1" })
       await config.api.datasource.fetchSchema(
         {
@@ -515,12 +515,87 @@ describe("Google Sheets Integration", () => {
           tablesFilter: ["Sheet1"],
         },
         {
-          status: 400,
+          status: 200,
           body: {
-            message: "",
+            errors: {
+              Sheet1:
+                'Failed to find a header row in sheet "Sheet1", is the first row blank?',
+            },
           },
         }
       )
     })
+
+    it("should fail to import multiple sheets with blank headers", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      mock.createSheet({ title: "Sheet2" })
+
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource!._id!,
+          tablesFilter: ["Sheet1", "Sheet2"],
+        },
+        {
+          status: 200,
+          body: {
+            errors: {
+              Sheet1:
+                'Failed to find a header row in sheet "Sheet1", is the first row blank?',
+              Sheet2:
+                'Failed to find a header row in sheet "Sheet2", is the first row blank?',
+            },
+          },
+        }
+      )
+    })
+
+    it("should only fail the sheet with missing headers", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      mock.createSheet({ title: "Sheet2" })
+      mock.createSheet({ title: "Sheet3" })
+
+      mock.set("Sheet1!A1", "name")
+      mock.set("Sheet1!B1", "dob")
+      mock.set("Sheet2!A1", "name")
+      mock.set("Sheet2!B1", "dob")
+
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource!._id!,
+          tablesFilter: ["Sheet1", "Sheet2", "Sheet3"],
+        },
+        {
+          status: 200,
+          body: {
+            errors: {
+              Sheet3:
+                'Failed to find a header row in sheet "Sheet3", is the first row blank?',
+            },
+          },
+        }
+      )
+    })
+
+    it("should only succeed if sheet with missing headers is not being imported", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      mock.createSheet({ title: "Sheet2" })
+      mock.createSheet({ title: "Sheet3" })
+
+      mock.set("Sheet1!A1", "name")
+      mock.set("Sheet1!B1", "dob")
+      mock.set("Sheet2!A1", "name")
+      mock.set("Sheet2!B1", "dob")
+
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource!._id!,
+          tablesFilter: ["Sheet1", "Sheet2"],
+        },
+        {
+          status: 200,
+          body: { errors: {} },
+        }
+      )
+    })
   })
 })
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 31ce8778cd..9ead522192 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -12,33 +12,29 @@ interface PromiseWithId<T> {
   id: number
 }
 
-export async function parallelForeach<T>(
+export async function parallelForEach<T>(
   items: T[],
   task: (item: T) => Promise<void>,
-  maxConcurrency: number
+  opts?: { maxConcurrency?: number }
 ): Promise<void> {
-  try {
-    let next = 0
-    let inProgress: PromiseWithId<number>[] = []
-    while (next < items.length) {
-      if (inProgress.length === maxConcurrency) {
-        const finished = await Promise.race(inProgress.map(t => t.promise))
-        inProgress = inProgress.filter(task => task.id !== finished)
-      }
-
-      const promise = async (next: number) => {
-        await task(items[next])
-        return next
-      }
-
-      inProgress.push({ promise: promise(next), id: next })
-      next++
+  const { maxConcurrency = 10 } = opts || {}
+  let next = 0
+  let inProgress: PromiseWithId<number>[] = []
+  while (next < items.length) {
+    if (inProgress.length === maxConcurrency) {
+      const finished = await Promise.race(inProgress.map(t => t.promise))
+      inProgress = inProgress.filter(task => task.id !== finished)
     }
-    await Promise.all(inProgress.map(t => t.promise))
-  } catch (e) {
-    console.error(e)
-    throw e
+
+    const promise = async (next: number) => {
+      await task(items[next])
+      return next
+    }
+
+    inProgress.push({ promise: promise(next), id: next })
+    next++
   }
+  await Promise.all(inProgress.map(t => t.promise))
 }
 
 export function filterValueToLabel() {
diff --git a/yarn.lock b/yarn.lock
index 110cbd7a15..34c9cb15d9 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2053,6 +2053,44 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.32.6":
+  version "0.0.0"
+  dependencies:
+    "@budibase/nano" "10.1.5"
+    "@budibase/pouchdb-replication-stream" "1.2.11"
+    "@budibase/shared-core" "0.0.0"
+    "@budibase/types" "0.0.0"
+    aws-cloudfront-sign "3.0.2"
+    aws-sdk "2.1030.0"
+    bcrypt "5.1.0"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dd-trace "5.2.0"
+    dotenv "16.0.1"
+    ioredis "5.3.2"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    knex "2.4.2"
+    koa-passport "^6.0.0"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    pino "8.11.0"
+    pino-http "8.3.3"
+    posthog-node "4.0.1"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    rotating-file-stream "3.1.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "^7.5.4"
+    tar-fs "2.1.1"
+    uuid "^8.3.2"
+
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2095,6 +2133,45 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@npm:@budibase/pro@latest":
+  version "2.32.6"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f"
+  integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ==
+  dependencies:
+    "@budibase/backend-core" "2.32.6"
+    "@budibase/shared-core" "2.32.6"
+    "@budibase/string-templates" "2.32.6"
+    "@budibase/types" "2.32.6"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    dd-trace "5.2.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    lru-cache "^7.14.1"
+    memorystream "^0.3.1"
+    node-fetch "2.6.7"
+    scim-patch "^0.8.1"
+    scim2-parse-filter "^0.2.8"
+
+"@budibase/shared-core@2.32.6":
+  version "0.0.0"
+  dependencies:
+    "@budibase/types" "0.0.0"
+    cron-validate "1.4.5"
+
+"@budibase/string-templates@2.32.6":
+  version "0.0.0"
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.13.2"
+    dayjs "^1.10.8"
+    handlebars "^4.7.8"
+    lodash.clonedeep "^4.5.0"
+
+"@budibase/types@2.32.6":
+  version "0.0.0"
+  dependencies:
+    scim-patch "^0.8.1"
+
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -12277,10 +12354,10 @@ google-p12-pem@^4.0.0:
   dependencies:
     node-forge "^1.3.1"
 
-"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
-  integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
+"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5":
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f"
+  integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ==
   dependencies:
     axios "^1.4.0"
     lodash "^4.17.21"

From ce105d8f4e590d68008427ce3702e10ae3398a62 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:51:00 +0100
Subject: [PATCH 091/101] Revert unnecessary change.

---
 .../server/src/integrations/googlesheets.ts   |  4 +-
 packages/shared-core/src/utils.ts             | 53 +++++++++++--------
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index ea06ca8d69..d5667c276b 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -339,7 +339,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
     const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
 
-    await utils.parallelForEach(
+    await utils.parallelForeach(
       sheets,
       async sheet => {
         try {
@@ -367,7 +367,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
           throw err
         }
       },
-      { maxConcurrency: 2 }
+      10
     )
 
     for (const sheet of sheets) {
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 9ead522192..b69a059745 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -7,34 +7,43 @@ export function unreachable(
   throw new Error(message)
 }
 
-interface PromiseWithId<T> {
-  promise: Promise<T>
-  id: number
-}
-
-export async function parallelForEach<T>(
+export async function parallelForeach<T>(
   items: T[],
   task: (item: T) => Promise<void>,
-  opts?: { maxConcurrency?: number }
+  maxConcurrency: number
 ): Promise<void> {
-  const { maxConcurrency = 10 } = opts || {}
-  let next = 0
-  let inProgress: PromiseWithId<number>[] = []
-  while (next < items.length) {
-    if (inProgress.length === maxConcurrency) {
-      const finished = await Promise.race(inProgress.map(t => t.promise))
-      inProgress = inProgress.filter(task => task.id !== finished)
-    }
+  const promises: Promise<void>[] = []
+  let index = 0
 
-    const promise = async (next: number) => {
-      await task(items[next])
-      return next
+  const processItem = async (item: T) => {
+    try {
+      await task(item)
+    } finally {
+      processNext()
     }
-
-    inProgress.push({ promise: promise(next), id: next })
-    next++
   }
-  await Promise.all(inProgress.map(t => t.promise))
+
+  const processNext = () => {
+    if (index >= items.length) {
+      // No more items to process
+      return
+    }
+
+    const item = items[index]
+    index++
+
+    const promise = processItem(item)
+    promises.push(promise)
+
+    if (promises.length >= maxConcurrency) {
+      Promise.race(promises).then(processNext)
+    } else {
+      processNext()
+    }
+  }
+  processNext()
+
+  await Promise.all(promises)
 }
 
 export function filterValueToLabel() {

From d0a0e74f393903143896d4c7ffda596bfb717705 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:51:26 +0100
Subject: [PATCH 092/101] Remove unused type.

---
 packages/server/src/integrations/googlesheets.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index d5667c276b..8710932ef6 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -25,11 +25,7 @@ import {
   checkExternalTables,
   finaliseExternalTables,
 } from "./utils"
-import {
-  GoogleSpreadsheet,
-  GoogleSpreadsheetRow,
-  GoogleSpreadsheetWorksheet,
-} from "google-spreadsheet"
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"

From bd618f2b006055ec70dcfce8e3233f335085bd8f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:53:00 +0100
Subject: [PATCH 093/101] Remove unneeded error message check.

---
 packages/server/src/integrations/googlesheets.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 8710932ef6..6012ff7789 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -349,7 +349,6 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
 
           if (
             err.message.startsWith("No values in the header row") ||
-            err.message.startsWith("Header values are not yet loaded") ||
             err.message.startsWith("All your header cells are blank")
           ) {
             errors[

From b45017dd5af5984009e399105a1c6859c5c43d0e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Thu, 19 Sep 2024 17:06:09 +0100
Subject: [PATCH 094/101] lint

---
 packages/worker/src/api/controllers/global/tests/configs.spec.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index ad2e2aa974..9091f29247 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -1,4 +1,3 @@
-import * as pro from "@budibase/pro"
 import { verifyAIConfig } from "../configs"
 import { TestConfiguration, structures } from "../../../../tests"
 import { AIInnerConfig } from "@budibase/types"

From af26b915b18e105f07d7a3bf9b525b08a3f246df Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 20 Sep 2024 10:17:45 +0100
Subject: [PATCH 095/101] Add Google Sheets search test.

---
 .../server/src/integrations/googlesheets.ts   |  11 +-
 .../integrations/tests/googlesheets.spec.ts   | 190 ++++++++++++++++++
 .../integrations/tests/utils/googlesheets.ts  |   2 +
 3 files changed, 197 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 6012ff7789..831528f84d 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -581,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
         rows = await sheet.getRows()
       }
 
-      if (hasFilters && query.paginate) {
-        rows = rows.slice(offset, offset + limit)
-      }
-      const headerValues = sheet.headerValues
-
       let response = rows.map(row =>
-        this.buildRowObject(headerValues, row.toObject(), row.rowNumber)
+        this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber)
       )
       response = dataFilters.runQuery(response, query.filters || {})
 
+      if (hasFilters && query.paginate) {
+        response = response.slice(offset, offset + limit)
+      }
+
       if (query.sort) {
         if (Object.keys(query.sort).length !== 1) {
           console.warn("Googlesheets does not support multiple sorting", {
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index dcf4a61b50..34be1c0c6c 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import {
   Datasource,
   FieldType,
+  Row,
   SourceName,
   Table,
   TableSourceType,
@@ -598,4 +599,193 @@ describe("Google Sheets Integration", () => {
       )
     })
   })
+
+  describe("search", () => {
+    let table: Table
+
+    beforeEach(async () => {
+      table = await config.api.table.save({
+        name: "Test Table",
+        type: "table",
+        sourceId: datasource._id!,
+        sourceType: TableSourceType.EXTERNAL,
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+            constraints: {
+              type: "string",
+            },
+          },
+        },
+      })
+
+      await config.api.row.bulkImport(table._id!, {
+        rows: [
+          {
+            name: "Foo",
+          },
+          {
+            name: "Bar",
+          },
+          {
+            name: "Baz",
+          },
+        ],
+      })
+    })
+
+    it("should be able to find rows with equals filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          equal: {
+            name: "Foo",
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(1)
+      expect(response.rows[0].name).toEqual("Foo")
+    })
+
+    it("should be able to find rows with not equals filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          notEqual: {
+            name: "Foo",
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(2)
+      expect(response.rows[0].name).toEqual("Bar")
+      expect(response.rows[1].name).toEqual("Baz")
+    })
+
+    it("should be able to find rows with empty filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          empty: {
+            name: null,
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(0)
+    })
+
+    it("should be able to find rows with not empty filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          notEmpty: {
+            name: null,
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(3)
+    })
+
+    it("should be able to find rows with one of filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          oneOf: {
+            name: ["Foo", "Bar"],
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(2)
+      expect(response.rows[0].name).toEqual("Foo")
+      expect(response.rows[1].name).toEqual("Bar")
+    })
+
+    it("should be able to find rows with fuzzy filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          fuzzy: {
+            name: "oo",
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(1)
+      expect(response.rows[0].name).toEqual("Foo")
+    })
+
+    it("should be able to find rows with range filter", async () => {
+      const response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {
+          range: {
+            name: {
+              low: "A",
+              high: "C",
+            },
+          },
+        },
+      })
+
+      expect(response.rows).toHaveLength(2)
+      expect(response.rows[0].name).toEqual("Bar")
+      expect(response.rows[1].name).toEqual("Baz")
+    })
+
+    it("should paginate correctly", async () => {
+      await config.api.row.bulkImport(table._id!, {
+        rows: Array.from({ length: 50 }, () => ({
+          name: `Unique value!`,
+        })),
+      })
+      await config.api.row.bulkImport(table._id!, {
+        rows: Array.from({ length: 50 }, () => ({
+          name: `Non-unique value!`,
+        })),
+      })
+
+      let response = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: { equal: { name: "Unique value!" } },
+        paginate: true,
+        limit: 10,
+      })
+      let rows: Row[] = response.rows
+
+      while (response.hasNextPage) {
+        response = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query: { equal: { name: "Unique value!" } },
+          paginate: true,
+          limit: 10,
+          bookmark: response.bookmark,
+        })
+
+        expect(response.rows.length).toBeLessThanOrEqual(10)
+        rows = rows.concat(response.rows)
+      }
+
+      // Make sure we only get rows matching the query.
+      expect(rows.length).toEqual(50)
+      expect(rows.map(row => row.name)).toEqual(
+        expect.arrayContaining(
+          Array.from({ length: 50 }, () => "Unique value!")
+        )
+      )
+
+      // Make sure all of the rows have a unique ID.
+      const ids = Object.keys(
+        rows.reduce((acc, row) => {
+          acc[row._id!] = true
+          return acc
+        }, {})
+      )
+      expect(ids.length).toEqual(50)
+    })
+  })
 })
diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts
index 4747f5f9bf..4b9445ebca 100644
--- a/packages/server/src/integrations/tests/utils/googlesheets.ts
+++ b/packages/server/src/integrations/tests/utils/googlesheets.ts
@@ -440,6 +440,8 @@ export class GoogleSheetsMock {
       endColumnIndex: 0,
     })
 
+    sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length
+
     return {
       spreadsheetId: this.spreadsheet.spreadsheetId,
       tableRange: range,

From 627c1e713eaaa05a3454bc51f3b29d782cb998e7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 20 Sep 2024 14:52:28 +0100
Subject: [PATCH 096/101] Updating CouchDB image versions across the board to
 v3.3.3-sqs-v2.1.1

---
 .github/workflows/budibase_ci.yml | 4 ++--
 charts/budibase/values.yaml       | 2 +-
 globalSetup.ts                    | 2 +-
 hosting/single/Dockerfile         | 2 +-
 scripts/build-single-image-sqs.sh | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 4c5cc94d2b..4b9ebf1e5d 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -108,7 +108,7 @@ jobs:
       - name: Pull testcontainers images
         run: |
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb:v3.3.3 &
+          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
           docker pull redis &
 
           wait $(jobs -p)
@@ -179,7 +179,7 @@ jobs:
           docker pull minio/minio &
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb:v3.3.3 &
+          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
 
           wait $(jobs -p)
 
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index dde912410c..2c1525bd90 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -641,7 +641,7 @@ couchdb:
     # @ignore
     repository: budibase/couchdb
     # @ignore
-    tag: v3.3.3
+    tag: v3.3.3-sqs-v2.1.1
     # @ignore
     pullPolicy: Always
 
diff --git a/globalSetup.ts b/globalSetup.ts
index aa1cb00fe1..5d8b0381c0 100644
--- a/globalSetup.ts
+++ b/globalSetup.ts
@@ -46,7 +46,7 @@ export default async function setup() {
   await killContainers(containers)
 
   try {
-    const couchdb = new GenericContainer("budibase/couchdb:v3.3.3")
+    const couchdb = new GenericContainer("budibase/couchdb:v3.3.3-sqs-v2.1.1")
       .withExposedPorts(5984, 4984)
       .withEnvironment({
         COUCHDB_PASSWORD: "budibase",
diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile
index dfcfe566bd..ded0bc17dc 100644
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASEIMG=budibase/couchdb:v3.3.3
+ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1
 FROM node:20-slim as build
 
 # install node-gyp dependencies
diff --git a/scripts/build-single-image-sqs.sh b/scripts/build-single-image-sqs.sh
index 502ba5fa14..40b97013a1 100644
--- a/scripts/build-single-image-sqs.sh
+++ b/scripts/build-single-image-sqs.sh
@@ -2,4 +2,4 @@
 
 yarn build:apps
 version=$(./scripts/getCurrentVersion.sh)
-docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs .
+docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 .

From 876d837c3301c03eb09fe3cad2692ea3acf47105 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 20 Sep 2024 14:58:10 +0100
Subject: [PATCH 097/101] adjust instantiation of llm class after static
 initialiser change

---
 packages/pro                                    | 2 +-
 packages/server/src/automations/steps/openai.ts | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/pro b/packages/pro
index 5929fbb3c6..922431260e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 5929fbb3c69b46c525faef4a5a805e0a0015bd8e
+Subproject commit 922431260e90d558a1ca55398475412e75088057
diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 5eff12db47..b1dfa3df5b 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -103,8 +103,7 @@ export async function run({
     const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
 
     if (budibaseAIEnabled || customConfigsEnabled) {
-      const llm = new pro.ai.LargeLanguageModel(inputs.model)
-      await llm.init()
+      const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model)
       response = await llm.run(inputs.prompt)
     } else {
       // fallback to the default that uses the environment variable for backwards compat

From 4fec5fcddd3e49ebdaebfe536ef7685ce3240fbd Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 20 Sep 2024 17:48:56 +0100
Subject: [PATCH 098/101] update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 922431260e..f088614945 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 922431260e90d558a1ca55398475412e75088057
+Subproject commit f088614945e4d3da8aafb4149abe0f1bfc62df98

From 3b984629ac9318899a642a824dfaecf609e999c7 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 20 Sep 2024 18:35:34 +0100
Subject: [PATCH 099/101] pro reference

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index f088614945..1da5c27fae 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit f088614945e4d3da8aafb4149abe0f1bfc62df98
+Subproject commit 1da5c27fae01692b901114351911fe7f2d41aeea

From dc6c6ad9c80e515aeb93508e85a5118fb03c9793 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 20 Sep 2024 18:42:50 +0100
Subject: [PATCH 100/101] ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 1da5c27fae..e2fe0f9cc8 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 1da5c27fae01692b901114351911fe7f2d41aeea
+Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733

From c8d1956c73baed7c025fa15b2144cdd7bc309de4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 23 Sep 2024 12:59:20 +0100
Subject: [PATCH 101/101] Fixing AI test cases.

---
 .../src/automations/tests/openai.spec.ts      | 23 ++++++-----
 yarn.lock                                     | 39 +++++++++++++++----
 2 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts
index 342288a6a1..6b5e1fbb6a 100644
--- a/packages/server/src/automations/tests/openai.spec.ts
+++ b/packages/server/src/automations/tests/openai.spec.ts
@@ -23,14 +23,15 @@ jest.mock("openai", () => ({
     },
   })),
 }))
-
 jest.mock("@budibase/pro", () => ({
   ...jest.requireActual("@budibase/pro"),
   ai: {
-    LargeLanguageModel: jest.fn().mockImplementation(() => ({
-      init: jest.fn(),
-      run: jest.fn(),
-    })),
+    LargeLanguageModel: {
+      forCurrentTenant: jest.fn().mockImplementation(() => ({
+        init: jest.fn(),
+        run: jest.fn(),
+      })),
+    },
   },
   features: {
     isAICustomConfigsEnabled: jest.fn(),
@@ -38,6 +39,7 @@ jest.mock("@budibase/pro", () => ({
   },
 }))
 
+const mockedPro = jest.mocked(pro)
 const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
 
 const OPENAI_PROMPT = "What is the meaning of life?"
@@ -121,11 +123,14 @@ describe("test the openai action", () => {
       prompt,
     })
 
-    expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini")
+    expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith(
+      "gpt-4o-mini"
+    )
 
-    // @ts-ignore
-    const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value
-    expect(llmInstance.init).toHaveBeenCalled()
+    const llmInstance =
+      mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value
+    // init does not appear to be called currently
+    // expect(llmInstance.init).toHaveBeenCalled()
     expect(llmInstance.run).toHaveBeenCalledWith(prompt)
   })
 })
diff --git a/yarn.lock b/yarn.lock
index 84d64637e4..cd850e833d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12372,10 +12372,10 @@ google-p12-pem@^4.0.0:
   dependencies:
     node-forge "^1.3.1"
 
-"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
-  integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
+"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5":
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f"
+  integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ==
   dependencies:
     axios "^1.4.0"
     lodash "^4.17.21"
@@ -20786,7 +20786,16 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
+"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20877,7 +20886,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20891,6 +20900,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22846,7 +22862,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22864,6 +22880,15 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"