From 406410d9168377cf84d3a6f61b8dfe1357721f8e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 19 Dec 2023 15:57:56 +0000
Subject: [PATCH 01/68] Add DataDog tracing to automations.

---
 packages/server/src/automations/utils.ts  |  69 ++-
 packages/server/src/threads/automation.ts | 580 ++++++++++++----------
 2 files changed, 378 insertions(+), 271 deletions(-)

diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index 53c4d9d3b7..0c28787f67 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -16,6 +16,7 @@ import {
 } from "@budibase/types"
 import sdk from "../sdk"
 import { automationsEnabled } from "../features"
+import tracer from "dd-trace"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -39,26 +40,62 @@ function loggingArgs(job: AutomationJob) {
 }
 
 export async function processEvent(job: AutomationJob) {
-  const appId = job.data.event.appId!
-  const automationId = job.data.automation._id!
-  const task = async () => {
-    try {
-      // need to actually await these so that an error can be captured properly
-      console.log("automation running", ...loggingArgs(job))
+  return tracer.trace(
+    "processEvent",
+    { resource: "automation" },
+    async span => {
+      const appId = job.data.event.appId!
+      const automationId = job.data.automation._id!
 
-      const runFn = () => Runner.run(job)
-      const result = await quotas.addAutomation(runFn, {
+      span?.addTags({
+        appId,
         automationId,
+        job: {
+          id: job.id,
+          name: job.name,
+          attemptsMade: job.attemptsMade,
+          opts: {
+            attempts: job.opts.attempts,
+            priority: job.opts.priority,
+            delay: job.opts.delay,
+            repeat: job.opts.repeat,
+            backoff: job.opts.backoff,
+            lifo: job.opts.lifo,
+            timeout: job.opts.timeout,
+            jobId: job.opts.jobId,
+            removeOnComplete: job.opts.removeOnComplete,
+            removeOnFail: job.opts.removeOnFail,
+            stackTraceLimit: job.opts.stackTraceLimit,
+            preventParsingData: job.opts.preventParsingData,
+          },
+        },
       })
-      console.log("automation completed", ...loggingArgs(job))
-      return result
-    } catch (err) {
-      console.error(`automation was unable to run`, err, ...loggingArgs(job))
-      return { err }
-    }
-  }
 
-  return await context.doInAutomationContext({ appId, automationId, task })
+      const task = async () => {
+        try {
+          // need to actually await these so that an error can be captured properly
+          console.log("automation running", ...loggingArgs(job))
+
+          const runFn = () => Runner.run(job)
+          const result = await quotas.addAutomation(runFn, {
+            automationId,
+          })
+          console.log("automation completed", ...loggingArgs(job))
+          return result
+        } catch (err) {
+          span?.addTags({ error: true })
+          console.error(
+            `automation was unable to run`,
+            err,
+            ...loggingArgs(job)
+          )
+          return { err }
+        }
+      }
+
+      return await context.doInAutomationContext({ appId, automationId, task })
+    }
+  )
 }
 
 export async function updateTestHistory(
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index d1fcc2be72..4447899f96 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -34,6 +34,7 @@ import { cloneDeep } from "lodash/fp"
 import { performance } from "perf_hooks"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
+import tracer from "dd-trace"
 
 threadUtils.threadSetup()
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
@@ -242,278 +243,347 @@ class Orchestrator {
   }
 
   async execute(): Promise<any> {
-    // this will retrieve from context created at start of thread
-    this._context.env = await sdkUtils.getEnvironmentVariables()
-    let automation = this._automation
-    let stopped = false
-    let loopStep: AutomationStep | undefined = undefined
+    return tracer.trace(
+      "Orchestrator.execute",
+      { resource: "automation" },
+      async span => {
+        span?.addTags({
+          appId: this._appId,
+          automationId: this._automation._id,
+        })
 
-    let stepCount = 0
-    let loopStepNumber: any = undefined
-    let loopSteps: LoopStep[] | undefined = []
-    let metadata
-    let timeoutFlag = false
-    let wasLoopStep = false
-    let timeout = this._job.data.event.timeout
-    // check if this is a recurring automation,
-    if (isProdAppID(this._appId) && isRecurring(automation)) {
-      metadata = await this.getMetadata()
-      const shouldStop = await this.checkIfShouldStop(metadata)
-      if (shouldStop) {
-        return
-      }
-    }
-    const start = performance.now()
-    for (let step of automation.definition.steps) {
-      if (timeoutFlag) {
-        break
-      }
+        // this will retrieve from context created at start of thread
+        this._context.env = await sdkUtils.getEnvironmentVariables()
+        let automation = this._automation
+        let stopped = false
+        let loopStep: AutomationStep | undefined = undefined
 
-      if (timeout) {
-        setTimeout(() => {
-          timeoutFlag = true
-        }, timeout || 12000)
-      }
-
-      stepCount++
-      let input: any,
-        iterations = 1,
-        iterationCount = 0
-
-      if (step.stepId === LOOP_STEP_ID) {
-        loopStep = step
-        loopStepNumber = stepCount
-        continue
-      }
-
-      if (loopStep) {
-        input = await processObject(loopStep.inputs, this._context)
-        iterations = getLoopIterations(loopStep as LoopStep)
-      }
-      for (let index = 0; index < iterations; index++) {
-        let originalStepInput = cloneDeep(step.inputs)
-        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
-        if (loopStep && input.binding) {
-          let tempOutput = { items: loopSteps, iterations: iterationCount }
-          try {
-            loopStep.inputs.binding = automationUtils.typecastForLooping(
-              loopStep as LoopStep,
-              loopStep.inputs as LoopInput
-            )
-          } catch (err) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.INCORRECT_TYPE,
-              success: false,
-            })
-            loopSteps = undefined
-            loopStep = undefined
-            break
-          }
-          let item = []
-          if (
-            typeof loopStep.inputs.binding === "string" &&
-            loopStep.inputs.option === "String"
-          ) {
-            item = automationUtils.stringSplit(loopStep.inputs.binding)
-          } else if (Array.isArray(loopStep.inputs.binding)) {
-            item = loopStep.inputs.binding
-          }
-          this._context.steps[loopStepNumber] = {
-            currentItem: item[index],
-          }
-
-          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
-          // Pretty hacky because we need to account for the row object
-          for (let [key, value] of Object.entries(originalStepInput)) {
-            if (typeof value === "object") {
-              for (let [innerKey, innerValue] of Object.entries(
-                originalStepInput[key]
-              )) {
-                if (typeof innerValue === "string") {
-                  originalStepInput[key][innerKey] =
-                    automationUtils.substituteLoopStep(
-                      innerValue,
-                      `steps.${loopStepNumber}`
-                    )
-                } else if (typeof value === "object") {
-                  for (let [innerObject, innerValue] of Object.entries(
-                    originalStepInput[key][innerKey]
-                  )) {
-                    originalStepInput[key][innerKey][innerObject] =
-                      automationUtils.substituteLoopStep(
-                        innerValue as string,
-                        `steps.${loopStepNumber}`
-                      )
-                  }
-                }
-              }
-            } else {
-              if (typeof value === "string") {
-                originalStepInput[key] = automationUtils.substituteLoopStep(
-                  value,
-                  `steps.${loopStepNumber}`
-                )
-              }
-            }
-          }
-
-          if (
-            index === env.AUTOMATION_MAX_ITERATIONS ||
-            index === parseInt(loopStep.inputs.iterations)
-          ) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.MAX_ITERATIONS,
-              success: true,
-            })
-            loopSteps = undefined
-            loopStep = undefined
-            break
-          }
-
-          let isFailure = false
-          const currentItem = this._context.steps[loopStepNumber]?.currentItem
-          if (currentItem && typeof currentItem === "object") {
-            isFailure = Object.keys(currentItem).some(value => {
-              return currentItem[value] === loopStep?.inputs.failure
-            })
-          } else {
-            isFailure = currentItem && currentItem === loopStep.inputs.failure
-          }
-
-          if (isFailure) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.FAILURE_CONDITION,
-              success: false,
-            })
-            loopSteps = undefined
-            loopStep = undefined
-            break
+        let stepCount = 0
+        let loopStepNumber: any = undefined
+        let loopSteps: LoopStep[] | undefined = []
+        let metadata
+        let timeoutFlag = false
+        let wasLoopStep = false
+        let timeout = this._job.data.event.timeout
+        // check if this is a recurring automation,
+        if (isProdAppID(this._appId) && isRecurring(automation)) {
+          span?.addTags({ recurring: true })
+          metadata = await this.getMetadata()
+          const shouldStop = await this.checkIfShouldStop(metadata)
+          if (shouldStop) {
+            span?.addTags({ shouldStop: true })
+            return
           }
         }
-
-        // execution stopped, record state for that
-        if (stopped) {
-          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
-          continue
-        }
-
-        // If it's a loop step, we need to manually add the bindings to the context
-        let stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await processObject(originalStepInput, this._context)
-        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
-
-        try {
-          // appId is always passed
-          const outputs = await stepFn({
-            inputs: inputs,
-            appId: this._appId,
-            emitter: this._emitter,
-            context: this._context,
+        const start = performance.now()
+        for (let step of automation.definition.steps) {
+          const stepSpan = tracer.startSpan("Orchestrator.execute.step", {
+            childOf: span,
+          })
+          stepSpan.addTags({
+            resource: "automation",
+            step: {
+              stepId: step.stepId,
+              id: step.id,
+              name: step.name,
+              type: step.type,
+              title: step.stepTitle,
+              internal: step.internal,
+              deprecated: step.deprecated,
+            },
           })
 
-          this._context.steps[stepCount] = outputs
-          // if filter causes us to stop execution don't break the loop, set a var
-          // so that we can finish iterating through the steps and record that it stopped
-          if (step.stepId === FILTER_STEP_ID && !outputs.result) {
-            stopped = true
-            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-              ...outputs,
-              ...STOPPED_STATUS,
-            })
-            continue
-          }
-          if (loopStep && loopSteps) {
-            loopSteps.push(outputs)
-          } else {
-            this.updateExecutionOutput(
-              step.id,
-              step.stepId,
-              step.inputs,
-              outputs
-            )
-          }
-        } catch (err) {
-          console.error(`Automation error - ${step.stepId} - ${err}`)
-          return err
-        }
+          let input: any,
+            iterations = 1,
+            iterationCount = 0
 
-        if (loopStep) {
-          iterationCount++
-          if (index === iterations - 1) {
+          try {
+            if (timeoutFlag) {
+              span?.addTags({ timedOut: true })
+              break
+            }
+
+            if (timeout) {
+              setTimeout(() => {
+                timeoutFlag = true
+              }, timeout || 12000)
+            }
+
+            stepCount++
+            if (step.stepId === LOOP_STEP_ID) {
+              loopStep = step
+              loopStepNumber = stepCount
+              continue
+            }
+
+            if (loopStep) {
+              input = await processObject(loopStep.inputs, this._context)
+              iterations = getLoopIterations(loopStep as LoopStep)
+              stepSpan?.addTags({ step: { iterations } })
+            }
+            for (let index = 0; index < iterations; index++) {
+              let originalStepInput = cloneDeep(step.inputs)
+              // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+              if (loopStep && input.binding) {
+                let tempOutput = {
+                  items: loopSteps,
+                  iterations: iterationCount,
+                }
+                try {
+                  loopStep.inputs.binding = automationUtils.typecastForLooping(
+                    loopStep as LoopStep,
+                    loopStep.inputs as LoopInput
+                  )
+                } catch (err) {
+                  this.updateContextAndOutput(
+                    loopStepNumber,
+                    step,
+                    tempOutput,
+                    {
+                      status: AutomationErrors.INCORRECT_TYPE,
+                      success: false,
+                    }
+                  )
+                  loopSteps = undefined
+                  loopStep = undefined
+                  break
+                }
+                let item = []
+                if (
+                  typeof loopStep.inputs.binding === "string" &&
+                  loopStep.inputs.option === "String"
+                ) {
+                  item = automationUtils.stringSplit(loopStep.inputs.binding)
+                } else if (Array.isArray(loopStep.inputs.binding)) {
+                  item = loopStep.inputs.binding
+                }
+                this._context.steps[loopStepNumber] = {
+                  currentItem: item[index],
+                }
+
+                // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+                // Pretty hacky because we need to account for the row object
+                for (let [key, value] of Object.entries(originalStepInput)) {
+                  if (typeof value === "object") {
+                    for (let [innerKey, innerValue] of Object.entries(
+                      originalStepInput[key]
+                    )) {
+                      if (typeof innerValue === "string") {
+                        originalStepInput[key][innerKey] =
+                          automationUtils.substituteLoopStep(
+                            innerValue,
+                            `steps.${loopStepNumber}`
+                          )
+                      } else if (typeof value === "object") {
+                        for (let [innerObject, innerValue] of Object.entries(
+                          originalStepInput[key][innerKey]
+                        )) {
+                          originalStepInput[key][innerKey][innerObject] =
+                            automationUtils.substituteLoopStep(
+                              innerValue as string,
+                              `steps.${loopStepNumber}`
+                            )
+                        }
+                      }
+                    }
+                  } else {
+                    if (typeof value === "string") {
+                      originalStepInput[key] =
+                        automationUtils.substituteLoopStep(
+                          value,
+                          `steps.${loopStepNumber}`
+                        )
+                    }
+                  }
+                }
+
+                if (
+                  index === env.AUTOMATION_MAX_ITERATIONS ||
+                  index === parseInt(loopStep.inputs.iterations)
+                ) {
+                  this.updateContextAndOutput(
+                    loopStepNumber,
+                    step,
+                    tempOutput,
+                    {
+                      status: AutomationErrors.MAX_ITERATIONS,
+                      success: true,
+                    }
+                  )
+                  loopSteps = undefined
+                  loopStep = undefined
+                  break
+                }
+
+                let isFailure = false
+                const currentItem =
+                  this._context.steps[loopStepNumber]?.currentItem
+                if (currentItem && typeof currentItem === "object") {
+                  isFailure = Object.keys(currentItem).some(value => {
+                    return currentItem[value] === loopStep?.inputs.failure
+                  })
+                } else {
+                  isFailure =
+                    currentItem && currentItem === loopStep.inputs.failure
+                }
+
+                if (isFailure) {
+                  this.updateContextAndOutput(
+                    loopStepNumber,
+                    step,
+                    tempOutput,
+                    {
+                      status: AutomationErrors.FAILURE_CONDITION,
+                      success: false,
+                    }
+                  )
+                  loopSteps = undefined
+                  loopStep = undefined
+                  break
+                }
+              }
+
+              // execution stopped, record state for that
+              if (stopped) {
+                this.updateExecutionOutput(
+                  step.id,
+                  step.stepId,
+                  {},
+                  STOPPED_STATUS
+                )
+                continue
+              }
+
+              // If it's a loop step, we need to manually add the bindings to the context
+              let stepFn = await this.getStepFunctionality(step.stepId)
+              let inputs = await processObject(originalStepInput, this._context)
+              inputs = automationUtils.cleanInputValues(
+                inputs,
+                step.schema.inputs
+              )
+
+              try {
+                // appId is always passed
+                const outputs = await stepFn({
+                  inputs: inputs,
+                  appId: this._appId,
+                  emitter: this._emitter,
+                  context: this._context,
+                })
+
+                this._context.steps[stepCount] = outputs
+                // if filter causes us to stop execution don't break the loop, set a var
+                // so that we can finish iterating through the steps and record that it stopped
+                if (step.stepId === FILTER_STEP_ID && !outputs.result) {
+                  stopped = true
+                  this.updateExecutionOutput(
+                    step.id,
+                    step.stepId,
+                    step.inputs,
+                    {
+                      ...outputs,
+                      ...STOPPED_STATUS,
+                    }
+                  )
+                  continue
+                }
+                if (loopStep && loopSteps) {
+                  loopSteps.push(outputs)
+                } else {
+                  this.updateExecutionOutput(
+                    step.id,
+                    step.stepId,
+                    step.inputs,
+                    outputs
+                  )
+                }
+              } catch (err) {
+                console.error(`Automation error - ${step.stepId} - ${err}`)
+                return err
+              }
+
+              if (loopStep) {
+                iterationCount++
+                if (index === iterations - 1) {
+                  loopStep = undefined
+                  this._context.steps.splice(loopStepNumber, 1)
+                  break
+                }
+              }
+            }
+          } finally {
+            stepSpan?.finish()
+          }
+
+          if (loopStep && iterations === 0) {
             loopStep = undefined
+            this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+              id: step.id,
+              stepId: step.stepId,
+              outputs: {
+                status: AutomationStepStatus.NO_ITERATIONS,
+                success: true,
+              },
+              inputs: {},
+            })
+
             this._context.steps.splice(loopStepNumber, 1)
-            break
+            iterations = 1
+          }
+
+          // Delete the step after the loop step as it's irrelevant, since information is included
+          // in the loop step
+          if (wasLoopStep && !loopStep) {
+            this._context.steps.splice(loopStepNumber + 1, 1)
+            wasLoopStep = false
+          }
+          if (loopSteps && loopSteps.length) {
+            let tempOutput = {
+              success: true,
+              items: loopSteps,
+              iterations: iterationCount,
+            }
+            this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+              id: step.id,
+              stepId: step.stepId,
+              outputs: tempOutput,
+              inputs: step.inputs,
+            })
+            this._context.steps[loopStepNumber] = tempOutput
+
+            wasLoopStep = true
+            loopSteps = []
           }
         }
-      }
 
-      if (loopStep && iterations === 0) {
-        loopStep = undefined
-        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-          id: step.id,
-          stepId: step.stepId,
-          outputs: {
-            status: AutomationStepStatus.NO_ITERATIONS,
-            success: true,
-          },
-          inputs: {},
-        })
+        const end = performance.now()
+        const executionTime = end - start
 
-        this._context.steps.splice(loopStepNumber, 1)
-        iterations = 1
-      }
+        console.info(
+          `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
+          {
+            _logKey: "automation",
+            executionTime,
+          }
+        )
 
-      // Delete the step after the loop step as it's irrelevant, since information is included
-      // in the loop step
-      if (wasLoopStep && !loopStep) {
-        this._context.steps.splice(loopStepNumber + 1, 1)
-        wasLoopStep = false
-      }
-      if (loopSteps && loopSteps.length) {
-        let tempOutput = {
-          success: true,
-          items: loopSteps,
-          iterations: iterationCount,
+        // store the logs for the automation run
+        try {
+          await storeLog(this._automation, this.executionOutput)
+        } catch (e: any) {
+          if (e.status === 413 && e.request?.data) {
+            // if content is too large we shouldn't log it
+            delete e.request.data
+            e.request.data = { message: "removed due to large size" }
+          }
+          logging.logAlert("Error writing automation log", e)
         }
-        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-          id: step.id,
-          stepId: step.stepId,
-          outputs: tempOutput,
-          inputs: step.inputs,
-        })
-        this._context.steps[loopStepNumber] = tempOutput
-
-        wasLoopStep = true
-        loopSteps = []
-      }
-    }
-
-    const end = performance.now()
-    const executionTime = end - start
-
-    console.info(
-      `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
-      {
-        _logKey: "automation",
-        executionTime,
+        if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
+          await this.updateMetadata(metadata)
+        }
+        return this.executionOutput
       }
     )
-
-    // store the logs for the automation run
-    try {
-      await storeLog(this._automation, this.executionOutput)
-    } catch (e: any) {
-      if (e.status === 413 && e.request?.data) {
-        // if content is too large we shouldn't log it
-        delete e.request.data
-        e.request.data = { message: "removed due to large size" }
-      }
-      logging.logAlert("Error writing automation log", e)
-    }
-    if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
-      await this.updateMetadata(metadata)
-    }
-    return this.executionOutput
   }
 }
 

From fef3107c0e75c89a145f38bd4b3522fd33980d71 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 19 Dec 2023 16:17:34 +0000
Subject: [PATCH 02/68] Fix automation tests.

---
 packages/backend-core/src/queue/inMemoryQueue.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts
index ac7cdf550b..c05bbffbe9 100644
--- a/packages/backend-core/src/queue/inMemoryQueue.ts
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@@ -15,6 +15,7 @@ function newJob(queue: string, message: any) {
     timestamp: Date.now(),
     queue: queue,
     data: message,
+    opts: {},
   }
 }
 

From c85b8f6a9627381b0f5728a319796f2bab5222f1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 19 Dec 2023 17:13:15 +0000
Subject: [PATCH 03/68] Baggage items don't work how I expected. Use tags
 again.

---
 packages/server/src/middleware/currentapp.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts
index 818c0ab171..ad6f2afa18 100644
--- a/packages/server/src/middleware/currentapp.ts
+++ b/packages/server/src/middleware/currentapp.ts
@@ -23,7 +23,7 @@ export default async (ctx: UserCtx, next: any) => {
 
   if (requestAppId) {
     const span = tracer.scope().active()
-    span?.setBaggageItem("appId", requestAppId)
+    span?.setTag("appId", requestAppId)
   }
 
   // deny access to application preview
@@ -79,9 +79,9 @@ export default async (ctx: UserCtx, next: any) => {
   if (ctx.user) {
     const span = tracer.scope().active()
     if (ctx.user._id) {
-      span?.setBaggageItem("userId", ctx.user._id)
+      span?.setTag("userId", ctx.user._id)
     }
-    span?.setBaggageItem("tenantId", ctx.user.tenantId)
+    span?.setTag("tenantId", ctx.user.tenantId)
   }
 
   const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined

From 2a0be7c8cdaa715ae858ce2ecd1268544bc1182d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 19 Dec 2023 17:18:40 +0000
Subject: [PATCH 04/68] Give db operation spans better names.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts |  2 +-
 packages/backend-core/src/db/db.ts            |  2 +-
 .../backend-core/src/db/instrumentation.ts    | 37 +++++++++----------
 3 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 45aefc36f7..3fec573bb9 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -37,7 +37,7 @@ export function DatabaseWithConnection(
   opts?: DatabaseOpts
 ) {
   const db = new DatabaseImpl(dbName, opts, connection)
-  return new DDInstrumentedDatabase(db, "couchdb")
+  return new DDInstrumentedDatabase(db)
 }
 
 export class DatabaseImpl implements Database {
diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts
index 3d0f522139..197770298e 100644
--- a/packages/backend-core/src/db/db.ts
+++ b/packages/backend-core/src/db/db.ts
@@ -3,7 +3,7 @@ import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types"
 import { DDInstrumentedDatabase } from "./instrumentation"
 
 export function getDB(dbName: string, opts?: DatabaseOpts): Database {
-  return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts), "couchdb")
+  return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts))
 }
 
 // we have to use a callback for this so that we can close
diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
index 6d34478952..ba5febcba6 100644
--- a/packages/backend-core/src/db/instrumentation.ts
+++ b/packages/backend-core/src/db/instrumentation.ts
@@ -18,31 +18,28 @@ import tracer from "dd-trace"
 import { Writable } from "stream"
 
 export class DDInstrumentedDatabase implements Database {
-  constructor(
-    private readonly db: Database,
-    private readonly resource: string
-  ) {}
+  constructor(private readonly db: Database) {}
 
   get name(): string {
     return this.db.name
   }
 
   exists(): Promise<boolean> {
-    return tracer.trace("exists", { resource: this.resource }, span => {
+    return tracer.trace("db.exists", span => {
       span?.addTags({ db_name: this.name })
       return this.db.exists()
     })
   }
 
   checkSetup(): Promise<DocumentScope<any>> {
-    return tracer.trace("checkSetup", { resource: this.resource }, span => {
+    return tracer.trace("db.checkSetup", span => {
       span?.addTags({ db_name: this.name })
       return this.db.checkSetup()
     })
   }
 
   get<T extends Document>(id?: string | undefined): Promise<T> {
-    return tracer.trace("get", { resource: this.resource }, span => {
+    return tracer.trace("db.get", span => {
       span?.addTags({ db_name: this.name, doc_id: id })
       return this.db.get(id)
     })
@@ -52,7 +49,7 @@ export class DDInstrumentedDatabase implements Database {
     ids: string[],
     opts?: { allowMissing?: boolean | undefined } | undefined
   ): Promise<T[]> {
-    return tracer.trace("getMultiple", { resource: this.resource }, span => {
+    return tracer.trace("db.getMultiple", span => {
       span?.addTags({
         db_name: this.name,
         num_docs: ids.length,
@@ -66,7 +63,7 @@ export class DDInstrumentedDatabase implements Database {
     id: string | Document,
     rev?: string | undefined
   ): Promise<DocumentDestroyResponse> {
-    return tracer.trace("remove", { resource: this.resource }, span => {
+    return tracer.trace("db.remove", span => {
       span?.addTags({ db_name: this.name, doc_id: id })
       return this.db.remove(id, rev)
     })
@@ -76,14 +73,14 @@ export class DDInstrumentedDatabase implements Database {
     document: AnyDocument,
     opts?: DatabasePutOpts | undefined
   ): Promise<DocumentInsertResponse> {
-    return tracer.trace("put", { resource: this.resource }, span => {
+    return tracer.trace("db.put", span => {
       span?.addTags({ db_name: this.name, doc_id: document._id })
       return this.db.put(document, opts)
     })
   }
 
   bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
-    return tracer.trace("bulkDocs", { resource: this.resource }, span => {
+    return tracer.trace("db.bulkDocs", span => {
       span?.addTags({ db_name: this.name, num_docs: documents.length })
       return this.db.bulkDocs(documents)
     })
@@ -92,7 +89,7 @@ export class DDInstrumentedDatabase implements Database {
   allDocs<T extends Document>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("allDocs", { resource: this.resource }, span => {
+    return tracer.trace("db.allDocs", span => {
       span?.addTags({ db_name: this.name })
       return this.db.allDocs(params)
     })
@@ -102,56 +99,56 @@ export class DDInstrumentedDatabase implements Database {
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("query", { resource: this.resource }, span => {
+    return tracer.trace("db.query", span => {
       span?.addTags({ db_name: this.name, view_name: viewName })
       return this.db.query(viewName, params)
     })
   }
 
   destroy(): Promise<void | OkResponse> {
-    return tracer.trace("destroy", { resource: this.resource }, span => {
+    return tracer.trace("db.destroy", span => {
       span?.addTags({ db_name: this.name })
       return this.db.destroy()
     })
   }
 
   compact(): Promise<void | OkResponse> {
-    return tracer.trace("compact", { resource: this.resource }, span => {
+    return tracer.trace("db.compact", span => {
       span?.addTags({ db_name: this.name })
       return this.db.compact()
     })
   }
 
   dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
-    return tracer.trace("dump", { resource: this.resource }, span => {
+    return tracer.trace("db.dump", span => {
       span?.addTags({ db_name: this.name })
       return this.db.dump(stream, opts)
     })
   }
 
   load(...args: any[]): Promise<any> {
-    return tracer.trace("load", { resource: this.resource }, span => {
+    return tracer.trace("db.load", span => {
       span?.addTags({ db_name: this.name })
       return this.db.load(...args)
     })
   }
 
   createIndex(...args: any[]): Promise<any> {
-    return tracer.trace("createIndex", { resource: this.resource }, span => {
+    return tracer.trace("db.createIndex", span => {
       span?.addTags({ db_name: this.name })
       return this.db.createIndex(...args)
     })
   }
 
   deleteIndex(...args: any[]): Promise<any> {
-    return tracer.trace("deleteIndex", { resource: this.resource }, span => {
+    return tracer.trace("db.deleteIndex", span => {
       span?.addTags({ db_name: this.name })
       return this.db.deleteIndex(...args)
     })
   }
 
   getIndexes(...args: any[]): Promise<any> {
-    return tracer.trace("getIndexes", { resource: this.resource }, span => {
+    return tracer.trace("db.getIndexes", span => {
       span?.addTags({ db_name: this.name })
       return this.db.getIndexes(...args)
     })

From 2e58f2cdde7c4c053407fc48c0815b1edc420e4f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 19 Dec 2023 18:20:13 +0000
Subject: [PATCH 05/68] Add traces to track running arbitrary JS.

---
 packages/backend-core/src/timers/timers.ts |  6 ++-
 packages/server/src/jsRunner.ts            | 53 +++++++++++++---------
 2 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/packages/backend-core/src/timers/timers.ts b/packages/backend-core/src/timers/timers.ts
index 3bb556703f..9de57af7f1 100644
--- a/packages/backend-core/src/timers/timers.ts
+++ b/packages/backend-core/src/timers/timers.ts
@@ -30,7 +30,7 @@ export class ExecutionTimeTracker {
     return new ExecutionTimeTracker(limitMs)
   }
 
-  constructor(private limitMs: number) {}
+  constructor(readonly limitMs: number) {}
 
   private totalTimeMs = 0
 
@@ -46,6 +46,10 @@ export class ExecutionTimeTracker {
     }
   }
 
+  get elapsedMS() {
+    return this.totalTimeMs
+  }
+
   private checkLimit() {
     if (this.totalTimeMs > this.limitMs) {
       throw new ExecutionTimeoutError(
diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts
index a9301feb60..ab0381a399 100644
--- a/packages/server/src/jsRunner.ts
+++ b/packages/server/src/jsRunner.ts
@@ -2,35 +2,44 @@ import vm from "vm"
 import env from "./environment"
 import { setJSRunner } from "@budibase/string-templates"
 import { context, timers } from "@budibase/backend-core"
+import tracer from "dd-trace"
 
 type TrackerFn = <T>(f: () => T) => T
 
 export function init() {
   setJSRunner((js: string, ctx: vm.Context) => {
-    const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
-    let track: TrackerFn = f => f()
-    if (perRequestLimit) {
-      const bbCtx = context.getCurrentContext()
-      if (bbCtx) {
-        if (!bbCtx.jsExecutionTracker) {
-          bbCtx.jsExecutionTracker =
-            timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+    return tracer.trace("runJS", {}, span => {
+      const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
+      let track: TrackerFn = f => f()
+      if (perRequestLimit) {
+        const bbCtx = context.getCurrentContext()
+        if (bbCtx) {
+          if (!bbCtx.jsExecutionTracker) {
+            bbCtx.jsExecutionTracker =
+              timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+          }
+          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
+          span?.addTags({
+            js: {
+              limitMS: bbCtx.jsExecutionTracker.limitMs,
+              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+            },
+          })
         }
-        track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
       }
-    }
 
-    ctx = {
-      ...ctx,
-      alert: undefined,
-      setInterval: undefined,
-      setTimeout: undefined,
-    }
-    vm.createContext(ctx)
-    return track(() =>
-      vm.runInNewContext(js, ctx, {
-        timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-      })
-    )
+      ctx = {
+        ...ctx,
+        alert: undefined,
+        setInterval: undefined,
+        setTimeout: undefined,
+      }
+      vm.createContext(ctx)
+      return track(() =>
+        vm.runInNewContext(js, ctx, {
+          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+        })
+      )
+    })
   })
 }

From 07203466bd7ccb1ca5ef017dfd03634690e62b98 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Wed, 20 Dec 2023 09:59:51 +0100
Subject: [PATCH 06/68] Upgrade account portal submodule

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 09dae295e3..d8fbdb956a 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 09dae295e3ba6149c4e1d7fe567870c3a38bd277
+Subproject commit d8fbdb956ae0c4119da5bf44b867ff25252073f3

From 753d95e2b40fb1204cf52ce6cae6554757829aa9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 09:55:30 +0000
Subject: [PATCH 07/68] Fix the command and args values in the Helm chart, I
 put them in the wrong place.

---
 .../templates/app-service-deployment.yaml        | 16 ++++++++--------
 .../automation-worker-service-deployment.yaml    | 14 +++++++-------
 .../templates/worker-service-deployment.yaml     | 14 +++++++-------
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 9ed863a611..c6ded3cee2 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -227,6 +227,14 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.apps.command }}
+        command:
+        {{- toYaml .Values.services.apps.command | nindent 10 }}
+        {{ end }}
+        {{ if .Values.services.apps.args }}
+        args:
+        {{- toYaml .Values.services.apps.args | nindent 10 }}
+        {{ end }}
     {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
@@ -244,12 +252,4 @@ spec:
       {{ end }}
       restartPolicy: Always
       serviceAccountName: ""
-      {{ if .Values.services.apps.command }}
-      command:
-      {{- toYaml .Values.services.apps.command | nindent 8 }}
-      {{ end }}
-      {{ if .Values.services.apps.args }}
-      args:
-      {{- toYaml .Values.services.apps.args | nindent 8 }}
-      {{ end }}
 status: {}
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index 6032f33bed..b7eece6b85 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -227,6 +227,13 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        command:
+        {{- toYaml .Values.services.automationWorkers.command | nindent 10 }}
+        {{ end }}
+        {{ if .Values.services.automationWorkers.args }}
+        args:
+        {{- toYaml .Values.services.automationWorkers.args | nindent 10 }}
+        {{ end }}
     {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
@@ -245,12 +252,5 @@ spec:
       restartPolicy: Always
       serviceAccountName: ""
       {{ if .Values.services.automationWorkers.command }}}
-      command:
-      {{- toYaml .Values.services.automationWorkers.command | nindent 8 }}
-      {{ end }}
-      {{ if .Values.services.automationWorkers.args }}
-      args:
-      {{- toYaml .Values.services.automationWorkers.args | nindent 8 }}
-      {{ end }}
 status: {}
 {{- end }}
\ No newline at end of file
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index 32ffb46a90..d97170f011 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -213,6 +213,13 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        command:
+        {{- toYaml .Values.services.worker.command | nindent 10 }}
+        {{ end }}
+        {{ if .Values.services.worker.args }}
+        args:
+        {{- toYaml .Values.services.worker.args | nindent 10 }}
+        {{ end }}
     {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
@@ -231,11 +238,4 @@ spec:
       restartPolicy: Always
       serviceAccountName: ""
       {{ if .Values.services.worker.command }}
-      command:
-      {{- toYaml .Values.services.worker.command | nindent 8 }}
-      {{ end }}
-      {{ if .Values.services.worker.args }}
-      args:
-      {{- toYaml .Values.services.worker.args | nindent 8 }}
-      {{ end }}
 status: {}

From be29bd9b0237a939ab73850cbe66726ae38c7e6e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 10:01:57 +0000
Subject: [PATCH 08/68] Update account-portal submodule to latest master.

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 09dae295e3..abf2d7804c 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 09dae295e3ba6149c4e1d7fe567870c3a38bd277
+Subproject commit abf2d7804c940b011328bb0979ebc6261420fe85

From 7a21576e74076b5930c6bd5994695e433630b269 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 10:18:11 +0000
Subject: [PATCH 09/68] Copy paste error in the Helm chart.

---
 charts/budibase/templates/worker-service-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index d97170f011..04791df869 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -213,6 +213,7 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.worker.command }}
         command:
         {{- toYaml .Values.services.worker.command | nindent 10 }}
         {{ end }}
@@ -237,5 +238,4 @@ spec:
       {{ end }}
       restartPolicy: Always
       serviceAccountName: ""
-      {{ if .Values.services.worker.command }}
 status: {}

From 70e5b1c551486bbf2625b102d7bafb980129568a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 10:22:50 +0000
Subject: [PATCH 10/68] Run helm lint during CI checks.

---
 .github/workflows/budibase_ci.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 14cc110214..39a79193d3 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -76,6 +76,18 @@ jobs:
             yarn check:types
           fi
 
+  helm-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: azure/setup-helm@v3
+      - run: cd charts/budibase && helm lint .
+
   test-libraries:
     runs-on: ubuntu-latest
     steps:

From f0a789ca5b4b14507a3982b76c91d08e401f8c4c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 11:52:05 +0100
Subject: [PATCH 11/68] Run yarn audit

---
 qa-core/yarn.lock | 1026 ++++++++++++++++++++-------------------------
 yarn.lock         |  195 ++++-----
 2 files changed, 545 insertions(+), 676 deletions(-)

diff --git a/qa-core/yarn.lock b/qa-core/yarn.lock
index 1de9d75e60..2c7d2a5f4d 100644
--- a/qa-core/yarn.lock
+++ b/qa-core/yarn.lock
@@ -10,19 +10,13 @@
     "@jridgewell/gen-mapping" "^0.1.0"
     "@jridgewell/trace-mapping" "^0.3.9"
 
-"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz"
-  integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==
+"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.21.4", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5":
+  version "7.23.5"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244"
+  integrity sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==
   dependencies:
-    "@babel/highlight" "^7.18.6"
-
-"@babel/code-frame@^7.21.4":
-  version "7.21.4"
-  resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.21.4.tgz"
-  integrity sha512-LYvhNKfwWSPpocw8GI7gpK2nq3HSDuEPC/uSYaALSJu9xjsalaaYFOq0Pwt5KmVqwEbZlDu81aLXwBOmD/Fv9g==
-  dependencies:
-    "@babel/highlight" "^7.18.6"
+    "@babel/highlight" "^7.23.4"
+    chalk "^2.4.2"
 
 "@babel/compat-data@^7.21.4":
   version "7.21.4"
@@ -50,12 +44,12 @@
     json5 "^2.2.2"
     semver "^6.3.0"
 
-"@babel/generator@^7.21.4", "@babel/generator@^7.7.2":
-  version "7.21.4"
-  resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.21.4.tgz"
-  integrity sha512-NieM3pVIYW2SwGzKoqfPrQsf4xGs9M9AIG3ThppsSRmO+m7eQhmI6amajKMUeIO37wFfsvnvcxQFx6x6iqxDnA==
+"@babel/generator@^7.21.4", "@babel/generator@^7.23.6", "@babel/generator@^7.7.2":
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.6.tgz#9e1fca4811c77a10580d17d26b57b036133f3c2e"
+  integrity sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==
   dependencies:
-    "@babel/types" "^7.21.4"
+    "@babel/types" "^7.23.6"
     "@jridgewell/gen-mapping" "^0.3.2"
     "@jridgewell/trace-mapping" "^0.3.17"
     jsesc "^2.5.1"
@@ -71,25 +65,25 @@
     lru-cache "^5.1.1"
     semver "^6.3.0"
 
-"@babel/helper-environment-visitor@^7.18.9":
-  version "7.18.9"
-  resolved "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz"
-  integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==
+"@babel/helper-environment-visitor@^7.18.9", "@babel/helper-environment-visitor@^7.22.20":
+  version "7.22.20"
+  resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167"
+  integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==
 
-"@babel/helper-function-name@^7.21.0":
-  version "7.21.0"
-  resolved "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.21.0.tgz"
-  integrity sha512-HfK1aMRanKHpxemaY2gqBmL04iAPOPRj7DxtNbiDOrJK+gdwkiNRVpCpUJYbUT+aZyemKN8brqTOxzCaG6ExRg==
+"@babel/helper-function-name@^7.23.0":
+  version "7.23.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759"
+  integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==
   dependencies:
-    "@babel/template" "^7.20.7"
-    "@babel/types" "^7.21.0"
+    "@babel/template" "^7.22.15"
+    "@babel/types" "^7.23.0"
 
-"@babel/helper-hoist-variables@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz"
-  integrity sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==
+"@babel/helper-hoist-variables@^7.22.5":
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb"
+  integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==
   dependencies:
-    "@babel/types" "^7.18.6"
+    "@babel/types" "^7.22.5"
 
 "@babel/helper-module-imports@^7.18.6":
   version "7.21.4"
@@ -124,22 +118,22 @@
   dependencies:
     "@babel/types" "^7.20.2"
 
-"@babel/helper-split-export-declaration@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz"
-  integrity sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==
+"@babel/helper-split-export-declaration@^7.18.6", "@babel/helper-split-export-declaration@^7.22.6":
+  version "7.22.6"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c"
+  integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==
   dependencies:
-    "@babel/types" "^7.18.6"
+    "@babel/types" "^7.22.5"
 
-"@babel/helper-string-parser@^7.19.4":
-  version "7.19.4"
-  resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz"
-  integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==
+"@babel/helper-string-parser@^7.23.4":
+  version "7.23.4"
+  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83"
+  integrity sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==
 
-"@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1":
-  version "7.19.1"
-  resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz"
-  integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==
+"@babel/helper-validator-identifier@^7.19.1", "@babel/helper-validator-identifier@^7.22.20":
+  version "7.22.20"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0"
+  integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==
 
 "@babel/helper-validator-option@^7.21.0":
   version "7.21.0"
@@ -155,19 +149,19 @@
     "@babel/traverse" "^7.21.0"
     "@babel/types" "^7.21.0"
 
-"@babel/highlight@^7.18.6":
-  version "7.18.6"
-  resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz"
-  integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==
+"@babel/highlight@^7.23.4":
+  version "7.23.4"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.23.4.tgz#edaadf4d8232e1a961432db785091207ead0621b"
+  integrity sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==
   dependencies:
-    "@babel/helper-validator-identifier" "^7.18.6"
-    chalk "^2.0.0"
+    "@babel/helper-validator-identifier" "^7.22.20"
+    chalk "^2.4.2"
     js-tokens "^4.0.0"
 
-"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.21.4":
-  version "7.21.4"
-  resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.21.4.tgz"
-  integrity sha512-alVJj7k7zIxqBZ7BTRhz0IqJFxW1VJbm6N8JbcYhQ186df9ZBPbZBmWSqAMXwHGsCJdYks7z/voa3ibiS5bCIw==
+"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.21.4", "@babel/parser@^7.22.15", "@babel/parser@^7.23.6":
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.6.tgz#ba1c9e512bda72a47e285ae42aff9d2a635a9e3b"
+  integrity sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==
 
 "@babel/plugin-syntax-async-generators@^7.8.4":
   version "7.8.4"
@@ -274,38 +268,38 @@
   dependencies:
     regenerator-runtime "^0.13.11"
 
-"@babel/template@^7.20.7", "@babel/template@^7.3.3":
-  version "7.20.7"
-  resolved "https://registry.npmjs.org/@babel/template/-/template-7.20.7.tgz"
-  integrity sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==
+"@babel/template@^7.20.7", "@babel/template@^7.22.15", "@babel/template@^7.3.3":
+  version "7.22.15"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38"
+  integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==
   dependencies:
-    "@babel/code-frame" "^7.18.6"
-    "@babel/parser" "^7.20.7"
-    "@babel/types" "^7.20.7"
+    "@babel/code-frame" "^7.22.13"
+    "@babel/parser" "^7.22.15"
+    "@babel/types" "^7.22.15"
 
 "@babel/traverse@^7.21.0", "@babel/traverse@^7.21.2", "@babel/traverse@^7.21.4":
-  version "7.21.4"
-  resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.21.4.tgz"
-  integrity sha512-eyKrRHKdyZxqDm+fV1iqL9UAHMoIg0nDaGqfIOd8rKH17m5snv7Gn4qgjBoFfLz9APvjFU/ICT00NVCv1Epp8Q==
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.6.tgz#b53526a2367a0dd6edc423637f3d2d0f2521abc5"
+  integrity sha512-czastdK1e8YByZqezMPFiZ8ahwVMh/ESl9vPgvgdB9AmFMGP5jfpFax74AQgl5zj4XHzqeYAg2l8PuUeRS1MgQ==
   dependencies:
-    "@babel/code-frame" "^7.21.4"
-    "@babel/generator" "^7.21.4"
-    "@babel/helper-environment-visitor" "^7.18.9"
-    "@babel/helper-function-name" "^7.21.0"
-    "@babel/helper-hoist-variables" "^7.18.6"
-    "@babel/helper-split-export-declaration" "^7.18.6"
-    "@babel/parser" "^7.21.4"
-    "@babel/types" "^7.21.4"
-    debug "^4.1.0"
+    "@babel/code-frame" "^7.23.5"
+    "@babel/generator" "^7.23.6"
+    "@babel/helper-environment-visitor" "^7.22.20"
+    "@babel/helper-function-name" "^7.23.0"
+    "@babel/helper-hoist-variables" "^7.22.5"
+    "@babel/helper-split-export-declaration" "^7.22.6"
+    "@babel/parser" "^7.23.6"
+    "@babel/types" "^7.23.6"
+    debug "^4.3.1"
     globals "^11.1.0"
 
-"@babel/types@^7.0.0", "@babel/types@^7.18.6", "@babel/types@^7.20.2", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.21.2", "@babel/types@^7.21.4", "@babel/types@^7.3.0", "@babel/types@^7.3.3":
-  version "7.21.4"
-  resolved "https://registry.npmjs.org/@babel/types/-/types-7.21.4.tgz"
-  integrity sha512-rU2oY501qDxE8Pyo7i/Orqma4ziCOrby0/9mvbDUGEfvZjb279Nk9k19e2fiCxHbRRpY2ZyrgW1eq22mvmOIzA==
+"@babel/types@^7.0.0", "@babel/types@^7.20.2", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.21.2", "@babel/types@^7.21.4", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6", "@babel/types@^7.3.0", "@babel/types@^7.3.3":
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd"
+  integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==
   dependencies:
-    "@babel/helper-string-parser" "^7.19.4"
-    "@babel/helper-validator-identifier" "^7.19.1"
+    "@babel/helper-string-parser" "^7.23.4"
+    "@babel/helper-validator-identifier" "^7.22.20"
     to-fast-properties "^2.0.0"
 
 "@balena/dockerignore@^1.0.2":
@@ -409,49 +403,49 @@
   resolved "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz"
   integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==
 
-"@jest/console@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/console/-/console-29.6.2.tgz#bf1d4101347c23e07c029a1b1ae07d550f5cc541"
-  integrity sha512-0N0yZof5hi44HAR2pPS+ikJ3nzKNoZdVu8FffRf3wy47I7Dm7etk/3KetMdRUqzVd16V4O2m2ISpNTbnIuqy1w==
+"@jest/console@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/console/-/console-29.7.0.tgz#cd4822dbdb84529265c5a2bdb529a3c9cc950ffc"
+  integrity sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==
   dependencies:
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     chalk "^4.0.0"
-    jest-message-util "^29.6.2"
-    jest-util "^29.6.2"
+    jest-message-util "^29.7.0"
+    jest-util "^29.7.0"
     slash "^3.0.0"
 
-"@jest/core@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/core/-/core-29.6.2.tgz#6f2d1dbe8aa0265fcd4fb8082ae1952f148209c8"
-  integrity sha512-Oj+5B+sDMiMWLhPFF+4/DvHOf+U10rgvCLGPHP8Xlsy/7QxS51aU/eBngudHlJXnaWD5EohAgJ4js+T6pa+zOg==
+"@jest/core@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/core/-/core-29.7.0.tgz#b6cccc239f30ff36609658c5a5e2291757ce448f"
+  integrity sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==
   dependencies:
-    "@jest/console" "^29.6.2"
-    "@jest/reporters" "^29.6.2"
-    "@jest/test-result" "^29.6.2"
-    "@jest/transform" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/console" "^29.7.0"
+    "@jest/reporters" "^29.7.0"
+    "@jest/test-result" "^29.7.0"
+    "@jest/transform" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     ansi-escapes "^4.2.1"
     chalk "^4.0.0"
     ci-info "^3.2.0"
     exit "^0.1.2"
     graceful-fs "^4.2.9"
-    jest-changed-files "^29.5.0"
-    jest-config "^29.6.2"
-    jest-haste-map "^29.6.2"
-    jest-message-util "^29.6.2"
-    jest-regex-util "^29.4.3"
-    jest-resolve "^29.6.2"
-    jest-resolve-dependencies "^29.6.2"
-    jest-runner "^29.6.2"
-    jest-runtime "^29.6.2"
-    jest-snapshot "^29.6.2"
-    jest-util "^29.6.2"
-    jest-validate "^29.6.2"
-    jest-watcher "^29.6.2"
+    jest-changed-files "^29.7.0"
+    jest-config "^29.7.0"
+    jest-haste-map "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-regex-util "^29.6.3"
+    jest-resolve "^29.7.0"
+    jest-resolve-dependencies "^29.7.0"
+    jest-runner "^29.7.0"
+    jest-runtime "^29.7.0"
+    jest-snapshot "^29.7.0"
+    jest-util "^29.7.0"
+    jest-validate "^29.7.0"
+    jest-watcher "^29.7.0"
     micromatch "^4.0.4"
-    pretty-format "^29.6.2"
+    pretty-format "^29.7.0"
     slash "^3.0.0"
     strip-ansi "^6.0.0"
 
@@ -462,70 +456,63 @@
   dependencies:
     "@jest/types" "^27.5.1"
 
-"@jest/environment@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.6.2.tgz#794c0f769d85e7553439d107d3f43186dc6874a9"
-  integrity sha512-AEcW43C7huGd/vogTddNNTDRpO6vQ2zaQNrttvWV18ArBx9Z56h7BIsXkNFJVOO4/kblWEQz30ckw0+L3izc+Q==
+"@jest/environment@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.7.0.tgz#24d61f54ff1f786f3cd4073b4b94416383baf2a7"
+  integrity sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==
   dependencies:
-    "@jest/fake-timers" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/fake-timers" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
-    jest-mock "^29.6.2"
+    jest-mock "^29.7.0"
 
-"@jest/expect-utils@^29.5.0":
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.5.0.tgz"
-  integrity sha512-fmKzsidoXQT2KwnrwE0SQq3uj8Z763vzR8LnLBwC2qYWEFpjX8daRsk6rHUM1QvNlEW/UJXNXm59ztmJJWs2Mg==
+"@jest/expect-utils@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.7.0.tgz#023efe5d26a8a70f21677d0a1afc0f0a44e3a1c6"
+  integrity sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==
   dependencies:
-    jest-get-type "^29.4.3"
+    jest-get-type "^29.6.3"
 
-"@jest/expect-utils@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.6.2.tgz#1b97f290d0185d264dd9fdec7567a14a38a90534"
-  integrity sha512-6zIhM8go3RV2IG4aIZaZbxwpOzz3ZiM23oxAlkquOIole+G6TrbeXnykxWYlqF7kz2HlBjdKtca20x9atkEQYg==
+"@jest/expect@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-29.7.0.tgz#76a3edb0cb753b70dfbfe23283510d3d45432bf2"
+  integrity sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==
   dependencies:
-    jest-get-type "^29.4.3"
+    expect "^29.7.0"
+    jest-snapshot "^29.7.0"
 
-"@jest/expect@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-29.6.2.tgz#5a2ad58bb345165d9ce0a1845bbf873c480a4b28"
-  integrity sha512-m6DrEJxVKjkELTVAztTLyS/7C92Y2b0VYqmDROYKLLALHn8T/04yPs70NADUYPrV3ruI+H3J0iUIuhkjp7vkfg==
+"@jest/fake-timers@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.7.0.tgz#fd91bf1fffb16d7d0d24a426ab1a47a49881a565"
+  integrity sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==
   dependencies:
-    expect "^29.6.2"
-    jest-snapshot "^29.6.2"
-
-"@jest/fake-timers@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.6.2.tgz#fe9d43c5e4b1b901168fe6f46f861b3e652a2df4"
-  integrity sha512-euZDmIlWjm1Z0lJ1D0f7a0/y5Kh/koLFMUBE5SUYWrmy8oNhJpbTBDAP6CxKnadcMLDoDf4waRYCe35cH6G6PA==
-  dependencies:
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     "@sinonjs/fake-timers" "^10.0.2"
     "@types/node" "*"
-    jest-message-util "^29.6.2"
-    jest-mock "^29.6.2"
-    jest-util "^29.6.2"
+    jest-message-util "^29.7.0"
+    jest-mock "^29.7.0"
+    jest-util "^29.7.0"
 
-"@jest/globals@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-29.6.2.tgz#74af81b9249122cc46f1eb25793617eec69bf21a"
-  integrity sha512-cjuJmNDjs6aMijCmSa1g2TNG4Lby/AeU7/02VtpW+SLcZXzOLK2GpN2nLqcFjmhy3B3AoPeQVx7BnyOf681bAw==
+"@jest/globals@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-29.7.0.tgz#8d9290f9ec47ff772607fa864ca1d5a2efae1d4d"
+  integrity sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==
   dependencies:
-    "@jest/environment" "^29.6.2"
-    "@jest/expect" "^29.6.2"
-    "@jest/types" "^29.6.1"
-    jest-mock "^29.6.2"
+    "@jest/environment" "^29.7.0"
+    "@jest/expect" "^29.7.0"
+    "@jest/types" "^29.6.3"
+    jest-mock "^29.7.0"
 
-"@jest/reporters@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-29.6.2.tgz#524afe1d76da33d31309c2c4a2c8062d0c48780a"
-  integrity sha512-sWtijrvIav8LgfJZlrGCdN0nP2EWbakglJY49J1Y5QihcQLfy7ovyxxjJBRXMNltgt4uPtEcFmIMbVshEDfFWw==
+"@jest/reporters@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-29.7.0.tgz#04b262ecb3b8faa83b0b3d321623972393e8f4c7"
+  integrity sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==
   dependencies:
     "@bcoe/v8-coverage" "^0.2.3"
-    "@jest/console" "^29.6.2"
-    "@jest/test-result" "^29.6.2"
-    "@jest/transform" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/console" "^29.7.0"
+    "@jest/test-result" "^29.7.0"
+    "@jest/transform" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@jridgewell/trace-mapping" "^0.3.18"
     "@types/node" "*"
     chalk "^4.0.0"
@@ -534,77 +521,70 @@
     glob "^7.1.3"
     graceful-fs "^4.2.9"
     istanbul-lib-coverage "^3.0.0"
-    istanbul-lib-instrument "^5.1.0"
+    istanbul-lib-instrument "^6.0.0"
     istanbul-lib-report "^3.0.0"
     istanbul-lib-source-maps "^4.0.0"
     istanbul-reports "^3.1.3"
-    jest-message-util "^29.6.2"
-    jest-util "^29.6.2"
-    jest-worker "^29.6.2"
+    jest-message-util "^29.7.0"
+    jest-util "^29.7.0"
+    jest-worker "^29.7.0"
     slash "^3.0.0"
     string-length "^4.0.1"
     strip-ansi "^6.0.0"
     v8-to-istanbul "^9.0.1"
 
-"@jest/schemas@^29.4.3":
-  version "29.4.3"
-  resolved "https://registry.npmjs.org/@jest/schemas/-/schemas-29.4.3.tgz"
-  integrity sha512-VLYKXQmtmuEz6IxJsrZwzG9NvtkQsWNnWMsKxqWNu3+CnfzJQhp0WDDKWLVV9hLKr0l3SLLFRqcYHjhtyuDVxg==
-  dependencies:
-    "@sinclair/typebox" "^0.25.16"
-
-"@jest/schemas@^29.6.0":
-  version "29.6.0"
-  resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.0.tgz#0f4cb2c8e3dca80c135507ba5635a4fd755b0040"
-  integrity sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ==
+"@jest/schemas@^29.6.3":
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03"
+  integrity sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==
   dependencies:
     "@sinclair/typebox" "^0.27.8"
 
-"@jest/source-map@^29.6.0":
-  version "29.6.0"
-  resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-29.6.0.tgz#bd34a05b5737cb1a99d43e1957020ac8e5b9ddb1"
-  integrity sha512-oA+I2SHHQGxDCZpbrsCQSoMLb3Bz547JnM+jUr9qEbuw0vQlWZfpPS7CO9J7XiwKicEz9OFn/IYoLkkiUD7bzA==
+"@jest/source-map@^29.6.3":
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-29.6.3.tgz#d90ba772095cf37a34a5eb9413f1b562a08554c4"
+  integrity sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==
   dependencies:
     "@jridgewell/trace-mapping" "^0.3.18"
     callsites "^3.0.0"
     graceful-fs "^4.2.9"
 
-"@jest/test-result@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-29.6.2.tgz#fdd11583cd1608e4db3114e8f0cce277bf7a32ed"
-  integrity sha512-3VKFXzcV42EYhMCsJQURptSqnyjqCGbtLuX5Xxb6Pm6gUf1wIRIl+mandIRGJyWKgNKYF9cnstti6Ls5ekduqw==
+"@jest/test-result@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-29.7.0.tgz#8db9a80aa1a097bb2262572686734baed9b1657c"
+  integrity sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==
   dependencies:
-    "@jest/console" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/console" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/istanbul-lib-coverage" "^2.0.0"
     collect-v8-coverage "^1.0.0"
 
-"@jest/test-sequencer@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-29.6.2.tgz#585eff07a68dd75225a7eacf319780cb9f6b9bf4"
-  integrity sha512-GVYi6PfPwVejO7slw6IDO0qKVum5jtrJ3KoLGbgBWyr2qr4GaxFV6su+ZAjdTX75Sr1DkMFRk09r2ZVa+wtCGw==
+"@jest/test-sequencer@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz#6cef977ce1d39834a3aea887a1726628a6f072ce"
+  integrity sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==
   dependencies:
-    "@jest/test-result" "^29.6.2"
+    "@jest/test-result" "^29.7.0"
     graceful-fs "^4.2.9"
-    jest-haste-map "^29.6.2"
+    jest-haste-map "^29.7.0"
     slash "^3.0.0"
 
-"@jest/transform@^29.6.2":
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-29.6.2.tgz#522901ebbb211af08835bc3bcdf765ab778094e3"
-  integrity sha512-ZqCqEISr58Ce3U+buNFJYUktLJZOggfyvR+bZMaiV1e8B1SIvJbwZMrYz3gx/KAPn9EXmOmN+uB08yLCjWkQQg==
+"@jest/transform@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-29.7.0.tgz#df2dd9c346c7d7768b8a06639994640c642e284c"
+  integrity sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==
   dependencies:
     "@babel/core" "^7.11.6"
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     "@jridgewell/trace-mapping" "^0.3.18"
     babel-plugin-istanbul "^6.1.1"
     chalk "^4.0.0"
     convert-source-map "^2.0.0"
     fast-json-stable-stringify "^2.1.0"
     graceful-fs "^4.2.9"
-    jest-haste-map "^29.6.2"
-    jest-regex-util "^29.4.3"
-    jest-util "^29.6.2"
+    jest-haste-map "^29.7.0"
+    jest-regex-util "^29.6.3"
+    jest-util "^29.7.0"
     micromatch "^4.0.4"
     pirates "^4.0.4"
     slash "^3.0.0"
@@ -621,24 +601,12 @@
     "@types/yargs" "^16.0.0"
     chalk "^4.0.0"
 
-"@jest/types@^29.5.0":
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/@jest/types/-/types-29.5.0.tgz"
-  integrity sha512-qbu7kN6czmVRc3xWFQcAN03RAUamgppVUdXrvl1Wr3jlNF93o9mJbGcDWrwGB6ht44u7efB1qCFgVQmca24Uog==
+"@jest/types@^29.6.3":
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59"
+  integrity sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==
   dependencies:
-    "@jest/schemas" "^29.4.3"
-    "@types/istanbul-lib-coverage" "^2.0.0"
-    "@types/istanbul-reports" "^3.0.0"
-    "@types/node" "*"
-    "@types/yargs" "^17.0.8"
-    chalk "^4.0.0"
-
-"@jest/types@^29.6.1":
-  version "29.6.1"
-  resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.1.tgz#ae79080278acff0a6af5eb49d063385aaa897bf2"
-  integrity sha512-tPKQNMPuXgvdOn2/Lg9HNfUvjYVGolt04Hp03f5hAk878uwOLikN+JzeLY0HcVgKgFl9Hs3EIqpu3WX27XNhnw==
-  dependencies:
-    "@jest/schemas" "^29.6.0"
+    "@jest/schemas" "^29.6.3"
     "@types/istanbul-lib-coverage" "^2.0.0"
     "@types/istanbul-reports" "^3.0.0"
     "@types/node" "*"
@@ -685,15 +653,7 @@
     "@jridgewell/resolve-uri" "^3.0.3"
     "@jridgewell/sourcemap-codec" "^1.4.10"
 
-"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9":
-  version "0.3.17"
-  resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz"
-  integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==
-  dependencies:
-    "@jridgewell/resolve-uri" "3.1.0"
-    "@jridgewell/sourcemap-codec" "1.4.14"
-
-"@jridgewell/trace-mapping@^0.3.18":
+"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.18", "@jridgewell/trace-mapping@^0.3.9":
   version "0.3.18"
   resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6"
   integrity sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==
@@ -771,11 +731,6 @@
   resolved "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz"
   integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==
 
-"@sinclair/typebox@^0.25.16":
-  version "0.25.24"
-  resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz"
-  integrity sha512-XJfwUVUKDHF5ugKwIcxEgc9k8b7HbznCp6eUfWgu710hMPNIO4aw4/zB5RogDQz8nd6gyCDpU9O/m6qYEWY6yQ==
-
 "@sinclair/typebox@^0.27.8":
   version "0.27.8"
   resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e"
@@ -1229,23 +1184,23 @@ axios@^0.21.1:
     follow-redirects "^1.14.0"
 
 axios@^1.1.3:
-  version "1.3.4"
-  resolved "https://registry.npmjs.org/axios/-/axios-1.3.4.tgz"
-  integrity sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==
+  version "1.6.2"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.2.tgz#de67d42c755b571d3e698df1b6504cde9b0ee9f2"
+  integrity sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==
   dependencies:
     follow-redirects "^1.15.0"
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
-babel-jest@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.6.2.tgz#cada0a59e07f5acaeb11cbae7e3ba92aec9c1126"
-  integrity sha512-BYCzImLos6J3BH/+HvUCHG1dTf2MzmAB4jaVxHV+29RZLjR29XuYTmsf2sdDwkrb+FczkGo3kOhE7ga6sI0P4A==
+babel-jest@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.7.0.tgz#f4369919225b684c56085998ac63dbd05be020d5"
+  integrity sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==
   dependencies:
-    "@jest/transform" "^29.6.2"
+    "@jest/transform" "^29.7.0"
     "@types/babel__core" "^7.1.14"
     babel-plugin-istanbul "^6.1.1"
-    babel-preset-jest "^29.5.0"
+    babel-preset-jest "^29.6.3"
     chalk "^4.0.0"
     graceful-fs "^4.2.9"
     slash "^3.0.0"
@@ -1261,10 +1216,10 @@ babel-plugin-istanbul@^6.1.1:
     istanbul-lib-instrument "^5.0.4"
     test-exclude "^6.0.0"
 
-babel-plugin-jest-hoist@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.5.0.tgz"
-  integrity sha512-zSuuuAlTMT4mzLj2nPnUm6fsE6270vdOfnpbJ+RmruU75UhLFvL0N2NgI7xpeS7NaB6hGqmd5pVpGTDYvi4Q3w==
+babel-plugin-jest-hoist@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz#aadbe943464182a8922c3c927c3067ff40d24626"
+  integrity sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==
   dependencies:
     "@babel/template" "^7.3.3"
     "@babel/types" "^7.3.3"
@@ -1289,12 +1244,12 @@ babel-preset-current-node-syntax@^1.0.0:
     "@babel/plugin-syntax-optional-chaining" "^7.8.3"
     "@babel/plugin-syntax-top-level-await" "^7.8.3"
 
-babel-preset-jest@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.5.0.tgz"
-  integrity sha512-JOMloxOqdiBSxMAzjRaH023/vvcaSaec49zvg+2LmNsktC7ei39LTJGw02J+9uUtTZUq6xbLyJ4dxe9sSmIuAg==
+babel-preset-jest@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz#fa05fa510e7d493896d7b0dd2033601c840f171c"
+  integrity sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==
   dependencies:
-    babel-plugin-jest-hoist "^29.5.0"
+    babel-plugin-jest-hoist "^29.6.3"
     babel-preset-current-node-syntax "^1.0.0"
 
 balanced-match@^1.0.0:
@@ -1483,7 +1438,7 @@ caseless@~0.12.0:
   resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz"
   integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==
 
-chalk@^2.0.0:
+chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz"
   integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
@@ -1670,6 +1625,19 @@ cpu-features@~0.0.8:
     buildcheck "~0.0.6"
     nan "^2.17.0"
 
+create-jest@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/create-jest/-/create-jest-29.7.0.tgz#a355c5b3cb1e1af02ba177fe7afd7feee49a5320"
+  integrity sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==
+  dependencies:
+    "@jest/types" "^29.6.3"
+    chalk "^4.0.0"
+    exit "^0.1.2"
+    graceful-fs "^4.2.9"
+    jest-config "^29.7.0"
+    jest-util "^29.7.0"
+    prompts "^2.0.1"
+
 create-require@^1.1.0:
   version "1.1.1"
   resolved "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz"
@@ -1793,10 +1761,10 @@ detect-newline@^3.0.0:
   resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz"
   integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==
 
-diff-sequences@^29.4.3:
-  version "29.4.3"
-  resolved "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.4.3.tgz"
-  integrity sha512-ofrBgwpPhCD85kMKtE9RYFFq6OC1A89oW2vvgWZNCwxrUpRUILopY7lsYyMDSjc8g6U6aiO0Qubg6r4Wgt5ZnA==
+diff-sequences@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921"
+  integrity sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==
 
 diff@^4.0.1:
   version "4.0.2"
@@ -2004,28 +1972,16 @@ expand-tilde@^1.2.2:
   dependencies:
     os-homedir "^1.0.1"
 
-expect@^29.0.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/expect/-/expect-29.5.0.tgz"
-  integrity sha512-yM7xqUrCO2JdpFo4XpM82t+PJBFybdqoQuJLDGeDX2ij8NZzqRHyu3Hp188/JX7SWqud+7t4MUdvcgGBICMHZg==
+expect@^29.0.0, expect@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/expect/-/expect-29.7.0.tgz#578874590dcb3214514084c08115d8aee61e11bc"
+  integrity sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==
   dependencies:
-    "@jest/expect-utils" "^29.5.0"
-    jest-get-type "^29.4.3"
-    jest-matcher-utils "^29.5.0"
-    jest-message-util "^29.5.0"
-    jest-util "^29.5.0"
-
-expect@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/expect/-/expect-29.6.2.tgz#7b08e83eba18ddc4a2cf62b5f2d1918f5cd84521"
-  integrity sha512-iAErsLxJ8C+S02QbLAwgSGSezLQK+XXRDt8IuFXFpwCNw2ECmzZSmjKcCaFVp5VRMk+WAvz6h6jokzEzBFZEuA==
-  dependencies:
-    "@jest/expect-utils" "^29.6.2"
-    "@types/node" "*"
-    jest-get-type "^29.4.3"
-    jest-matcher-utils "^29.6.2"
-    jest-message-util "^29.6.2"
-    jest-util "^29.6.2"
+    "@jest/expect-utils" "^29.7.0"
+    jest-get-type "^29.6.3"
+    jest-matcher-utils "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-util "^29.7.0"
 
 extend@~3.0.2:
   version "3.0.2"
@@ -2544,7 +2500,7 @@ istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0:
   resolved "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz"
   integrity sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==
 
-istanbul-lib-instrument@^5.0.4, istanbul-lib-instrument@^5.1.0:
+istanbul-lib-instrument@^5.0.4:
   version "5.2.1"
   resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz"
   integrity sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==
@@ -2555,6 +2511,17 @@ istanbul-lib-instrument@^5.0.4, istanbul-lib-instrument@^5.1.0:
     istanbul-lib-coverage "^3.2.0"
     semver "^6.3.0"
 
+istanbul-lib-instrument@^6.0.0:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.1.tgz#71e87707e8041428732518c6fb5211761753fbdf"
+  integrity sha512-EAMEJBsYuyyztxMxW3g7ugGPkrZsV57v0Hmv3mm1uQsmB+QnZuepg731CRaIgeUVSdmsTngOkSnauNF8p7FIhA==
+  dependencies:
+    "@babel/core" "^7.12.3"
+    "@babel/parser" "^7.14.7"
+    "@istanbuljs/schema" "^0.1.2"
+    istanbul-lib-coverage "^3.2.0"
+    semver "^7.5.4"
+
 istanbul-lib-report@^3.0.0:
   version "3.0.0"
   resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz"
@@ -2581,410 +2548,363 @@ istanbul-reports@^3.1.3:
     html-escaper "^2.0.0"
     istanbul-lib-report "^3.0.0"
 
-jest-changed-files@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.5.0.tgz"
-  integrity sha512-IFG34IUMUaNBIxjQXF/iu7g6EcdMrGRRxaUSw92I/2g2YC6vCdTltl4nHvt7Ci5nSJwXIkCu8Ka1DKF+X7Z1Ag==
+jest-changed-files@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-29.7.0.tgz#1c06d07e77c78e1585d020424dedc10d6e17ac3a"
+  integrity sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==
   dependencies:
     execa "^5.0.0"
+    jest-util "^29.7.0"
     p-limit "^3.1.0"
 
-jest-circus@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-29.6.2.tgz#1e6ffca60151ac66cad63fce34f443f6b5bb4258"
-  integrity sha512-G9mN+KOYIUe2sB9kpJkO9Bk18J4dTDArNFPwoZ7WKHKel55eKIS/u2bLthxgojwlf9NLCVQfgzM/WsOVvoC6Fw==
+jest-circus@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-29.7.0.tgz#b6817a45fcc835d8b16d5962d0c026473ee3668a"
+  integrity sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==
   dependencies:
-    "@jest/environment" "^29.6.2"
-    "@jest/expect" "^29.6.2"
-    "@jest/test-result" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/environment" "^29.7.0"
+    "@jest/expect" "^29.7.0"
+    "@jest/test-result" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     chalk "^4.0.0"
     co "^4.6.0"
     dedent "^1.0.0"
     is-generator-fn "^2.0.0"
-    jest-each "^29.6.2"
-    jest-matcher-utils "^29.6.2"
-    jest-message-util "^29.6.2"
-    jest-runtime "^29.6.2"
-    jest-snapshot "^29.6.2"
-    jest-util "^29.6.2"
+    jest-each "^29.7.0"
+    jest-matcher-utils "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-runtime "^29.7.0"
+    jest-snapshot "^29.7.0"
+    jest-util "^29.7.0"
     p-limit "^3.1.0"
-    pretty-format "^29.6.2"
+    pretty-format "^29.7.0"
     pure-rand "^6.0.0"
     slash "^3.0.0"
     stack-utils "^2.0.3"
 
-jest-cli@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-29.6.2.tgz#edb381763398d1a292cd1b636a98bfa5644b8fda"
-  integrity sha512-TT6O247v6dCEX2UGHGyflMpxhnrL0DNqP2fRTKYm3nJJpCTfXX3GCMQPGFjXDoj0i5/Blp3jriKXFgdfmbYB6Q==
+jest-cli@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-29.7.0.tgz#5592c940798e0cae677eec169264f2d839a37995"
+  integrity sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==
   dependencies:
-    "@jest/core" "^29.6.2"
-    "@jest/test-result" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/core" "^29.7.0"
+    "@jest/test-result" "^29.7.0"
+    "@jest/types" "^29.6.3"
     chalk "^4.0.0"
+    create-jest "^29.7.0"
     exit "^0.1.2"
-    graceful-fs "^4.2.9"
     import-local "^3.0.2"
-    jest-config "^29.6.2"
-    jest-util "^29.6.2"
-    jest-validate "^29.6.2"
-    prompts "^2.0.1"
+    jest-config "^29.7.0"
+    jest-util "^29.7.0"
+    jest-validate "^29.7.0"
     yargs "^17.3.1"
 
-jest-config@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-29.6.2.tgz#c68723f06b31ca5e63030686e604727d406cd7c3"
-  integrity sha512-VxwFOC8gkiJbuodG9CPtMRjBUNZEHxwfQXmIudSTzFWxaci3Qub1ddTRbFNQlD/zUeaifLndh/eDccFX4wCMQw==
+jest-config@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-29.7.0.tgz#bcbda8806dbcc01b1e316a46bb74085a84b0245f"
+  integrity sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==
   dependencies:
     "@babel/core" "^7.11.6"
-    "@jest/test-sequencer" "^29.6.2"
-    "@jest/types" "^29.6.1"
-    babel-jest "^29.6.2"
+    "@jest/test-sequencer" "^29.7.0"
+    "@jest/types" "^29.6.3"
+    babel-jest "^29.7.0"
     chalk "^4.0.0"
     ci-info "^3.2.0"
     deepmerge "^4.2.2"
     glob "^7.1.3"
     graceful-fs "^4.2.9"
-    jest-circus "^29.6.2"
-    jest-environment-node "^29.6.2"
-    jest-get-type "^29.4.3"
-    jest-regex-util "^29.4.3"
-    jest-resolve "^29.6.2"
-    jest-runner "^29.6.2"
-    jest-util "^29.6.2"
-    jest-validate "^29.6.2"
+    jest-circus "^29.7.0"
+    jest-environment-node "^29.7.0"
+    jest-get-type "^29.6.3"
+    jest-regex-util "^29.6.3"
+    jest-resolve "^29.7.0"
+    jest-runner "^29.7.0"
+    jest-util "^29.7.0"
+    jest-validate "^29.7.0"
     micromatch "^4.0.4"
     parse-json "^5.2.0"
-    pretty-format "^29.6.2"
+    pretty-format "^29.7.0"
     slash "^3.0.0"
     strip-json-comments "^3.1.1"
 
-jest-diff@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/jest-diff/-/jest-diff-29.5.0.tgz"
-  integrity sha512-LtxijLLZBduXnHSniy0WMdaHjmQnt3g5sa16W4p0HqukYTTsyTW3GD1q41TyGl5YFXj/5B2U6dlh5FM1LIMgxw==
+jest-diff@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a"
+  integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==
   dependencies:
     chalk "^4.0.0"
-    diff-sequences "^29.4.3"
-    jest-get-type "^29.4.3"
-    pretty-format "^29.5.0"
+    diff-sequences "^29.6.3"
+    jest-get-type "^29.6.3"
+    pretty-format "^29.7.0"
 
-jest-diff@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.6.2.tgz#c36001e5543e82a0805051d3ceac32e6825c1c46"
-  integrity sha512-t+ST7CB9GX5F2xKwhwCf0TAR17uNDiaPTZnVymP9lw0lssa9vG+AFyDZoeIHStU3WowFFwT+ky+er0WVl2yGhA==
-  dependencies:
-    chalk "^4.0.0"
-    diff-sequences "^29.4.3"
-    jest-get-type "^29.4.3"
-    pretty-format "^29.6.2"
-
-jest-docblock@^29.4.3:
-  version "29.4.3"
-  resolved "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.4.3.tgz"
-  integrity sha512-fzdTftThczeSD9nZ3fzA/4KkHtnmllawWrXO69vtI+L9WjEIuXWs4AmyME7lN5hU7dB0sHhuPfcKofRsUb/2Fg==
+jest-docblock@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-29.7.0.tgz#8fddb6adc3cdc955c93e2a87f61cfd350d5d119a"
+  integrity sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==
   dependencies:
     detect-newline "^3.0.0"
 
-jest-each@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.6.2.tgz#c9e4b340bcbe838c73adf46b76817b15712d02ce"
-  integrity sha512-MsrsqA0Ia99cIpABBc3izS1ZYoYfhIy0NNWqPSE0YXbQjwchyt6B1HD2khzyPe1WiJA7hbxXy77ZoUQxn8UlSw==
+jest-each@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.7.0.tgz#162a9b3f2328bdd991beaabffbb74745e56577d1"
+  integrity sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==
   dependencies:
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     chalk "^4.0.0"
-    jest-get-type "^29.4.3"
-    jest-util "^29.6.2"
-    pretty-format "^29.6.2"
+    jest-get-type "^29.6.3"
+    jest-util "^29.7.0"
+    pretty-format "^29.7.0"
 
-jest-environment-node@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.6.2.tgz#a9ea2cabff39b08eca14ccb32c8ceb924c8bb1ad"
-  integrity sha512-YGdFeZ3T9a+/612c5mTQIllvWkddPbYcN2v95ZH24oWMbGA4GGS2XdIF92QMhUhvrjjuQWYgUGW2zawOyH63MQ==
+jest-environment-node@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.7.0.tgz#0b93e111dda8ec120bc8300e6d1fb9576e164376"
+  integrity sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==
   dependencies:
-    "@jest/environment" "^29.6.2"
-    "@jest/fake-timers" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/environment" "^29.7.0"
+    "@jest/fake-timers" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
-    jest-mock "^29.6.2"
-    jest-util "^29.6.2"
+    jest-mock "^29.7.0"
+    jest-util "^29.7.0"
 
-jest-get-type@^29.4.3:
-  version "29.4.3"
-  resolved "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.4.3.tgz"
-  integrity sha512-J5Xez4nRRMjk8emnTpWrlkyb9pfRQQanDrvWHhsR1+VUfbwxi30eVcZFlcdGInRibU4G5LwHXpI7IRHU0CY+gg==
+jest-get-type@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1"
+  integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==
 
-jest-haste-map@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-29.6.2.tgz#298c25ea5255cfad8b723179d4295cf3a50a70d1"
-  integrity sha512-+51XleTDAAysvU8rT6AnS1ZJ+WHVNqhj1k6nTvN2PYP+HjU3kqlaKQ1Lnw3NYW3bm2r8vq82X0Z1nDDHZMzHVA==
+jest-haste-map@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-29.7.0.tgz#3c2396524482f5a0506376e6c858c3bbcc17b104"
+  integrity sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==
   dependencies:
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     "@types/graceful-fs" "^4.1.3"
     "@types/node" "*"
     anymatch "^3.0.3"
     fb-watchman "^2.0.0"
     graceful-fs "^4.2.9"
-    jest-regex-util "^29.4.3"
-    jest-util "^29.6.2"
-    jest-worker "^29.6.2"
+    jest-regex-util "^29.6.3"
+    jest-util "^29.7.0"
+    jest-worker "^29.7.0"
     micromatch "^4.0.4"
     walker "^1.0.8"
   optionalDependencies:
     fsevents "^2.3.2"
 
-jest-leak-detector@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-29.6.2.tgz#e2b307fee78cab091c37858a98c7e1d73cdf5b38"
-  integrity sha512-aNqYhfp5uYEO3tdWMb2bfWv6f0b4I0LOxVRpnRLAeque2uqOVVMLh6khnTcE2qJ5wAKop0HcreM1btoysD6bPQ==
+jest-leak-detector@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz#5b7ec0dadfdfec0ca383dc9aa016d36b5ea4c728"
+  integrity sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==
   dependencies:
-    jest-get-type "^29.4.3"
-    pretty-format "^29.6.2"
+    jest-get-type "^29.6.3"
+    pretty-format "^29.7.0"
 
-jest-matcher-utils@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.5.0.tgz"
-  integrity sha512-lecRtgm/rjIK0CQ7LPQwzCs2VwW6WAahA55YBuI+xqmhm7LAaxokSB8C97yJeYyT+HvQkH741StzpU41wohhWw==
+jest-matcher-utils@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz#ae8fec79ff249fd592ce80e3ee474e83a6c44f12"
+  integrity sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==
   dependencies:
     chalk "^4.0.0"
-    jest-diff "^29.5.0"
-    jest-get-type "^29.4.3"
-    pretty-format "^29.5.0"
+    jest-diff "^29.7.0"
+    jest-get-type "^29.6.3"
+    pretty-format "^29.7.0"
 
-jest-matcher-utils@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.6.2.tgz#39de0be2baca7a64eacb27291f0bd834fea3a535"
-  integrity sha512-4LiAk3hSSobtomeIAzFTe+N8kL6z0JtF3n6I4fg29iIW7tt99R7ZcIFW34QkX+DuVrf+CUe6wuVOpm7ZKFJzZQ==
-  dependencies:
-    chalk "^4.0.0"
-    jest-diff "^29.6.2"
-    jest-get-type "^29.4.3"
-    pretty-format "^29.6.2"
-
-jest-message-util@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.5.0.tgz"
-  integrity sha512-Kijeg9Dag6CKtIDA7O21zNTACqD5MD/8HfIV8pdD94vFyFuer52SigdC3IQMhab3vACxXMiFk+yMHNdbqtyTGA==
+jest-message-util@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.7.0.tgz#8bc392e204e95dfe7564abbe72a404e28e51f7f3"
+  integrity sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==
   dependencies:
     "@babel/code-frame" "^7.12.13"
-    "@jest/types" "^29.5.0"
+    "@jest/types" "^29.6.3"
     "@types/stack-utils" "^2.0.0"
     chalk "^4.0.0"
     graceful-fs "^4.2.9"
     micromatch "^4.0.4"
-    pretty-format "^29.5.0"
+    pretty-format "^29.7.0"
     slash "^3.0.0"
     stack-utils "^2.0.3"
 
-jest-message-util@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.6.2.tgz#af7adc2209c552f3f5ae31e77cf0a261f23dc2bb"
-  integrity sha512-vnIGYEjoPSuRqV8W9t+Wow95SDp6KPX2Uf7EoeG9G99J2OVh7OSwpS4B6J0NfpEIpfkBNHlBZpA2rblEuEFhZQ==
+jest-mock@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.7.0.tgz#4e836cf60e99c6fcfabe9f99d017f3fdd50a6347"
+  integrity sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==
   dependencies:
-    "@babel/code-frame" "^7.12.13"
-    "@jest/types" "^29.6.1"
-    "@types/stack-utils" "^2.0.0"
-    chalk "^4.0.0"
-    graceful-fs "^4.2.9"
-    micromatch "^4.0.4"
-    pretty-format "^29.6.2"
-    slash "^3.0.0"
-    stack-utils "^2.0.3"
-
-jest-mock@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.6.2.tgz#ef9c9b4d38c34a2ad61010a021866dad41ce5e00"
-  integrity sha512-hoSv3lb3byzdKfwqCuT6uTscan471GUECqgNYykg6ob0yiAw3zYc7OrPnI9Qv8Wwoa4lC7AZ9hyS4AiIx5U2zg==
-  dependencies:
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
-    jest-util "^29.6.2"
+    jest-util "^29.7.0"
 
 jest-pnp-resolver@^1.2.2:
   version "1.2.3"
   resolved "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz"
   integrity sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==
 
-jest-regex-util@^29.4.3:
-  version "29.4.3"
-  resolved "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.4.3.tgz"
-  integrity sha512-O4FglZaMmWXbGHSQInfXewIsd1LMn9p3ZXB/6r4FOkyhX2/iP/soMG98jGvk/A3HAN78+5VWcBGO0BJAPRh4kg==
+jest-regex-util@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-29.6.3.tgz#4a556d9c776af68e1c5f48194f4d0327d24e8a52"
+  integrity sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==
 
-jest-resolve-dependencies@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-29.6.2.tgz#36435269b6672c256bcc85fb384872c134cc4cf2"
-  integrity sha512-LGqjDWxg2fuQQm7ypDxduLu/m4+4Lb4gczc13v51VMZbVP5tSBILqVx8qfWcsdP8f0G7aIqByIALDB0R93yL+w==
+jest-resolve-dependencies@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz#1b04f2c095f37fc776ff40803dc92921b1e88428"
+  integrity sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==
   dependencies:
-    jest-regex-util "^29.4.3"
-    jest-snapshot "^29.6.2"
+    jest-regex-util "^29.6.3"
+    jest-snapshot "^29.7.0"
 
-jest-resolve@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-29.6.2.tgz#f18405fe4b50159b7b6d85e81f6a524d22afb838"
-  integrity sha512-G/iQUvZWI5e3SMFssc4ug4dH0aZiZpsDq9o1PtXTV1210Ztyb2+w+ZgQkB3iOiC5SmAEzJBOHWz6Hvrd+QnNPw==
+jest-resolve@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-29.7.0.tgz#64d6a8992dd26f635ab0c01e5eef4399c6bcbc30"
+  integrity sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==
   dependencies:
     chalk "^4.0.0"
     graceful-fs "^4.2.9"
-    jest-haste-map "^29.6.2"
+    jest-haste-map "^29.7.0"
     jest-pnp-resolver "^1.2.2"
-    jest-util "^29.6.2"
-    jest-validate "^29.6.2"
+    jest-util "^29.7.0"
+    jest-validate "^29.7.0"
     resolve "^1.20.0"
     resolve.exports "^2.0.0"
     slash "^3.0.0"
 
-jest-runner@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-29.6.2.tgz#89e8e32a8fef24781a7c4c49cd1cb6358ac7fc01"
-  integrity sha512-wXOT/a0EspYgfMiYHxwGLPCZfC0c38MivAlb2lMEAlwHINKemrttu1uSbcGbfDV31sFaPWnWJPmb2qXM8pqZ4w==
+jest-runner@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-29.7.0.tgz#809af072d408a53dcfd2e849a4c976d3132f718e"
+  integrity sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==
   dependencies:
-    "@jest/console" "^29.6.2"
-    "@jest/environment" "^29.6.2"
-    "@jest/test-result" "^29.6.2"
-    "@jest/transform" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/console" "^29.7.0"
+    "@jest/environment" "^29.7.0"
+    "@jest/test-result" "^29.7.0"
+    "@jest/transform" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     chalk "^4.0.0"
     emittery "^0.13.1"
     graceful-fs "^4.2.9"
-    jest-docblock "^29.4.3"
-    jest-environment-node "^29.6.2"
-    jest-haste-map "^29.6.2"
-    jest-leak-detector "^29.6.2"
-    jest-message-util "^29.6.2"
-    jest-resolve "^29.6.2"
-    jest-runtime "^29.6.2"
-    jest-util "^29.6.2"
-    jest-watcher "^29.6.2"
-    jest-worker "^29.6.2"
+    jest-docblock "^29.7.0"
+    jest-environment-node "^29.7.0"
+    jest-haste-map "^29.7.0"
+    jest-leak-detector "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-resolve "^29.7.0"
+    jest-runtime "^29.7.0"
+    jest-util "^29.7.0"
+    jest-watcher "^29.7.0"
+    jest-worker "^29.7.0"
     p-limit "^3.1.0"
     source-map-support "0.5.13"
 
-jest-runtime@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-29.6.2.tgz#692f25e387f982e89ab83270e684a9786248e545"
-  integrity sha512-2X9dqK768KufGJyIeLmIzToDmsN0m7Iek8QNxRSI/2+iPFYHF0jTwlO3ftn7gdKd98G/VQw9XJCk77rbTGZnJg==
+jest-runtime@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-29.7.0.tgz#efecb3141cf7d3767a3a0cc8f7c9990587d3d817"
+  integrity sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==
   dependencies:
-    "@jest/environment" "^29.6.2"
-    "@jest/fake-timers" "^29.6.2"
-    "@jest/globals" "^29.6.2"
-    "@jest/source-map" "^29.6.0"
-    "@jest/test-result" "^29.6.2"
-    "@jest/transform" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/environment" "^29.7.0"
+    "@jest/fake-timers" "^29.7.0"
+    "@jest/globals" "^29.7.0"
+    "@jest/source-map" "^29.6.3"
+    "@jest/test-result" "^29.7.0"
+    "@jest/transform" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     chalk "^4.0.0"
     cjs-module-lexer "^1.0.0"
     collect-v8-coverage "^1.0.0"
     glob "^7.1.3"
     graceful-fs "^4.2.9"
-    jest-haste-map "^29.6.2"
-    jest-message-util "^29.6.2"
-    jest-mock "^29.6.2"
-    jest-regex-util "^29.4.3"
-    jest-resolve "^29.6.2"
-    jest-snapshot "^29.6.2"
-    jest-util "^29.6.2"
+    jest-haste-map "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-mock "^29.7.0"
+    jest-regex-util "^29.6.3"
+    jest-resolve "^29.7.0"
+    jest-snapshot "^29.7.0"
+    jest-util "^29.7.0"
     slash "^3.0.0"
     strip-bom "^4.0.0"
 
-jest-snapshot@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-29.6.2.tgz#9b431b561a83f2bdfe041e1cab8a6becdb01af9c"
-  integrity sha512-1OdjqvqmRdGNvWXr/YZHuyhh5DeaLp1p/F8Tht/MrMw4Kr1Uu/j4lRG+iKl1DAqUJDWxtQBMk41Lnf/JETYBRA==
+jest-snapshot@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-29.7.0.tgz#c2c574c3f51865da1bb329036778a69bf88a6be5"
+  integrity sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==
   dependencies:
     "@babel/core" "^7.11.6"
     "@babel/generator" "^7.7.2"
     "@babel/plugin-syntax-jsx" "^7.7.2"
     "@babel/plugin-syntax-typescript" "^7.7.2"
     "@babel/types" "^7.3.3"
-    "@jest/expect-utils" "^29.6.2"
-    "@jest/transform" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/expect-utils" "^29.7.0"
+    "@jest/transform" "^29.7.0"
+    "@jest/types" "^29.6.3"
     babel-preset-current-node-syntax "^1.0.0"
     chalk "^4.0.0"
-    expect "^29.6.2"
+    expect "^29.7.0"
     graceful-fs "^4.2.9"
-    jest-diff "^29.6.2"
-    jest-get-type "^29.4.3"
-    jest-matcher-utils "^29.6.2"
-    jest-message-util "^29.6.2"
-    jest-util "^29.6.2"
+    jest-diff "^29.7.0"
+    jest-get-type "^29.6.3"
+    jest-matcher-utils "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-util "^29.7.0"
     natural-compare "^1.4.0"
-    pretty-format "^29.6.2"
+    pretty-format "^29.7.0"
     semver "^7.5.3"
 
-jest-util@^29.0.0, jest-util@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/jest-util/-/jest-util-29.5.0.tgz"
-  integrity sha512-RYMgG/MTadOr5t8KdhejfvUU82MxsCu5MF6KuDUHl+NuwzUt+Sm6jJWxTJVrDR1j5M/gJVCPKQEpWXY+yIQ6lQ==
+jest-util@^29.0.0, jest-util@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.7.0.tgz#23c2b62bfb22be82b44de98055802ff3710fc0bc"
+  integrity sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==
   dependencies:
-    "@jest/types" "^29.5.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     chalk "^4.0.0"
     ci-info "^3.2.0"
     graceful-fs "^4.2.9"
     picomatch "^2.2.3"
 
-jest-util@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.6.2.tgz#8a052df8fff2eebe446769fd88814521a517664d"
-  integrity sha512-3eX1qb6L88lJNCFlEADKOkjpXJQyZRiavX1INZ4tRnrBVr2COd3RgcTLyUiEXMNBlDU/cgYq6taUS0fExrWW4w==
+jest-validate@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-29.7.0.tgz#7bf705511c64da591d46b15fce41400d52147d9c"
+  integrity sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==
   dependencies:
-    "@jest/types" "^29.6.1"
-    "@types/node" "*"
-    chalk "^4.0.0"
-    ci-info "^3.2.0"
-    graceful-fs "^4.2.9"
-    picomatch "^2.2.3"
-
-jest-validate@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-29.6.2.tgz#25d972af35b2415b83b1373baf1a47bb266c1082"
-  integrity sha512-vGz0yMN5fUFRRbpJDPwxMpgSXW1LDKROHfBopAvDcmD6s+B/s8WJrwi+4bfH4SdInBA5C3P3BI19dBtKzx1Arg==
-  dependencies:
-    "@jest/types" "^29.6.1"
+    "@jest/types" "^29.6.3"
     camelcase "^6.2.0"
     chalk "^4.0.0"
-    jest-get-type "^29.4.3"
+    jest-get-type "^29.6.3"
     leven "^3.1.0"
-    pretty-format "^29.6.2"
+    pretty-format "^29.7.0"
 
-jest-watcher@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-29.6.2.tgz#77c224674f0620d9f6643c4cfca186d8893ca088"
-  integrity sha512-GZitlqkMkhkefjfN/p3SJjrDaxPflqxEAv3/ik10OirZqJGYH5rPiIsgVcfof0Tdqg3shQGdEIxDBx+B4tuLzA==
+jest-watcher@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-29.7.0.tgz#7810d30d619c3a62093223ce6bb359ca1b28a2f2"
+  integrity sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==
   dependencies:
-    "@jest/test-result" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/test-result" "^29.7.0"
+    "@jest/types" "^29.6.3"
     "@types/node" "*"
     ansi-escapes "^4.2.1"
     chalk "^4.0.0"
     emittery "^0.13.1"
-    jest-util "^29.6.2"
+    jest-util "^29.7.0"
     string-length "^4.0.1"
 
-jest-worker@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-29.6.2.tgz#682fbc4b6856ad0aa122a5403c6d048b83f3fb44"
-  integrity sha512-l3ccBOabTdkng8I/ORCkADz4eSMKejTYv1vB/Z83UiubqhC1oQ5Li6dWCyqOIvSifGjUBxuvxvlm6KGK2DtuAQ==
+jest-worker@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-29.7.0.tgz#acad073acbbaeb7262bd5389e1bcf43e10058d4a"
+  integrity sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==
   dependencies:
     "@types/node" "*"
-    jest-util "^29.6.2"
+    jest-util "^29.7.0"
     merge-stream "^2.0.0"
     supports-color "^8.0.0"
 
-jest@29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/jest/-/jest-29.6.2.tgz#3bd55b9fd46a161b2edbdf5f1d1bd0d1eab76c42"
-  integrity sha512-8eQg2mqFbaP7CwfsTpCxQ+sHzw1WuNWL5UUvjnWP4hx2riGz9fPSzYOaU5q8/GqWn1TfgZIVTqYJygbGbWAANg==
+jest@29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest/-/jest-29.7.0.tgz#994676fc24177f088f1c5e3737f5697204ff2613"
+  integrity sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==
   dependencies:
-    "@jest/core" "^29.6.2"
-    "@jest/types" "^29.6.1"
+    "@jest/core" "^29.7.0"
+    "@jest/types" "^29.6.3"
     import-local "^3.0.2"
-    jest-cli "^29.6.2"
+    jest-cli "^29.7.0"
 
 jmespath@0.15.0:
   version "0.15.0"
@@ -3511,12 +3431,12 @@ mkdirp@^1.0.3:
   resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
-ms@2.1.2, ms@^2.1.1:
+ms@2.1.2:
   version "2.1.2"
   resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
-ms@^2.1.3:
+ms@^2.1.1, ms@^2.1.3:
   version "2.1.3"
   resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
@@ -4085,21 +4005,12 @@ prettier@2.7.1:
   resolved "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz"
   integrity sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==
 
-pretty-format@^29.0.0, pretty-format@^29.5.0:
-  version "29.5.0"
-  resolved "https://registry.npmjs.org/pretty-format/-/pretty-format-29.5.0.tgz"
-  integrity sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw==
+pretty-format@^29.0.0, pretty-format@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.7.0.tgz#ca42c758310f365bfa71a0bda0a807160b776812"
+  integrity sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==
   dependencies:
-    "@jest/schemas" "^29.4.3"
-    ansi-styles "^5.0.0"
-    react-is "^18.0.0"
-
-pretty-format@^29.6.2:
-  version "29.6.2"
-  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.6.2.tgz#3d5829261a8a4d89d8b9769064b29c50ed486a47"
-  integrity sha512-1q0oC8eRveTg5nnBEWMXAU2qpv65Gnuf2eCQzSjxpWFkPaPARwqZZDGuNE0zPAZfTCHzIk3A8dIjwlQKKLphyg==
-  dependencies:
-    "@jest/schemas" "^29.6.0"
+    "@jest/schemas" "^29.6.3"
     ansi-styles "^5.0.0"
     react-is "^18.0.0"
 
@@ -4368,12 +4279,12 @@ sanitize-s3-objectkey@0.0.1:
   resolved "https://registry.npmjs.org/sanitize-s3-objectkey/-/sanitize-s3-objectkey-0.0.1.tgz"
   integrity sha512-ZTk7aqLxy4sD40GWcYWoLfbe05XLmkKvh6vGKe13ADlei24xlezcvjgKy1qRArlaIbIMYaqK7PCalvZtulZlaQ==
 
-sax@1.2.1, sax@>=0.6.0:
+sax@1.2.1:
   version "1.2.1"
   resolved "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
-sax@>=0.1.1:
+sax@>=0.1.1, sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -4395,14 +4306,7 @@ semver@^6.0.0, semver@^6.3.0:
   resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz"
   integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
 
-semver@^7.3.2, semver@^7.3.5, semver@^7.3.8:
-  version "7.3.8"
-  resolved "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz"
-  integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==
-  dependencies:
-    lru-cache "^6.0.0"
-
-semver@^7.5.3:
+semver@^7.3.2, semver@^7.3.5, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4:
   version "7.5.4"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e"
   integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==
@@ -4789,9 +4693,9 @@ toidentifier@1.0.1:
   integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
 
 "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.1.2:
-  version "4.1.2"
-  resolved "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz"
-  integrity sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
+  integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
   dependencies:
     psl "^1.1.33"
     punycode "^2.1.1"
diff --git a/yarn.lock b/yarn.lock
index eaf7a72e80..521c4dbaf4 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -829,12 +829,13 @@
     events "^3.0.0"
     tslib "^2.2.0"
 
-"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.5.tgz#234d98e1551960604f1246e6475891a570ad5658"
-  integrity sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ==
+"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5", "@babel/code-frame@^7.23.5":
+  version "7.23.5"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244"
+  integrity sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==
   dependencies:
-    "@babel/highlight" "^7.22.5"
+    "@babel/highlight" "^7.23.4"
+    chalk "^2.4.2"
 
 "@babel/compat-data@^7.16.8", "@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.5", "@babel/compat-data@^7.22.5":
   version "7.22.5"
@@ -880,12 +881,12 @@
     "@jridgewell/gen-mapping" "^0.3.0"
     jsesc "^2.5.1"
 
-"@babel/generator@^7.22.5", "@babel/generator@^7.7.2":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.22.5.tgz#1e7bf768688acfb05cf30b2369ef855e82d984f7"
-  integrity sha512-+lcUbnTRhd0jOewtFSedLyiPsD5tswKkbgcezOqqWFUVNEwoUTlpPOBmvhG7OXWLR4jMdv0czPGH5XbflnD1EA==
+"@babel/generator@^7.22.5", "@babel/generator@^7.23.6", "@babel/generator@^7.7.2":
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.6.tgz#9e1fca4811c77a10580d17d26b57b036133f3c2e"
+  integrity sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==
   dependencies:
-    "@babel/types" "^7.22.5"
+    "@babel/types" "^7.23.6"
     "@jridgewell/gen-mapping" "^0.3.2"
     "@jridgewell/trace-mapping" "^0.3.17"
     jsesc "^2.5.1"
@@ -963,18 +964,18 @@
     resolve "^1.14.2"
     semver "^6.1.2"
 
-"@babel/helper-environment-visitor@^7.18.9", "@babel/helper-environment-visitor@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98"
-  integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==
+"@babel/helper-environment-visitor@^7.18.9", "@babel/helper-environment-visitor@^7.22.20", "@babel/helper-environment-visitor@^7.22.5":
+  version "7.22.20"
+  resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167"
+  integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==
 
-"@babel/helper-function-name@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz#ede300828905bb15e582c037162f99d5183af1be"
-  integrity sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==
+"@babel/helper-function-name@^7.22.5", "@babel/helper-function-name@^7.23.0":
+  version "7.23.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759"
+  integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==
   dependencies:
-    "@babel/template" "^7.22.5"
-    "@babel/types" "^7.22.5"
+    "@babel/template" "^7.22.15"
+    "@babel/types" "^7.23.0"
 
 "@babel/helper-hoist-variables@^7.22.5":
   version "7.22.5"
@@ -1059,22 +1060,22 @@
   dependencies:
     "@babel/types" "^7.22.5"
 
-"@babel/helper-split-export-declaration@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.5.tgz#88cf11050edb95ed08d596f7a044462189127a08"
-  integrity sha512-thqK5QFghPKWLhAV321lxF95yCg2K3Ob5yw+M3VHWfdia0IkPXUtoLH8x/6Fh486QUvzhb8YOWHChTVen2/PoQ==
+"@babel/helper-split-export-declaration@^7.22.5", "@babel/helper-split-export-declaration@^7.22.6":
+  version "7.22.6"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c"
+  integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==
   dependencies:
     "@babel/types" "^7.22.5"
 
-"@babel/helper-string-parser@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f"
-  integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==
+"@babel/helper-string-parser@^7.23.4":
+  version "7.23.4"
+  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83"
+  integrity sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==
 
-"@babel/helper-validator-identifier@^7.16.7", "@babel/helper-validator-identifier@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193"
-  integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==
+"@babel/helper-validator-identifier@^7.16.7", "@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.22.5":
+  version "7.22.20"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0"
+  integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==
 
 "@babel/helper-validator-option@^7.16.7", "@babel/helper-validator-option@^7.22.5":
   version "7.22.5"
@@ -1100,13 +1101,13 @@
     "@babel/traverse" "^7.22.5"
     "@babel/types" "^7.22.5"
 
-"@babel/highlight@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.5.tgz#aa6c05c5407a67ebce408162b7ede789b4d22031"
-  integrity sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw==
+"@babel/highlight@^7.23.4":
+  version "7.23.4"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.23.4.tgz#edaadf4d8232e1a961432db785091207ead0621b"
+  integrity sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==
   dependencies:
-    "@babel/helper-validator-identifier" "^7.22.5"
-    chalk "^2.0.0"
+    "@babel/helper-validator-identifier" "^7.22.20"
+    chalk "^2.4.2"
     js-tokens "^4.0.0"
 
 "@babel/parser@7.18.4":
@@ -1114,10 +1115,10 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.4.tgz#6774231779dd700e0af29f6ad8d479582d7ce5ef"
   integrity sha512-FDge0dFazETFcxGw/EXzOkN8uJp0PC7Qbm+Pe9T+av2zlBpOgunFHkQPPn+eRuClU73JF+98D531UgayY89tow==
 
-"@babel/parser@^7.0.0", "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.15", "@babel/parser@^7.20.7", "@babel/parser@^7.22.5":
-  version "7.23.0"
-  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719"
-  integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==
+"@babel/parser@^7.0.0", "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.15", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.22.5", "@babel/parser@^7.23.6":
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.6.tgz#ba1c9e512bda72a47e285ae42aff9d2a635a9e3b"
+  integrity sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==
 
 "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.7", "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.22.5":
   version "7.22.5"
@@ -1999,43 +2000,36 @@
   resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310"
   integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==
 
-"@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
-  version "7.23.2"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.2.tgz#062b0ac103261d68a966c4c7baf2ae3e62ec3885"
-  integrity sha512-mM8eg4yl5D6i3lu2QKPuPH4FArvJ8KhTofbE7jwMUv9KX5mBvwPAqnV3MlyBNqdp9RyRKP6Yck8TrfYrPvX3bg==
-  dependencies:
-    regenerator-runtime "^0.14.0"
-
-"@babel/runtime@^7.13.10":
+"@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.15.4", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
   version "7.23.6"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.6.tgz#c05e610dc228855dc92ef1b53d07389ed8ab521d"
   integrity sha512-zHd0eUrf5GZoOWVCXp6koAKQTfZV07eit6bGPmJgnZdnSAvvZee6zniW2XMF7Cmc4ISOOnPy3QaSiIJGJkVEDQ==
   dependencies:
     regenerator-runtime "^0.14.0"
 
-"@babel/template@^7.22.5", "@babel/template@^7.3.3":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.5.tgz#0c8c4d944509875849bd0344ff0050756eefc6ec"
-  integrity sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==
+"@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.3.3":
+  version "7.22.15"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38"
+  integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==
   dependencies:
-    "@babel/code-frame" "^7.22.5"
-    "@babel/parser" "^7.22.5"
-    "@babel/types" "^7.22.5"
+    "@babel/code-frame" "^7.22.13"
+    "@babel/parser" "^7.22.15"
+    "@babel/types" "^7.22.15"
 
 "@babel/traverse@^7.22.5":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.5.tgz#44bd276690db6f4940fdb84e1cb4abd2f729ccd1"
-  integrity sha512-7DuIjPgERaNo6r+PZwItpjCZEa5vyw4eJGufeLxrPdBXBoLcCJCIasvK6pK/9DVNrLZTLFhUGqaC6X/PA007TQ==
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.6.tgz#b53526a2367a0dd6edc423637f3d2d0f2521abc5"
+  integrity sha512-czastdK1e8YByZqezMPFiZ8ahwVMh/ESl9vPgvgdB9AmFMGP5jfpFax74AQgl5zj4XHzqeYAg2l8PuUeRS1MgQ==
   dependencies:
-    "@babel/code-frame" "^7.22.5"
-    "@babel/generator" "^7.22.5"
-    "@babel/helper-environment-visitor" "^7.22.5"
-    "@babel/helper-function-name" "^7.22.5"
+    "@babel/code-frame" "^7.23.5"
+    "@babel/generator" "^7.23.6"
+    "@babel/helper-environment-visitor" "^7.22.20"
+    "@babel/helper-function-name" "^7.23.0"
     "@babel/helper-hoist-variables" "^7.22.5"
-    "@babel/helper-split-export-declaration" "^7.22.5"
-    "@babel/parser" "^7.22.5"
-    "@babel/types" "^7.22.5"
-    debug "^4.1.0"
+    "@babel/helper-split-export-declaration" "^7.22.6"
+    "@babel/parser" "^7.23.6"
+    "@babel/types" "^7.23.6"
+    debug "^4.3.1"
     globals "^11.1.0"
 
 "@babel/types@7.18.4":
@@ -2046,13 +2040,13 @@
     "@babel/helper-validator-identifier" "^7.16.7"
     to-fast-properties "^2.0.0"
 
-"@babel/types@^7.0.0", "@babel/types@^7.16.8", "@babel/types@^7.18.2", "@babel/types@^7.20.7", "@babel/types@^7.22.5", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4":
-  version "7.22.5"
-  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe"
-  integrity sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA==
+"@babel/types@^7.0.0", "@babel/types@^7.16.8", "@babel/types@^7.18.2", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4":
+  version "7.23.6"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd"
+  integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==
   dependencies:
-    "@babel/helper-string-parser" "^7.22.5"
-    "@babel/helper-validator-identifier" "^7.22.5"
+    "@babel/helper-string-parser" "^7.23.4"
+    "@babel/helper-validator-identifier" "^7.22.20"
     to-fast-properties "^2.0.0"
 
 "@balena/dockerignore@^1.0.2":
@@ -3795,16 +3789,11 @@
   dependencies:
     "@octokit/openapi-types" "^18.0.0"
 
-"@opentelemetry/api@^1.0.0":
+"@opentelemetry/api@^1.0.0", "@opentelemetry/api@^1.0.1":
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.7.0.tgz#b139c81999c23e3c8d3c0a7234480e945920fc40"
   integrity sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw==
 
-"@opentelemetry/api@^1.0.1":
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.4.0.tgz#2c91791a9ba6ca0a0f4aaac5e45d58df13639ac8"
-  integrity sha512-IgMK9i3sFGNUqPMbjABm0G26g0QCKCUBfglhQ7rQq6WcxbKfEHRcmwsoER4hZcuYqJgkYn2OeuoJIv7Jsftp7g==
-
 "@opentelemetry/core@^1.14.0":
   version "1.19.0"
   resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.19.0.tgz#6563bb65465bf232d8435553b9a122d9351c0fbb"
@@ -5678,10 +5667,10 @@
     "@types/node" "*"
     form-data "^3.0.0"
 
-"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.13.4", "@types/node@>=13.7.0":
-  version "20.8.9"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.9.tgz#646390b4fab269abce59c308fc286dcd818a2b08"
-  integrity sha512-UzykFsT3FhHb1h7yD4CA4YhBHq545JC0YnEz41xkipN88eKQtL6rSgocL5tbAP6Ola9Izm/Aw4Ora8He4x0BHg==
+"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.13.4", "@types/node@>=13.7.0", "@types/node@>=8.1.0":
+  version "20.10.5"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.5.tgz#47ad460b514096b7ed63a1dae26fad0914ed3ab2"
+  integrity sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==
   dependencies:
     undici-types "~5.26.4"
 
@@ -5705,13 +5694,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.37.tgz#0bfcd173e8e1e328337473a8317e37b3b14fd30d"
   integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
 
-"@types/node@>=8.1.0":
-  version "20.10.5"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.5.tgz#47ad460b514096b7ed63a1dae26fad0914ed3ab2"
-  integrity sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==
-  dependencies:
-    undici-types "~5.26.4"
-
 "@types/nodemailer@^6.4.4":
   version "6.4.14"
   resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.14.tgz#5c81a5e856db7f8ede80013e6dbad7c5fb2283e2"
@@ -7176,9 +7158,9 @@ axios@^0.26.0:
     follow-redirects "^1.14.8"
 
 axios@^1.0.0, axios@^1.1.3, axios@^1.5.0:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.5.1.tgz#11fbaa11fc35f431193a9564109c88c1f27b585f"
-  integrity sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==
+  version "1.6.2"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.2.tgz#de67d42c755b571d3e698df1b6504cde9b0ee9f2"
+  integrity sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==
   dependencies:
     follow-redirects "^1.15.0"
     form-data "^4.0.0"
@@ -8031,7 +8013,7 @@ chalk@4.1.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
-chalk@^2.0.0, chalk@^2.4.2:
+chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
   integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
@@ -12585,12 +12567,7 @@ ignore-walk@^6.0.0:
   dependencies:
     minimatch "^7.4.2"
 
-ignore@^5.0.4, ignore@^5.2.0:
-  version "5.2.4"
-  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324"
-  integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==
-
-ignore@^5.2.4:
+ignore@^5.0.4, ignore@^5.2.0, ignore@^5.2.4:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78"
   integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==
@@ -18203,20 +18180,13 @@ postgres-interval@^1.1.0:
   dependencies:
     xtend "^4.0.0"
 
-posthog-js@^1.13.4:
+posthog-js@^1.13.4, posthog-js@^1.36.0:
   version "1.96.1"
   resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.96.1.tgz#4f9719a24e4e14037b0e72d430194d7cdb576447"
   integrity sha512-kv1vQqYMt2BV3YHS+wxsbGuP+tz+M3y1AzNhz8TfkpY1HT8W/ONT0i0eQpeRr9Y+d4x/fZ6M4cXG5GMvi9lRCA==
   dependencies:
     fflate "^0.4.1"
 
-posthog-js@^1.36.0:
-  version "1.87.2"
-  resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.87.2.tgz#0e9395ea93d50bb624f0be19f254e62e54ae504d"
-  integrity sha512-pdxEylfxwEDwwz7g5dunPucvAN51RAOWWQmkcqHsLNHlV5o5bTaTwcAXaWB1IUn3xKPuKYE2lqbdB3vC4H4rFQ==
-  dependencies:
-    fflate "^0.4.1"
-
 posthog-node@1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-1.3.0.tgz#804ed2f213a2f05253f798bf9569d55a9cad94f7"
@@ -22236,12 +22206,7 @@ uuid@^7.0.3:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-7.0.3.tgz#c5c9f2c8cf25dc0a372c4df1441c41f5bd0c680b"
   integrity sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==
 
-uuid@^9.0.0:
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
-  integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
-
-uuid@^9.0.1:
+uuid@^9.0.0, uuid@^9.0.1:
   version "9.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30"
   integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==

From 6adb5cfe7955447d0d576a1d7b785e20e3988a41 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Wed, 20 Dec 2023 11:02:59 +0000
Subject: [PATCH 12/68] Do not throw error on 204 no content (#12643)

---
 packages/server/src/integrations/rest.ts      |  5 ++++-
 .../src/integrations/tests/rest.spec.ts       | 22 +++++++++++++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts
index e9d1d8268e..9cb8f8e2c1 100644
--- a/packages/server/src/integrations/rest.ts
+++ b/packages/server/src/integrations/rest.ts
@@ -131,7 +131,10 @@ class RestIntegration implements IntegrationBase {
     let data, raw, headers
     const contentType = response.headers.get("content-type") || ""
     try {
-      if (contentType.includes("application/json")) {
+      if (response.status === 204) {
+        data = []
+        raw = []
+      } else if (contentType.includes("application/json")) {
         data = await response.json()
         raw = JSON.stringify(data)
       } else if (
diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts
index 98c57580a3..bc0c0cac2f 100644
--- a/packages/server/src/integrations/tests/rest.spec.ts
+++ b/packages/server/src/integrations/tests/rest.spec.ts
@@ -186,9 +186,15 @@ describe("REST Integration", () => {
   })
 
   describe("response", () => {
-    function buildInput(json: any, text: any, header: any) {
+    const contentTypes = ["application/json", "text/plain", "application/xml"]
+    function buildInput(
+      json: any,
+      text: any,
+      header: any,
+      status: number = 200
+    ) {
       return {
-        status: 200,
+        status,
         json: json ? async () => json : undefined,
         text: text ? async () => text : undefined,
         headers: {
@@ -225,6 +231,18 @@ describe("REST Integration", () => {
       expect(output.extra.raw).toEqual(text)
       expect(output.extra.headers["content-type"]).toEqual("application/xml")
     })
+
+    test.each(contentTypes)(
+      "should not throw an error on 204 no content",
+      async contentType => {
+        const input = buildInput(undefined, null, contentType, 204)
+        const output = await config.integration.parseResponse(input)
+        expect(output.data).toEqual([])
+        expect(output.extra.raw).toEqual([])
+        expect(output.info.code).toEqual(204)
+        expect(output.extra.headers["content-type"]).toEqual(contentType)
+      }
+    )
   })
 
   describe("authentication", () => {

From b4ee112d0af3ea3c5be526f48dab57a6edebc39e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Wed, 20 Dec 2023 12:03:38 +0000
Subject: [PATCH 13/68] Update runner.sh

---
 hosting/single/runner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh
index eb9ec07464..886da4c916 100644
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@@ -79,7 +79,7 @@ redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /bbcouch-runner.sh &
 
 # only start minio if use s3 isn't passed
-if [[ -z "${USE_S3}"]]; then
+if [[ -z "${USE_S3}" ]]; then
   /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 fi
 

From 1c1e305f03972735b91062da4a6c26c7e39a78a4 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Wed, 20 Dec 2023 13:27:23 +0100
Subject: [PATCH 14/68] Upgrade account portal submodule

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index d8fbdb956a..cb04efac5a 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit d8fbdb956ae0c4119da5bf44b867ff25252073f3
+Subproject commit cb04efac5a22cf5f68e921e807d1fc500078dbd7

From 915b96d78d4cde5ef01050fde6eb63b41674a7b6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 16:25:20 +0000
Subject: [PATCH 15/68] Allow TEMPLATES_BUCKET_NAME to be configured in the
 Helm chart.

---
 charts/budibase/templates/app-service-deployment.yaml           | 2 ++
 .../templates/automation-worker-service-deployment.yaml         | 2 ++
 charts/budibase/templates/worker-service-deployment.yaml        | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index c6ded3cee2..e04a327479 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -106,6 +106,8 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: TEMPLATES_BUCKET_NAME
+          value: {{ .values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.apps.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index b7eece6b85..7f429c5451 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -107,6 +107,8 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: TEMPLATES_BUCKET_NAME
+          value: {{ .values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.automationWorkers.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index 04791df869..1a5d22cf54 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -106,6 +106,8 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: TEMPLATES_BUCKET_NAME
+          value: {{ .values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY

From 5083369a997095793f5192540efe7bcd262cca5f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 16:26:25 +0000
Subject: [PATCH 16/68] Fix incorrect casing.

---
 charts/budibase/templates/app-service-deployment.yaml           | 2 +-
 .../templates/automation-worker-service-deployment.yaml         | 2 +-
 charts/budibase/templates/worker-service-deployment.yaml        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index e04a327479..2cf15a7ade 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -107,7 +107,7 @@ spec:
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
         - name: TEMPLATES_BUCKET_NAME
-          value: {{ .values.services.objectStore.templatesBucketName | quote }}
+          value: {{ .Values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.apps.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index 7f429c5451..0d855c3859 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -108,7 +108,7 @@ spec:
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
         - name: TEMPLATES_BUCKET_NAME
-          value: {{ .values.services.objectStore.templatesBucketName | quote }}
+          value: {{ .Values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.automationWorkers.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index 1a5d22cf54..77d719ab70 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -107,7 +107,7 @@ spec:
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
         - name: TEMPLATES_BUCKET_NAME
-          value: {{ .values.services.objectStore.templatesBucketName | quote }}
+          value: {{ .Values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY

From 8dfdddd2468cada6b0271428d9690ae414dca5ba Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 20 Dec 2023 16:47:51 +0000
Subject: [PATCH 17/68] Bump version to 2.13.49

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d2a623fcd2..8aef7c70ec 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.48",
+  "version": "2.13.49",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From d3de8acd77c25290d28bf87b252cc3bddd74e55b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 17:08:49 +0000
Subject: [PATCH 18/68] Revert "Allow TEMPLATES_BUCKET_NAME to be configured in
 the Helm chart."

---
 charts/budibase/templates/app-service-deployment.yaml           | 2 --
 .../templates/automation-worker-service-deployment.yaml         | 2 --
 charts/budibase/templates/worker-service-deployment.yaml        | 2 --
 3 files changed, 6 deletions(-)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 2cf15a7ade..c6ded3cee2 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -106,8 +106,6 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
-        - name: TEMPLATES_BUCKET_NAME
-          value: {{ .Values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.apps.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index 0d855c3859..b7eece6b85 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -107,8 +107,6 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
-        - name: TEMPLATES_BUCKET_NAME
-          value: {{ .Values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.automationWorkers.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index 77d719ab70..04791df869 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -106,8 +106,6 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
-        - name: TEMPLATES_BUCKET_NAME
-          value: {{ .Values.services.objectStore.templatesBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY

From 1f3a75e8920cb5fe1b5fb788fa47a84e8c1eec4e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 17:20:30 +0000
Subject: [PATCH 19/68] Downgrade back to dd-trace 3.13.2

---
 packages/backend-core/package.json | 2 +-
 packages/server/package.json       | 2 +-
 packages/worker/package.json       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index a0dd320dff..36b8de0f56 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -32,7 +32,7 @@
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
-    "dd-trace": "4.20.0",
+    "dd-trace": "3.13.2",
     "dotenv": "16.0.1",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
diff --git a/packages/server/package.json b/packages/server/package.json
index 0274db48ee..8f5b526a62 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -65,7 +65,7 @@
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
-    "dd-trace": "4.20.0",
+    "dd-trace": "3.13.2",
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index e9c2f4c5de..e7e63680ea 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -48,7 +48,7 @@
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
-    "dd-trace": "4.20.0",
+    "dd-trace": "3.13.2",
     "dotenv": "8.6.0",
     "global-agent": "3.0.0",
     "ical-generator": "4.1.0",

From 794ff7c9ef4d244cc0a09287a74778095952bc36 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 20 Dec 2023 17:21:54 +0000
Subject: [PATCH 20/68] run yarn install

---
 yarn.lock | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 5 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 521c4dbaf4..6245e0c101 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2259,6 +2259,13 @@
     enabled "2.0.x"
     kuler "^2.0.0"
 
+"@datadog/native-appsec@2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-2.0.0.tgz#ad65ba19bfd68e6b6c6cf64bb8ef55d099af8edc"
+  integrity sha512-XHARZ6MVgbnfOUO6/F3ZoZ7poXHJCNYFlgcyS2Xetuk9ITA5bfcooX2B2F7tReVB+RLJ+j8bsm0t55SyF04KDw==
+  dependencies:
+    node-gyp-build "^3.9.0"
+
 "@datadog/native-appsec@4.0.0":
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-4.0.0.tgz#ee08138b987dec557eac3650a43a972dac85b6a6"
@@ -2266,6 +2273,13 @@
   dependencies:
     node-gyp-build "^3.9.0"
 
+"@datadog/native-iast-rewriter@1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-1.1.2.tgz#793cbf92d218ec80d645be0830023656b81018ea"
+  integrity sha512-pigRfRtAjZjMjqIXyXb98S4aDnuHz/EmqpoxAajFZsNjBLM87YonwSY5zoBdCsOyA46ddKOJRoCQd5ZalpOFMQ==
+  dependencies:
+    node-gyp-build "^4.5.0"
+
 "@datadog/native-iast-rewriter@2.2.1":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.2.1.tgz#3c74c5a8caa0b876e091e9c5a95256add0d73e1c"
@@ -2274,6 +2288,13 @@
     lru-cache "^7.14.0"
     node-gyp-build "^4.5.0"
 
+"@datadog/native-iast-taint-tracking@1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-1.1.0.tgz#8f7d0016157b32dbf5c01b15b8afb1c4286b4a18"
+  integrity sha512-TOrngpt6Qh52zWFOz1CkFXw0g43rnuUziFBtIMUsOLGzSHr9wdnTnE6HAyuvKy3f3ecAoZESlMfilGRKP93hXQ==
+  dependencies:
+    node-gyp-build "^3.9.0"
+
 "@datadog/native-iast-taint-tracking@1.6.4":
   version "1.6.4"
   resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-1.6.4.tgz#16c21ad7c36a53420c0d3c5a3720731809cc7e98"
@@ -2281,6 +2302,13 @@
   dependencies:
     node-gyp-build "^3.9.0"
 
+"@datadog/native-metrics@^1.5.0":
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-1.6.0.tgz#1c7958964460149911f6964c32b1a8692ee3ce8f"
+  integrity sha512-+8jBzd0nlLV+ay3Vb87DLwz8JHAS817hRhSRQ6zxhud9TyvvcNTNN+VA2sb2fe5UK4aMDvj/sGVJjEtgr4RHew==
+  dependencies:
+    node-gyp-build "^3.9.0"
+
 "@datadog/native-metrics@^2.0.0":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-2.0.0.tgz#65bf03313ee419956361e097551db36173e85712"
@@ -2300,6 +2328,20 @@
     pprof-format "^2.0.7"
     source-map "^0.7.4"
 
+"@datadog/pprof@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-1.1.1.tgz#17e86035140523ac3a96f3662e5dd29822042d61"
+  integrity sha512-5lYXUpikQhrJwzODtJ7aFM0oKmPccISnTCecuWhjxIj4/7UJv0DamkLak634bgEW+kiChgkKFDapHSesuXRDXQ==
+  dependencies:
+    delay "^5.0.0"
+    findit2 "^2.2.3"
+    node-gyp-build "^3.9.0"
+    p-limit "^3.1.0"
+    pify "^5.0.0"
+    protobufjs "^7.0.0"
+    source-map "^0.7.3"
+    split "^1.0.1"
+
 "@datadog/sketches-js@^2.1.0":
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/@datadog/sketches-js/-/sketches-js-2.1.0.tgz#8c7e8028a5fc22ad102fa542b0a446c956830455"
@@ -9076,6 +9118,39 @@ dc-polyfill@^0.1.2:
   resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.3.tgz#fe9eefc86813439dd46d6f9ad9582ec079c39720"
   integrity sha512-Wyk5n/5KUj3GfVKV2jtDbtChC/Ff9fjKsBcg4ZtYW1yQe3DXNHcGURvmoxhqQdfOQ9TwyMjnfyv1lyYcOkFkFA==
 
+dd-trace@3.13.2:
+  version "3.13.2"
+  resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-3.13.2.tgz#95b1ec480ab9ac406e1da7591a8c6f678d3799fd"
+  integrity sha512-POO9nEcAufe5pgp2xV1X3PfWip6wh+6TpEcRSlSgZJCIIMvWVCkcIVL/J2a6KAZq6V3Yjbkl8Ktfe+MOzQf5kw==
+  dependencies:
+    "@datadog/native-appsec" "2.0.0"
+    "@datadog/native-iast-rewriter" "1.1.2"
+    "@datadog/native-iast-taint-tracking" "1.1.0"
+    "@datadog/native-metrics" "^1.5.0"
+    "@datadog/pprof" "^1.1.1"
+    "@datadog/sketches-js" "^2.1.0"
+    crypto-randomuuid "^1.0.0"
+    diagnostics_channel "^1.1.0"
+    ignore "^5.2.0"
+    import-in-the-middle "^1.3.4"
+    ipaddr.js "^2.0.1"
+    istanbul-lib-coverage "3.2.0"
+    koalas "^1.0.2"
+    limiter "^1.1.4"
+    lodash.kebabcase "^4.1.1"
+    lodash.pick "^4.4.0"
+    lodash.sortby "^4.7.0"
+    lodash.uniq "^4.5.0"
+    lru-cache "^7.14.0"
+    methods "^1.1.2"
+    module-details-from-path "^1.0.3"
+    node-abort-controller "^3.0.1"
+    opentracing ">=0.12.1"
+    path-to-regexp "^0.1.2"
+    protobufjs "^7.1.2"
+    retry "^0.10.1"
+    semver "^5.5.0"
+
 dd-trace@4.20.0:
   version "4.20.0"
   resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-4.20.0.tgz#9a2cc3f28ff558c5605927b1362eb64605df76c1"
@@ -9637,6 +9712,11 @@ dezalgo@^1.0.4:
     asap "^2.0.0"
     wrappy "1"
 
+diagnostics_channel@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/diagnostics_channel/-/diagnostics_channel-1.1.0.tgz#bd66c49124ce3bac697dff57466464487f57cea5"
+  integrity sha512-OE1ngLDjSBPG6Tx0YATELzYzy3RKHC+7veQ8gLa8yS7AAgw65mFbVdcsu3501abqOZCEZqZyAIemB0zXlqDSuw==
+
 diff-match-patch@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/diff-match-patch/-/diff-match-patch-1.0.5.tgz#abb584d5f10cd1196dfc55aa03701592ae3f7b37"
@@ -11162,6 +11242,11 @@ find-up@^5.0.0:
     locate-path "^6.0.0"
     path-exists "^4.0.0"
 
+findit2@^2.2.3:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6"
+  integrity sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==
+
 flat-cache@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11"
@@ -12616,7 +12701,7 @@ import-from@^3.0.0:
   dependencies:
     resolve-from "^5.0.0"
 
-import-in-the-middle@^1.4.2:
+import-in-the-middle@^1.3.4, import-in-the-middle@^1.4.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.7.2.tgz#31c44088271b50ecb9cacbdfb1e5732c802e0658"
   integrity sha512-coz7AjRnPyKW36J6JX5Bjz1mcX7MX1H2XsEGseVcnXMdzsAbbAu0HBZhiAem+3SAmuZdi+p8OwoB2qUpTRgjOQ==
@@ -12838,7 +12923,7 @@ ip@^2.0.0:
   resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da"
   integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==
 
-ipaddr.js@^2.1.0:
+ipaddr.js@^2.0.1, ipaddr.js@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.1.0.tgz#2119bc447ff8c257753b196fc5f1ce08a4cdf39f"
   integrity sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==
@@ -18696,7 +18781,7 @@ protobufjs@7.2.4:
     "@types/node" ">=13.7.0"
     long "^5.0.0"
 
-protobufjs@^7.0.0, protobufjs@^7.2.4, protobufjs@^7.2.5:
+protobufjs@^7.0.0, protobufjs@^7.1.2, protobufjs@^7.2.4, protobufjs@^7.2.5:
   version "7.2.5"
   resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.5.tgz#45d5c57387a6d29a17aab6846dcc283f9b8e7f2d"
   integrity sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==
@@ -19470,7 +19555,7 @@ retry@0.13.1, retry@^0.13.1:
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
   integrity "sha1-GFsVh6z2eRnWOzVzSeA1N7JIRlg= sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="
 
-retry@^0.10.0:
+retry@^0.10.0, retry@^0.10.1:
   version "0.10.1"
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4"
   integrity sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==
@@ -20368,7 +20453,7 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
-source-map@^0.7.4:
+source-map@^0.7.3, source-map@^0.7.4:
   version "0.7.4"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.4.tgz#a9bbe705c9d8846f4e08ff6765acf0f1b0898656"
   integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==

From 3c0901f53022e1d117fc50d57b6e10909ec0ef03 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 21 Dec 2023 11:06:05 +0000
Subject: [PATCH 21/68] Remove all custom tracing to see if it's the cause of
 the memory leak.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts |   4 +-
 packages/backend-core/src/db/db.ts            |   3 +-
 .../backend-core/src/db/instrumentation.ts    | 156 -----
 .../backend-core/src/logging/pino/logger.ts   |   6 -
 packages/server/src/automations/utils.ts      |  70 +--
 packages/server/src/jsRunner.ts               |  54 +-
 packages/server/src/middleware/currentapp.ts  |  14 -
 packages/server/src/threads/automation.ts     | 563 ++++++++----------
 .../src/utilities/rowProcessor/utils.ts       |  56 +-
 9 files changed, 313 insertions(+), 613 deletions(-)
 delete mode 100644 packages/backend-core/src/db/instrumentation.ts

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 3fec573bb9..c2c0b6b21d 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -17,7 +17,6 @@ import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
-import { DDInstrumentedDatabase } from "../instrumentation"
 
 function buildNano(couchInfo: { url: string; cookie: string }) {
   return Nano({
@@ -36,8 +35,7 @@ export function DatabaseWithConnection(
   connection: string,
   opts?: DatabaseOpts
 ) {
-  const db = new DatabaseImpl(dbName, opts, connection)
-  return new DDInstrumentedDatabase(db)
+  return new DatabaseImpl(dbName, opts, connection)
 }
 
 export class DatabaseImpl implements Database {
diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts
index 197770298e..3e69d49f0e 100644
--- a/packages/backend-core/src/db/db.ts
+++ b/packages/backend-core/src/db/db.ts
@@ -1,9 +1,8 @@
 import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types"
-import { DDInstrumentedDatabase } from "./instrumentation"
 
 export function getDB(dbName: string, opts?: DatabaseOpts): Database {
-  return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts))
+  return new DatabaseImpl(dbName, opts)
 }
 
 // we have to use a callback for this so that we can close
diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
deleted file mode 100644
index ba5febcba6..0000000000
--- a/packages/backend-core/src/db/instrumentation.ts
+++ /dev/null
@@ -1,156 +0,0 @@
-import {
-  DocumentScope,
-  DocumentDestroyResponse,
-  DocumentInsertResponse,
-  DocumentBulkResponse,
-  OkResponse,
-} from "@budibase/nano"
-import {
-  AllDocsResponse,
-  AnyDocument,
-  Database,
-  DatabaseDumpOpts,
-  DatabasePutOpts,
-  DatabaseQueryOpts,
-  Document,
-} from "@budibase/types"
-import tracer from "dd-trace"
-import { Writable } from "stream"
-
-export class DDInstrumentedDatabase implements Database {
-  constructor(private readonly db: Database) {}
-
-  get name(): string {
-    return this.db.name
-  }
-
-  exists(): Promise<boolean> {
-    return tracer.trace("db.exists", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.exists()
-    })
-  }
-
-  checkSetup(): Promise<DocumentScope<any>> {
-    return tracer.trace("db.checkSetup", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.checkSetup()
-    })
-  }
-
-  get<T extends Document>(id?: string | undefined): Promise<T> {
-    return tracer.trace("db.get", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
-      return this.db.get(id)
-    })
-  }
-
-  getMultiple<T extends Document>(
-    ids: string[],
-    opts?: { allowMissing?: boolean | undefined } | undefined
-  ): Promise<T[]> {
-    return tracer.trace("db.getMultiple", span => {
-      span?.addTags({
-        db_name: this.name,
-        num_docs: ids.length,
-        allow_missing: opts?.allowMissing,
-      })
-      return this.db.getMultiple(ids, opts)
-    })
-  }
-
-  remove(
-    id: string | Document,
-    rev?: string | undefined
-  ): Promise<DocumentDestroyResponse> {
-    return tracer.trace("db.remove", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
-      return this.db.remove(id, rev)
-    })
-  }
-
-  put(
-    document: AnyDocument,
-    opts?: DatabasePutOpts | undefined
-  ): Promise<DocumentInsertResponse> {
-    return tracer.trace("db.put", span => {
-      span?.addTags({ db_name: this.name, doc_id: document._id })
-      return this.db.put(document, opts)
-    })
-  }
-
-  bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
-    return tracer.trace("db.bulkDocs", span => {
-      span?.addTags({ db_name: this.name, num_docs: documents.length })
-      return this.db.bulkDocs(documents)
-    })
-  }
-
-  allDocs<T extends Document>(
-    params: DatabaseQueryOpts
-  ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("db.allDocs", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.allDocs(params)
-    })
-  }
-
-  query<T extends Document>(
-    viewName: string,
-    params: DatabaseQueryOpts
-  ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("db.query", span => {
-      span?.addTags({ db_name: this.name, view_name: viewName })
-      return this.db.query(viewName, params)
-    })
-  }
-
-  destroy(): Promise<void | OkResponse> {
-    return tracer.trace("db.destroy", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.destroy()
-    })
-  }
-
-  compact(): Promise<void | OkResponse> {
-    return tracer.trace("db.compact", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.compact()
-    })
-  }
-
-  dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
-    return tracer.trace("db.dump", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.dump(stream, opts)
-    })
-  }
-
-  load(...args: any[]): Promise<any> {
-    return tracer.trace("db.load", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.load(...args)
-    })
-  }
-
-  createIndex(...args: any[]): Promise<any> {
-    return tracer.trace("db.createIndex", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.createIndex(...args)
-    })
-  }
-
-  deleteIndex(...args: any[]): Promise<any> {
-    return tracer.trace("db.deleteIndex", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.deleteIndex(...args)
-    })
-  }
-
-  getIndexes(...args: any[]): Promise<any> {
-    return tracer.trace("db.getIndexes", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.getIndexes(...args)
-    })
-  }
-}
diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts
index 7a051e7f12..ad68bd300d 100644
--- a/packages/backend-core/src/logging/pino/logger.ts
+++ b/packages/backend-core/src/logging/pino/logger.ts
@@ -5,7 +5,6 @@ import { IdentityType } from "@budibase/types"
 import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-import tracer from "dd-trace"
 import { formats } from "dd-trace/ext"
 
 import { localFileDestination } from "../system"
@@ -117,11 +116,6 @@ if (!env.DISABLE_PINO_LOGGER) {
       correlationId: correlation.getId(),
     }
 
-    const span = tracer.scope().active()
-    if (span) {
-      tracer.inject(span.context(), formats.LOG, contextObject)
-    }
-
     const mergingObject: any = {
       err: error,
       pid: process.pid,
diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index 0c28787f67..04fd36e3d1 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -16,7 +16,6 @@ import {
 } from "@budibase/types"
 import sdk from "../sdk"
 import { automationsEnabled } from "../features"
-import tracer from "dd-trace"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -40,62 +39,27 @@ function loggingArgs(job: AutomationJob) {
 }
 
 export async function processEvent(job: AutomationJob) {
-  return tracer.trace(
-    "processEvent",
-    { resource: "automation" },
-    async span => {
-      const appId = job.data.event.appId!
-      const automationId = job.data.automation._id!
+  const appId = job.data.event.appId!
+  const automationId = job.data.automation._id!
 
-      span?.addTags({
-        appId,
+  const task = async () => {
+    try {
+      // need to actually await these so that an error can be captured properly
+      console.log("automation running", ...loggingArgs(job))
+
+      const runFn = () => Runner.run(job)
+      const result = await quotas.addAutomation(runFn, {
         automationId,
-        job: {
-          id: job.id,
-          name: job.name,
-          attemptsMade: job.attemptsMade,
-          opts: {
-            attempts: job.opts.attempts,
-            priority: job.opts.priority,
-            delay: job.opts.delay,
-            repeat: job.opts.repeat,
-            backoff: job.opts.backoff,
-            lifo: job.opts.lifo,
-            timeout: job.opts.timeout,
-            jobId: job.opts.jobId,
-            removeOnComplete: job.opts.removeOnComplete,
-            removeOnFail: job.opts.removeOnFail,
-            stackTraceLimit: job.opts.stackTraceLimit,
-            preventParsingData: job.opts.preventParsingData,
-          },
-        },
       })
-
-      const task = async () => {
-        try {
-          // need to actually await these so that an error can be captured properly
-          console.log("automation running", ...loggingArgs(job))
-
-          const runFn = () => Runner.run(job)
-          const result = await quotas.addAutomation(runFn, {
-            automationId,
-          })
-          console.log("automation completed", ...loggingArgs(job))
-          return result
-        } catch (err) {
-          span?.addTags({ error: true })
-          console.error(
-            `automation was unable to run`,
-            err,
-            ...loggingArgs(job)
-          )
-          return { err }
-        }
-      }
-
-      return await context.doInAutomationContext({ appId, automationId, task })
+      console.log("automation completed", ...loggingArgs(job))
+      return result
+    } catch (err) {
+      console.error(`automation was unable to run`, err, ...loggingArgs(job))
+      return { err }
     }
-  )
+  }
+
+  return await context.doInAutomationContext({ appId, automationId, task })
 }
 
 export async function updateTestHistory(
diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts
index ab0381a399..d6472c0e36 100644
--- a/packages/server/src/jsRunner.ts
+++ b/packages/server/src/jsRunner.ts
@@ -2,44 +2,34 @@ import vm from "vm"
 import env from "./environment"
 import { setJSRunner } from "@budibase/string-templates"
 import { context, timers } from "@budibase/backend-core"
-import tracer from "dd-trace"
-
 type TrackerFn = <T>(f: () => T) => T
 
 export function init() {
   setJSRunner((js: string, ctx: vm.Context) => {
-    return tracer.trace("runJS", {}, span => {
-      const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
-      let track: TrackerFn = f => f()
-      if (perRequestLimit) {
-        const bbCtx = context.getCurrentContext()
-        if (bbCtx) {
-          if (!bbCtx.jsExecutionTracker) {
-            bbCtx.jsExecutionTracker =
-              timers.ExecutionTimeTracker.withLimit(perRequestLimit)
-          }
-          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
-          span?.addTags({
-            js: {
-              limitMS: bbCtx.jsExecutionTracker.limitMs,
-              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-            },
-          })
+    const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
+    let track: TrackerFn = f => f()
+    if (perRequestLimit) {
+      const bbCtx = context.getCurrentContext()
+      if (bbCtx) {
+        if (!bbCtx.jsExecutionTracker) {
+          bbCtx.jsExecutionTracker =
+            timers.ExecutionTimeTracker.withLimit(perRequestLimit)
         }
+        track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
       }
+    }
 
-      ctx = {
-        ...ctx,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
-      vm.createContext(ctx)
-      return track(() =>
-        vm.runInNewContext(js, ctx, {
-          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-        })
-      )
-    })
+    ctx = {
+      ...ctx,
+      alert: undefined,
+      setInterval: undefined,
+      setTimeout: undefined,
+    }
+    vm.createContext(ctx)
+    return track(() =>
+      vm.runInNewContext(js, ctx, {
+        timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+      })
+    )
   })
 }
diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts
index ad6f2afa18..984dd8e5e9 100644
--- a/packages/server/src/middleware/currentapp.ts
+++ b/packages/server/src/middleware/currentapp.ts
@@ -12,7 +12,6 @@ import { getCachedSelf } from "../utilities/global"
 import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
 import { UserCtx, ContextUser } from "@budibase/types"
-import tracer from "dd-trace"
 
 export default async (ctx: UserCtx, next: any) => {
   // try to get the appID from the request
@@ -21,11 +20,6 @@ export default async (ctx: UserCtx, next: any) => {
     return next()
   }
 
-  if (requestAppId) {
-    const span = tracer.scope().active()
-    span?.setTag("appId", requestAppId)
-  }
-
   // deny access to application preview
   if (!env.isTest()) {
     if (
@@ -76,14 +70,6 @@ export default async (ctx: UserCtx, next: any) => {
     return next()
   }
 
-  if (ctx.user) {
-    const span = tracer.scope().active()
-    if (ctx.user._id) {
-      span?.setTag("userId", ctx.user._id)
-    }
-    span?.setTag("tenantId", ctx.user.tenantId)
-  }
-
   const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined
 
   // if the user is not in the right tenant then make sure to wipe their cookie
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 4447899f96..9bb1717f3c 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -34,7 +34,6 @@ import { cloneDeep } from "lodash/fp"
 import { performance } from "perf_hooks"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
-import tracer from "dd-trace"
 
 threadUtils.threadSetup()
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
@@ -243,347 +242,281 @@ class Orchestrator {
   }
 
   async execute(): Promise<any> {
-    return tracer.trace(
-      "Orchestrator.execute",
-      { resource: "automation" },
-      async span => {
-        span?.addTags({
-          appId: this._appId,
-          automationId: this._automation._id,
-        })
+    // this will retrieve from context created at start of thread
+    this._context.env = await sdkUtils.getEnvironmentVariables()
+    let automation = this._automation
+    let stopped = false
+    let loopStep: AutomationStep | undefined = undefined
 
-        // this will retrieve from context created at start of thread
-        this._context.env = await sdkUtils.getEnvironmentVariables()
-        let automation = this._automation
-        let stopped = false
-        let loopStep: AutomationStep | undefined = undefined
+    let stepCount = 0
+    let loopStepNumber: any = undefined
+    let loopSteps: LoopStep[] | undefined = []
+    let metadata
+    let timeoutFlag = false
+    let wasLoopStep = false
+    let timeout = this._job.data.event.timeout
+    // check if this is a recurring automation,
+    if (isProdAppID(this._appId) && isRecurring(automation)) {
+      metadata = await this.getMetadata()
+      const shouldStop = await this.checkIfShouldStop(metadata)
+      if (shouldStop) {
+        return
+      }
+    }
+    const start = performance.now()
+    for (let step of automation.definition.steps) {
+      let input: any,
+        iterations = 1,
+        iterationCount = 0
 
-        let stepCount = 0
-        let loopStepNumber: any = undefined
-        let loopSteps: LoopStep[] | undefined = []
-        let metadata
-        let timeoutFlag = false
-        let wasLoopStep = false
-        let timeout = this._job.data.event.timeout
-        // check if this is a recurring automation,
-        if (isProdAppID(this._appId) && isRecurring(automation)) {
-          span?.addTags({ recurring: true })
-          metadata = await this.getMetadata()
-          const shouldStop = await this.checkIfShouldStop(metadata)
-          if (shouldStop) {
-            span?.addTags({ shouldStop: true })
-            return
+      if (timeoutFlag) {
+        break
+      }
+
+      if (timeout) {
+        setTimeout(() => {
+          timeoutFlag = true
+        }, timeout || 12000)
+      }
+
+      stepCount++
+      if (step.stepId === LOOP_STEP_ID) {
+        loopStep = step
+        loopStepNumber = stepCount
+        continue
+      }
+
+      if (loopStep) {
+        input = await processObject(loopStep.inputs, this._context)
+        iterations = getLoopIterations(loopStep as LoopStep)
+      }
+      for (let index = 0; index < iterations; index++) {
+        let originalStepInput = cloneDeep(step.inputs)
+        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+        if (loopStep && input.binding) {
+          let tempOutput = {
+            items: loopSteps,
+            iterations: iterationCount,
           }
-        }
-        const start = performance.now()
-        for (let step of automation.definition.steps) {
-          const stepSpan = tracer.startSpan("Orchestrator.execute.step", {
-            childOf: span,
-          })
-          stepSpan.addTags({
-            resource: "automation",
-            step: {
-              stepId: step.stepId,
-              id: step.id,
-              name: step.name,
-              type: step.type,
-              title: step.stepTitle,
-              internal: step.internal,
-              deprecated: step.deprecated,
-            },
-          })
-
-          let input: any,
-            iterations = 1,
-            iterationCount = 0
-
           try {
-            if (timeoutFlag) {
-              span?.addTags({ timedOut: true })
-              break
-            }
+            loopStep.inputs.binding = automationUtils.typecastForLooping(
+              loopStep as LoopStep,
+              loopStep.inputs as LoopInput
+            )
+          } catch (err) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.INCORRECT_TYPE,
+              success: false,
+            })
+            loopSteps = undefined
+            loopStep = undefined
+            break
+          }
+          let item = []
+          if (
+            typeof loopStep.inputs.binding === "string" &&
+            loopStep.inputs.option === "String"
+          ) {
+            item = automationUtils.stringSplit(loopStep.inputs.binding)
+          } else if (Array.isArray(loopStep.inputs.binding)) {
+            item = loopStep.inputs.binding
+          }
+          this._context.steps[loopStepNumber] = {
+            currentItem: item[index],
+          }
 
-            if (timeout) {
-              setTimeout(() => {
-                timeoutFlag = true
-              }, timeout || 12000)
-            }
-
-            stepCount++
-            if (step.stepId === LOOP_STEP_ID) {
-              loopStep = step
-              loopStepNumber = stepCount
-              continue
-            }
-
-            if (loopStep) {
-              input = await processObject(loopStep.inputs, this._context)
-              iterations = getLoopIterations(loopStep as LoopStep)
-              stepSpan?.addTags({ step: { iterations } })
-            }
-            for (let index = 0; index < iterations; index++) {
-              let originalStepInput = cloneDeep(step.inputs)
-              // Handle if the user has set a max iteration count or if it reaches the max limit set by us
-              if (loopStep && input.binding) {
-                let tempOutput = {
-                  items: loopSteps,
-                  iterations: iterationCount,
-                }
-                try {
-                  loopStep.inputs.binding = automationUtils.typecastForLooping(
-                    loopStep as LoopStep,
-                    loopStep.inputs as LoopInput
-                  )
-                } catch (err) {
-                  this.updateContextAndOutput(
-                    loopStepNumber,
-                    step,
-                    tempOutput,
-                    {
-                      status: AutomationErrors.INCORRECT_TYPE,
-                      success: false,
-                    }
-                  )
-                  loopSteps = undefined
-                  loopStep = undefined
-                  break
-                }
-                let item = []
-                if (
-                  typeof loopStep.inputs.binding === "string" &&
-                  loopStep.inputs.option === "String"
-                ) {
-                  item = automationUtils.stringSplit(loopStep.inputs.binding)
-                } else if (Array.isArray(loopStep.inputs.binding)) {
-                  item = loopStep.inputs.binding
-                }
-                this._context.steps[loopStepNumber] = {
-                  currentItem: item[index],
-                }
-
-                // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
-                // Pretty hacky because we need to account for the row object
-                for (let [key, value] of Object.entries(originalStepInput)) {
-                  if (typeof value === "object") {
-                    for (let [innerKey, innerValue] of Object.entries(
-                      originalStepInput[key]
-                    )) {
-                      if (typeof innerValue === "string") {
-                        originalStepInput[key][innerKey] =
-                          automationUtils.substituteLoopStep(
-                            innerValue,
-                            `steps.${loopStepNumber}`
-                          )
-                      } else if (typeof value === "object") {
-                        for (let [innerObject, innerValue] of Object.entries(
-                          originalStepInput[key][innerKey]
-                        )) {
-                          originalStepInput[key][innerKey][innerObject] =
-                            automationUtils.substituteLoopStep(
-                              innerValue as string,
-                              `steps.${loopStepNumber}`
-                            )
-                        }
-                      }
-                    }
-                  } else {
-                    if (typeof value === "string") {
-                      originalStepInput[key] =
-                        automationUtils.substituteLoopStep(
-                          value,
-                          `steps.${loopStepNumber}`
-                        )
-                    }
+          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+          // Pretty hacky because we need to account for the row object
+          for (let [key, value] of Object.entries(originalStepInput)) {
+            if (typeof value === "object") {
+              for (let [innerKey, innerValue] of Object.entries(
+                originalStepInput[key]
+              )) {
+                if (typeof innerValue === "string") {
+                  originalStepInput[key][innerKey] =
+                    automationUtils.substituteLoopStep(
+                      innerValue,
+                      `steps.${loopStepNumber}`
+                    )
+                } else if (typeof value === "object") {
+                  for (let [innerObject, innerValue] of Object.entries(
+                    originalStepInput[key][innerKey]
+                  )) {
+                    originalStepInput[key][innerKey][innerObject] =
+                      automationUtils.substituteLoopStep(
+                        innerValue as string,
+                        `steps.${loopStepNumber}`
+                      )
                   }
                 }
-
-                if (
-                  index === env.AUTOMATION_MAX_ITERATIONS ||
-                  index === parseInt(loopStep.inputs.iterations)
-                ) {
-                  this.updateContextAndOutput(
-                    loopStepNumber,
-                    step,
-                    tempOutput,
-                    {
-                      status: AutomationErrors.MAX_ITERATIONS,
-                      success: true,
-                    }
-                  )
-                  loopSteps = undefined
-                  loopStep = undefined
-                  break
-                }
-
-                let isFailure = false
-                const currentItem =
-                  this._context.steps[loopStepNumber]?.currentItem
-                if (currentItem && typeof currentItem === "object") {
-                  isFailure = Object.keys(currentItem).some(value => {
-                    return currentItem[value] === loopStep?.inputs.failure
-                  })
-                } else {
-                  isFailure =
-                    currentItem && currentItem === loopStep.inputs.failure
-                }
-
-                if (isFailure) {
-                  this.updateContextAndOutput(
-                    loopStepNumber,
-                    step,
-                    tempOutput,
-                    {
-                      status: AutomationErrors.FAILURE_CONDITION,
-                      success: false,
-                    }
-                  )
-                  loopSteps = undefined
-                  loopStep = undefined
-                  break
-                }
               }
-
-              // execution stopped, record state for that
-              if (stopped) {
-                this.updateExecutionOutput(
-                  step.id,
-                  step.stepId,
-                  {},
-                  STOPPED_STATUS
+            } else {
+              if (typeof value === "string") {
+                originalStepInput[key] = automationUtils.substituteLoopStep(
+                  value,
+                  `steps.${loopStepNumber}`
                 )
-                continue
-              }
-
-              // If it's a loop step, we need to manually add the bindings to the context
-              let stepFn = await this.getStepFunctionality(step.stepId)
-              let inputs = await processObject(originalStepInput, this._context)
-              inputs = automationUtils.cleanInputValues(
-                inputs,
-                step.schema.inputs
-              )
-
-              try {
-                // appId is always passed
-                const outputs = await stepFn({
-                  inputs: inputs,
-                  appId: this._appId,
-                  emitter: this._emitter,
-                  context: this._context,
-                })
-
-                this._context.steps[stepCount] = outputs
-                // if filter causes us to stop execution don't break the loop, set a var
-                // so that we can finish iterating through the steps and record that it stopped
-                if (step.stepId === FILTER_STEP_ID && !outputs.result) {
-                  stopped = true
-                  this.updateExecutionOutput(
-                    step.id,
-                    step.stepId,
-                    step.inputs,
-                    {
-                      ...outputs,
-                      ...STOPPED_STATUS,
-                    }
-                  )
-                  continue
-                }
-                if (loopStep && loopSteps) {
-                  loopSteps.push(outputs)
-                } else {
-                  this.updateExecutionOutput(
-                    step.id,
-                    step.stepId,
-                    step.inputs,
-                    outputs
-                  )
-                }
-              } catch (err) {
-                console.error(`Automation error - ${step.stepId} - ${err}`)
-                return err
-              }
-
-              if (loopStep) {
-                iterationCount++
-                if (index === iterations - 1) {
-                  loopStep = undefined
-                  this._context.steps.splice(loopStepNumber, 1)
-                  break
-                }
               }
             }
-          } finally {
-            stepSpan?.finish()
           }
 
-          if (loopStep && iterations === 0) {
-            loopStep = undefined
-            this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-              id: step.id,
-              stepId: step.stepId,
-              outputs: {
-                status: AutomationStepStatus.NO_ITERATIONS,
-                success: true,
-              },
-              inputs: {},
-            })
-
-            this._context.steps.splice(loopStepNumber, 1)
-            iterations = 1
-          }
-
-          // Delete the step after the loop step as it's irrelevant, since information is included
-          // in the loop step
-          if (wasLoopStep && !loopStep) {
-            this._context.steps.splice(loopStepNumber + 1, 1)
-            wasLoopStep = false
-          }
-          if (loopSteps && loopSteps.length) {
-            let tempOutput = {
+          if (
+            index === env.AUTOMATION_MAX_ITERATIONS ||
+            index === parseInt(loopStep.inputs.iterations)
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.MAX_ITERATIONS,
               success: true,
-              items: loopSteps,
-              iterations: iterationCount,
-            }
-            this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-              id: step.id,
-              stepId: step.stepId,
-              outputs: tempOutput,
-              inputs: step.inputs,
             })
-            this._context.steps[loopStepNumber] = tempOutput
+            loopSteps = undefined
+            loopStep = undefined
+            break
+          }
 
-            wasLoopStep = true
-            loopSteps = []
+          let isFailure = false
+          const currentItem = this._context.steps[loopStepNumber]?.currentItem
+          if (currentItem && typeof currentItem === "object") {
+            isFailure = Object.keys(currentItem).some(value => {
+              return currentItem[value] === loopStep?.inputs.failure
+            })
+          } else {
+            isFailure = currentItem && currentItem === loopStep.inputs.failure
+          }
+
+          if (isFailure) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.FAILURE_CONDITION,
+              success: false,
+            })
+            loopSteps = undefined
+            loopStep = undefined
+            break
           }
         }
 
-        const end = performance.now()
-        const executionTime = end - start
+        // execution stopped, record state for that
+        if (stopped) {
+          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+          continue
+        }
 
-        console.info(
-          `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
-          {
-            _logKey: "automation",
-            executionTime,
-          }
-        )
+        // If it's a loop step, we need to manually add the bindings to the context
+        let stepFn = await this.getStepFunctionality(step.stepId)
+        let inputs = await processObject(originalStepInput, this._context)
+        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
 
-        // store the logs for the automation run
         try {
-          await storeLog(this._automation, this.executionOutput)
-        } catch (e: any) {
-          if (e.status === 413 && e.request?.data) {
-            // if content is too large we shouldn't log it
-            delete e.request.data
-            e.request.data = { message: "removed due to large size" }
+          // appId is always passed
+          const outputs = await stepFn({
+            inputs: inputs,
+            appId: this._appId,
+            emitter: this._emitter,
+            context: this._context,
+          })
+
+          this._context.steps[stepCount] = outputs
+          // if filter causes us to stop execution don't break the loop, set a var
+          // so that we can finish iterating through the steps and record that it stopped
+          if (step.stepId === FILTER_STEP_ID && !outputs.result) {
+            stopped = true
+            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+              ...outputs,
+              ...STOPPED_STATUS,
+            })
+            continue
           }
-          logging.logAlert("Error writing automation log", e)
+          if (loopStep && loopSteps) {
+            loopSteps.push(outputs)
+          } else {
+            this.updateExecutionOutput(
+              step.id,
+              step.stepId,
+              step.inputs,
+              outputs
+            )
+          }
+        } catch (err) {
+          console.error(`Automation error - ${step.stepId} - ${err}`)
+          return err
         }
-        if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
-          await this.updateMetadata(metadata)
+
+        if (loopStep) {
+          iterationCount++
+          if (index === iterations - 1) {
+            loopStep = undefined
+            this._context.steps.splice(loopStepNumber, 1)
+            break
+          }
         }
-        return this.executionOutput
+      }
+
+      if (loopStep && iterations === 0) {
+        loopStep = undefined
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: {
+            status: AutomationStepStatus.NO_ITERATIONS,
+            success: true,
+          },
+          inputs: {},
+        })
+
+        this._context.steps.splice(loopStepNumber, 1)
+        iterations = 1
+      }
+
+      // Delete the step after the loop step as it's irrelevant, since information is included
+      // in the loop step
+      if (wasLoopStep && !loopStep) {
+        this._context.steps.splice(loopStepNumber + 1, 1)
+        wasLoopStep = false
+      }
+      if (loopSteps && loopSteps.length) {
+        let tempOutput = {
+          success: true,
+          items: loopSteps,
+          iterations: iterationCount,
+        }
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: tempOutput,
+          inputs: step.inputs,
+        })
+        this._context.steps[loopStepNumber] = tempOutput
+
+        wasLoopStep = true
+        loopSteps = []
+      }
+    }
+
+    const end = performance.now()
+    const executionTime = end - start
+
+    console.info(
+      `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
+      {
+        _logKey: "automation",
+        executionTime,
       }
     )
+
+    // store the logs for the automation run
+    try {
+      await storeLog(this._automation, this.executionOutput)
+    } catch (e: any) {
+      if (e.status === 413 && e.request?.data) {
+        // if content is too large we shouldn't log it
+        delete e.request.data
+        e.request.data = { message: "removed due to large size" }
+      }
+      logging.logAlert("Error writing automation log", e)
+    }
+    if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
+      await this.updateMetadata(metadata)
+    }
+    return this.executionOutput
   }
 }
 
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index cafd366cae..22c099c58c 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -11,7 +11,6 @@ import {
   Row,
   Table,
 } from "@budibase/types"
-import tracer from "dd-trace"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -51,42 +50,35 @@ export function processFormulas<T extends Row | Row[]>(
   inputRows: T,
   { dynamic, contextRows }: FormulaOpts = { dynamic: true }
 ): T {
-  return tracer.trace("processFormulas", {}, span => {
-    const numRows = Array.isArray(inputRows) ? inputRows.length : 1
-    span?.addTags({ table_id: table._id, dynamic, numRows })
-    const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
-    if (rows) {
-      for (let [column, schema] of Object.entries(table.schema)) {
-        if (schema.type !== FieldTypes.FORMULA) {
-          continue
-        }
+  const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+  if (rows) {
+    for (let [column, schema] of Object.entries(table.schema)) {
+      if (schema.type !== FieldTypes.FORMULA) {
+        continue
+      }
 
-        const isStatic = schema.formulaType === FormulaTypes.STATIC
+      const isStatic = schema.formulaType === FormulaTypes.STATIC
 
-        if (
-          schema.formula == null ||
-          (dynamic && isStatic) ||
-          (!dynamic && !isStatic)
-        ) {
-          continue
-        }
-        // iterate through rows and process formula
-        for (let i = 0; i < rows.length; i++) {
-          let row = rows[i]
-          let context = contextRows ? contextRows[i] : row
-          let formula = schema.formula
-          rows[i] = {
-            ...row,
-            [column]: tracer.trace("processStringSync", {}, span => {
-              span?.addTags({ table_id: table._id, column, static: isStatic })
-              return processStringSync(formula, context)
-            }),
-          }
+      if (
+        schema.formula == null ||
+        (dynamic && isStatic) ||
+        (!dynamic && !isStatic)
+      ) {
+        continue
+      }
+      // iterate through rows and process formula
+      for (let i = 0; i < rows.length; i++) {
+        let row = rows[i]
+        let context = contextRows ? contextRows[i] : row
+        let formula = schema.formula
+        rows[i] = {
+          ...row,
+          [column]: processStringSync(formula, context),
         }
       }
     }
-    return Array.isArray(inputRows) ? rows : rows[0]
-  })
+  }
+  return Array.isArray(inputRows) ? rows : rows[0]
 }
 
 /**

From df73cbd0018cd643cb8fb46a6e0ad8e56e766371 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 21 Dec 2023 11:13:39 +0000
Subject: [PATCH 22/68] Fix lint warning.

---
 packages/server/src/jsRunner.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts
index d6472c0e36..a9301feb60 100644
--- a/packages/server/src/jsRunner.ts
+++ b/packages/server/src/jsRunner.ts
@@ -2,6 +2,7 @@ import vm from "vm"
 import env from "./environment"
 import { setJSRunner } from "@budibase/string-templates"
 import { context, timers } from "@budibase/backend-core"
+
 type TrackerFn = <T>(f: () => T) => T
 
 export function init() {

From 8e241c657fde5a2e390824978f18c4d7811756ae Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 21 Dec 2023 15:46:28 +0000
Subject: [PATCH 23/68] Bump version to 2.13.50

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 8aef7c70ec..671935c34b 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.49",
+  "version": "2.13.50",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 258ac0c8dabd3e6689dc661c89278f0d5745d352 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 12:45:30 +0100
Subject: [PATCH 24/68] Update submodule ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index cb04efac5a..0da3f58c4b 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit cb04efac5a22cf5f68e921e807d1fc500078dbd7
+Subproject commit 0da3f58c4b52fff4ab2705ac761bd70cc76f6b4b

From 3d22276c2b6da4e351b7c1bf7c20667bdff36d4b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 13:47:39 +0100
Subject: [PATCH 25/68] Update submodule ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 0da3f58c4b..e46a352a63 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 0da3f58c4b52fff4ab2705ac761bd70cc76f6b4b
+Subproject commit e46a352a6326a838faa00f912de069aee95d7300

From 1b0c551602cb1841489e354a1ca0bf211e3476d6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 16:59:23 +0100
Subject: [PATCH 26/68] Create create-many script

---
 .../scripts/load/create-many-relationships.js | 47 +++++++++++++++++++
 packages/server/scripts/load/utils.js         | 43 ++++++++++-------
 2 files changed, 73 insertions(+), 17 deletions(-)
 create mode 100755 packages/server/scripts/load/create-many-relationships.js

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
new file mode 100755
index 0000000000..78353c029c
--- /dev/null
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -0,0 +1,47 @@
+#!/bin/node
+const { createApp, getTable, createRow, createTable } = require("./utils")
+
+const ROW_COUNT = 10
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const app = await createApp(apiKey)
+  console.log(`App created: ${app._id}`)
+  const table = await getTable(apiKey, app._id)
+  console.log(`Table found: ${table.name}`)
+
+  const subjectTable = await createTable(apiKey, app._id, {
+    schema: {
+      title: {
+        name: "Name",
+        type: "string",
+      },
+    },
+    name: "Subjects",
+  })
+  for (let i = 0; i < 10; i++) {
+    await createRow(apiKey, app._id, subjectTable)
+  }
+
+  const promises = []
+  for (let i = 0; i < ROW_COUNT; i++) {
+    promises.push(await createRow(apiKey, app._id, table))
+    console.log(`Row ${i + 1} of ${ROW_COUNT} created`)
+  }
+  await Promise.all(promises)
+
+  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${ROW_COUNT} rows.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 97ff8f1e13..7af867ab1d 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -2,7 +2,8 @@ const fetch = require("node-fetch")
 const uuid = require("uuid/v4")
 
 const URL_APP = "http://localhost:10000/api/public/v1/applications"
-const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search"
+const URL_TABLE = "http://localhost:10000/api/public/v1/tables"
+const URL_SEARCH_TABLE = "http://localhost:10000/api/public/v1/tables/search"
 
 async function request(apiKey, url, method, body, appId = undefined) {
   const headers = {
@@ -38,29 +39,37 @@ exports.createApp = async apiKey => {
 }
 
 exports.getTable = async (apiKey, appId) => {
-  const res = await request(apiKey, URL_TABLE, "POST", {}, appId)
+  const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId)
   const json = await res.json()
   return json.data[0]
 }
 
-exports.createRow = async (apiKey, appId, table) => {
-  const body = {}
-  for (let [key, schema] of Object.entries(table.schema)) {
-    let fake
-    switch (schema.type) {
-      default:
-      case "string":
-        fake = schema.constraints.inclusion
-          ? schema.constraints.inclusion[0]
-          : "a"
-        break
-      case "number":
-        fake = 1
-        break
+exports.createRow = async (apiKey, appId, table, body) => {
+  if (!body) {
+    body = {}
+    for (let [key, schema] of Object.entries(table.schema)) {
+      let fake
+      switch (schema.type) {
+        default:
+        case "string":
+          fake = schema.constraints?.inclusion
+            ? schema.constraints.inclusion[0]
+            : "a"
+          break
+        case "number":
+          fake = 1
+          break
+      }
+      body[key] = fake
     }
-    body[key] = fake
   }
   const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows`
   const res = await request(apiKey, url, "POST", body, appId)
   return (await res.json()).data
 }
+
+exports.createTable = async (apiKey, appId, config) => {
+  const res = await request(apiKey, URL_TABLE, "POST", config, appId)
+  const json = await res.json()
+  return json.data
+}

From 1b6c7f27297f72b28f67966b5d10fde117c27506 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 11:51:35 +0100
Subject: [PATCH 27/68] Add content to subjects

---
 .../scripts/load/create-many-relationships.js  | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 78353c029c..1589c1111e 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -1,6 +1,9 @@
 #!/bin/node
 const { createApp, getTable, createRow, createTable } = require("./utils")
 
+const Chance = require("chance")
+const generator = new Chance()
+
 const ROW_COUNT = 10
 
 if (!process.argv[2]) {
@@ -17,23 +20,26 @@ async function run() {
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
-      title: {
+      Name: {
         name: "Name",
         type: "string",
       },
     },
     name: "Subjects",
   })
-  for (let i = 0; i < 10; i++) {
-    await createRow(apiKey, app._id, subjectTable)
+
+  const SUBJECT_COUNT = 10
+  for (let i = 0; i < SUBJECT_COUNT; i++) {
+    await createRow(apiKey, app._id, subjectTable, {
+      Name: generator.profession(),
+    })
+    console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
   }
 
-  const promises = []
   for (let i = 0; i < ROW_COUNT; i++) {
-    promises.push(await createRow(apiKey, app._id, table))
+    await createRow(apiKey, app._id, table)
     console.log(`Row ${i + 1} of ${ROW_COUNT} created`)
   }
-  await Promise.all(promises)
 
   console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
 }

From e986374c23618703e6fc3dce7b00490ac65808b6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 12:18:29 +0100
Subject: [PATCH 28/68] Check fetched table

---
 .../scripts/load/create-many-relationships.js | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 1589c1111e..926d0ac342 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -4,19 +4,27 @@ const { createApp, getTable, createRow, createTable } = require("./utils")
 const Chance = require("chance")
 const generator = new Chance()
 
-const ROW_COUNT = 10
+const STUDENT_COUNT = 10
+const SUBJECT_COUNT = 10
 
 if (!process.argv[2]) {
   console.error("Please specify an API key as script argument.")
   process.exit(-1)
 }
 
+async function sleep(ms) {
+  return new Promise(r => setTimeout(() => r(), ms))
+}
+
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
   console.log(`App created: ${app._id}`)
-  const table = await getTable(apiKey, app._id)
-  console.log(`Table found: ${table.name}`)
+  const studentsTable = await getTable(apiKey, app._id)
+  if (studentsTable.name !== "Students") {
+    throw 'Fetched table should be "Students"'
+  }
+  console.log(`Table found: ${studentsTable.name}`)
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
@@ -28,17 +36,17 @@ async function run() {
     name: "Subjects",
   })
 
-  const SUBJECT_COUNT = 10
   for (let i = 0; i < SUBJECT_COUNT; i++) {
     await createRow(apiKey, app._id, subjectTable, {
       Name: generator.profession(),
     })
     console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
+    await sleep(50)
   }
 
-  for (let i = 0; i < ROW_COUNT; i++) {
-    await createRow(apiKey, app._id, table)
-    console.log(`Row ${i + 1} of ${ROW_COUNT} created`)
+  for (let i = 0; i < STUDENT_COUNT; i++) {
+    await createRow(apiKey, app._id, studentsTable)
+    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
   }
 
   console.log(`App created: http://localhost:10000/builder/app/${app._id}`)

From e4acd91237a0cb56b849d85cdeab162b2701df4c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 12:30:35 +0100
Subject: [PATCH 29/68] Populate rows

---
 .../scripts/load/create-many-relationships.js | 29 +++++++++++--------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 926d0ac342..374dda58c2 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -4,7 +4,7 @@ const { createApp, getTable, createRow, createTable } = require("./utils")
 const Chance = require("chance")
 const generator = new Chance()
 
-const STUDENT_COUNT = 10
+const STUDENT_COUNT = 100
 const SUBJECT_COUNT = 10
 
 if (!process.argv[2]) {
@@ -19,7 +19,8 @@ async function sleep(ms) {
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
-  console.log(`App created: ${app._id}`)
+  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
+
   const studentsTable = await getTable(apiKey, app._id)
   if (studentsTable.name !== "Students") {
     throw 'Fetched table should be "Students"'
@@ -44,18 +45,22 @@ async function run() {
     await sleep(50)
   }
 
+  let studentNumber = studentsTable.schema["Auto ID"].lastID
   for (let i = 0; i < STUDENT_COUNT; i++) {
-    await createRow(apiKey, app._id, studentsTable)
+    await createRow(apiKey, app._id, studentsTable, {
+      "Student Number": (++studentNumber).toString(),
+      "First Name": generator.first(),
+      "Last Name": generator.last(),
+      Gender: generator.pickone(["M", "F"]),
+      Grade: generator.pickone(["8", "9", "10", "11"]),
+      "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
+      "Home Number": generator.phone(),
+      "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
+    })
     console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
   }
-
-  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
 }
 
-run()
-  .then(() => {
-    console.log(`Finished creating ${ROW_COUNT} rows.`)
-  })
-  .catch(err => {
-    console.error(err)
-  })
+run().catch(err => {
+  console.error(err)
+})

From 6d6100eaf40f552b46f1bf94f4229dda2526024c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 12:52:25 +0100
Subject: [PATCH 30/68] Add grades

---
 .../scripts/load/create-many-relationships.js | 79 ++++++++++++++++---
 1 file changed, 67 insertions(+), 12 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 374dda58c2..5d0cf3046a 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -27,6 +27,26 @@ async function run() {
   }
   console.log(`Table found: ${studentsTable.name}`)
 
+  const students = [],
+    subjects = []
+
+  let studentNumber = studentsTable.schema["Auto ID"].lastID
+  for (let i = 0; i < STUDENT_COUNT; i++) {
+    students.push(
+      await createRow(apiKey, app._id, studentsTable, {
+        "Student Number": (++studentNumber).toString(),
+        "First Name": generator.first(),
+        "Last Name": generator.last(),
+        Gender: generator.pickone(["M", "F"]),
+        Grade: generator.pickone(["8", "9", "10", "11"]),
+        "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
+        "Home Number": generator.phone(),
+        "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
+      })
+    )
+    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
+  }
+
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
       Name: {
@@ -35,29 +55,64 @@ async function run() {
       },
     },
     name: "Subjects",
+    primaryDisplay: "Name",
   })
 
   for (let i = 0; i < SUBJECT_COUNT; i++) {
+    subjects.push(
     await createRow(apiKey, app._id, subjectTable, {
       Name: generator.profession(),
     })
+    )
     console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
     await sleep(50)
   }
 
-  let studentNumber = studentsTable.schema["Auto ID"].lastID
-  for (let i = 0; i < STUDENT_COUNT; i++) {
-    await createRow(apiKey, app._id, studentsTable, {
-      "Student Number": (++studentNumber).toString(),
-      "First Name": generator.first(),
-      "Last Name": generator.last(),
-      Gender: generator.pickone(["M", "F"]),
-      Grade: generator.pickone(["8", "9", "10", "11"]),
-      "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
-      "Home Number": generator.phone(),
-      "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
+  const gradesTable = await createTable(apiKey, app._id, {
+    schema: {
+      Score: {
+        name: "Score",
+        type: "number",
+      },
+      Student: {
+        name: "Student",
+        tableId: studentsTable._id,
+        constraints: {
+          presence: true,
+          type: "array",
+        },
+        fieldName: "Grades",
+        relationshipType: "one-to-many",
+        type: "link",
+      },
+      Subject: {
+        name: "Subject",
+        tableId: subjectTable._id,
+        constraints: {
+          presence: true,
+          type: "array",
+        },
+        fieldName: "Grades",
+        relationshipType: "one-to-many",
+        type: "link",
+      },
+    },
+    name: "Grades",
+  })
+
+  let i = 0
+  for (const student of students) {
+    for (const subject of subjects) {
+      await createRow(apiKey, app._id, gradesTable, {
+        Score: generator.integer({ min: 0, max: 100 }),
+        Student: [student],
+        Subject: [subject],
     })
-    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
+      console.log(
+        `Grade ${++i} of ${students.length * subjects.length} created`
+      )
+      await sleep(20)
+    }
   }
 }
 

From 192d980e53d7e6e9c2c215de6e8857436661976a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 13:04:16 +0100
Subject: [PATCH 31/68] Log timings

---
 .../scripts/load/create-many-relationships.js | 38 +++++++++++++------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 5d0cf3046a..1c146ccaa0 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -4,7 +4,7 @@ const { createApp, getTable, createRow, createTable } = require("./utils")
 const Chance = require("chance")
 const generator = new Chance()
 
-const STUDENT_COUNT = 100
+const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
 if (!process.argv[2]) {
@@ -16,6 +16,8 @@ async function sleep(ms) {
   return new Promise(r => setTimeout(() => r(), ms))
 }
 
+const start = Date.now()
+
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
@@ -44,7 +46,11 @@ async function run() {
         "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
       })
     )
-    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
+    console.log(
+      `Student ${i + 1} of ${STUDENT_COUNT} created (${
+        (Date.now() - start) / 1000
+      }s)`
+    )
   }
 
   const subjectTable = await createTable(apiKey, app._id, {
@@ -60,11 +66,15 @@ async function run() {
 
   for (let i = 0; i < SUBJECT_COUNT; i++) {
     subjects.push(
-    await createRow(apiKey, app._id, subjectTable, {
-      Name: generator.profession(),
-    })
+      await createRow(apiKey, app._id, subjectTable, {
+        Name: generator.profession(),
+      })
+    )
+    console.log(
+      `Subject ${i + 1} of ${SUBJECT_COUNT} created (${
+        (Date.now() - start) / 1000
+      }s)`
     )
-    console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
     await sleep(50)
   }
 
@@ -107,15 +117,21 @@ async function run() {
         Score: generator.integer({ min: 0, max: 100 }),
         Student: [student],
         Subject: [subject],
-    })
+      })
       console.log(
-        `Grade ${++i} of ${students.length * subjects.length} created`
+        `Grade ${++i} of ${students.length * subjects.length} created (${
+          (Date.now() - start) / 1000
+        }s)`
       )
       await sleep(20)
     }
   }
 }
 
-run().catch(err => {
-  console.error(err)
-})
+run()
+  .then(() => {
+    console.log(`Done in(${(Date.now() - start) / 1000} seconds`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

From 5240b04a3a73997842d904c13dd9add656722da4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 13:09:02 +0100
Subject: [PATCH 32/68] Remove 429 in dev

---
 packages/server/scripts/load/create-many-relationships.js | 6 ------
 packages/server/src/api/routes/public/index.ts            | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 1c146ccaa0..ee05cc8fba 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -12,10 +12,6 @@ if (!process.argv[2]) {
   process.exit(-1)
 }
 
-async function sleep(ms) {
-  return new Promise(r => setTimeout(() => r(), ms))
-}
-
 const start = Date.now()
 
 async function run() {
@@ -75,7 +71,6 @@ async function run() {
         (Date.now() - start) / 1000
       }s)`
     )
-    await sleep(50)
   }
 
   const gradesTable = await createTable(apiKey, app._id, {
@@ -123,7 +118,6 @@ async function run() {
           (Date.now() - start) / 1000
         }s)`
       )
-      await sleep(20)
     }
   }
 }
diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts
index f27f3f8857..36e0f74bee 100644
--- a/packages/server/src/api/routes/public/index.ts
+++ b/packages/server/src/api/routes/public/index.ts
@@ -77,7 +77,7 @@ const publicRouter = new Router({
   prefix: PREFIX,
 })
 
-if (limiter) {
+if (limiter && !env.isDev()) {
   publicRouter.use(limiter)
 }
 

From 57a7be7a266d7d49379166766805ab8a05523051 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 15:02:21 +0100
Subject: [PATCH 33/68] Run creation in parallel

---
 .../scripts/load/create-many-relationships.js | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index ee05cc8fba..49d4efb28d 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -25,12 +25,11 @@ async function run() {
   }
   console.log(`Table found: ${studentsTable.name}`)
 
-  const students = [],
-    subjects = []
-
+  const studentsPromise = []
   let studentNumber = studentsTable.schema["Auto ID"].lastID
   for (let i = 0; i < STUDENT_COUNT; i++) {
-    students.push(
+    studentsPromise.push(
+      (async () => {
       await createRow(apiKey, app._id, studentsTable, {
         "Student Number": (++studentNumber).toString(),
         "First Name": generator.first(),
@@ -41,14 +40,18 @@ async function run() {
         "Home Number": generator.phone(),
         "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
       })
-    )
+
     console.log(
       `Student ${i + 1} of ${STUDENT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
+        )
+      })()
     )
   }
 
+  const students = await Promise.all(studentsPromise)
+
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
       Name: {
@@ -60,19 +63,24 @@ async function run() {
     primaryDisplay: "Name",
   })
 
+  const subjectPromises = []
   for (let i = 0; i < SUBJECT_COUNT; i++) {
-    subjects.push(
+    subjectPromises.push(
+      (async () => {
       await createRow(apiKey, app._id, subjectTable, {
         Name: generator.profession(),
       })
-    )
     console.log(
       `Subject ${i + 1} of ${SUBJECT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
+        )
+      })()
     )
   }
 
+  const subjects = await Promise.all(subjectPromises)
+
   const gradesTable = await createTable(apiKey, app._id, {
     schema: {
       Score: {
@@ -124,7 +132,7 @@ async function run() {
 
 run()
   .then(() => {
-    console.log(`Done in(${(Date.now() - start) / 1000} seconds`)
+    console.log(`Done in ${(Date.now() - start) / 1000} seconds`)
   })
   .catch(err => {
     console.error(err)

From ed32b701ecd82bd0d1971839a830ec0b46b3c4df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 15:41:25 +0100
Subject: [PATCH 34/68] Move row save to sdk

---
 .../src/api/controllers/row/external.ts       | 46 +---------------
 .../server/src/api/controllers/row/index.ts   | 11 ++--
 .../src/api/controllers/row/internal.ts       | 41 +-------------
 packages/server/src/sdk/app/rows/external.ts  | 54 ++++++++++++++++++-
 packages/server/src/sdk/app/rows/internal.ts  | 49 +++++++++++++++++
 packages/server/src/sdk/app/rows/rows.ts      | 18 +++++++
 6 files changed, 129 insertions(+), 90 deletions(-)
 create mode 100644 packages/server/src/sdk/app/rows/internal.ts

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 287b2ae6aa..d741247687 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -26,7 +26,7 @@ import {
   inputProcessing,
   outputProcessing,
 } from "../../../utilities/rowProcessor"
-import { cloneDeep, isEqual } from "lodash"
+import { cloneDeep } from "lodash"
 
 export async function handleRequest<T extends Operation>(
   operation: T,
@@ -86,50 +86,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   }
 }
 
-export async function save(ctx: UserCtx) {
-  const inputs = ctx.request.body
-  const tableId = utils.getTableId(ctx)
-
-  const table = await sdk.tables.getTable(tableId)
-  const { table: updatedTable, row } = await inputProcessing(
-    ctx.user?._id,
-    cloneDeep(table),
-    inputs
-  )
-
-  const validateResult = await sdk.rows.utils.validate({
-    row,
-    tableId,
-  })
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }
-
-  const response = await handleRequest(Operation.CREATE, tableId, {
-    row,
-  })
-
-  if (!isEqual(table, updatedTable)) {
-    await sdk.tables.saveTable(updatedTable)
-  }
-
-  const rowId = response.row._id
-  if (rowId) {
-    const row = await sdk.rows.external.getRow(tableId, rowId, {
-      relationships: true,
-    })
-    return {
-      ...response,
-      row: await outputProcessing(table, row, {
-        preserveLinks: true,
-        squash: true,
-      }),
-    }
-  } else {
-    return response
-  }
-}
-
 export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 38731a87e1..7ff8d83e71 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -30,7 +30,7 @@ import { Format } from "../view/exporters"
 
 export * as views from "./views"
 
-function pickApi(tableId: any) {
+function pickApi(tableId: string) {
   if (isExternalTableID(tableId)) {
     return external
   }
@@ -84,9 +84,12 @@ export const save = async (ctx: UserCtx<Row, Row>) => {
     return patch(ctx as UserCtx<PatchRowRequest, PatchRowResponse>)
   }
   const { row, table, squashed } = await quotas.addRow(() =>
-    quotas.addQuery(() => pickApi(tableId).save(ctx), {
-      datasourceId: tableId,
-    })
+    quotas.addQuery(
+      () => sdk.rows.save(tableId, ctx.request.body, ctx.user?._id),
+      {
+        datasourceId: tableId,
+      }
+    )
   )
   ctx.status = 200
   ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index a251724b0a..5c714c098a 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -1,5 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import { generateRowID, InternalTables } from "../../../db/utils"
+import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import {
   AttachmentCleanup,
@@ -94,45 +94,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   })
 }
 
-export async function save(ctx: UserCtx) {
-  let inputs = ctx.request.body
-  inputs.tableId = utils.getTableId(ctx)
-
-  if (!inputs._rev && !inputs._id) {
-    inputs._id = generateRowID(inputs.tableId)
-  }
-
-  // this returns the table and row incase they have been updated
-  const dbTable = await sdk.tables.getTable(inputs.tableId)
-
-  // need to copy the table so it can be differenced on way out
-  const tableClone = cloneDeep(dbTable)
-
-  let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs)
-
-  const validateResult = await sdk.rows.utils.validate({
-    row,
-    table,
-  })
-
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }
-
-  // make sure link rows are up-to-date
-  row = (await linkRows.updateLinks({
-    eventType: linkRows.EventType.ROW_SAVE,
-    row,
-    tableId: row.tableId,
-    table,
-  })) as Row
-
-  return finaliseRow(table, row, {
-    oldTable: dbTable,
-    updateFormula: true,
-  })
-}
-
 export async function find(ctx: UserCtx): Promise<Row> {
   const tableId = utils.getTableId(ctx),
     rowId = ctx.params.rowId
diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts
index beae02e134..7ad5ea37ff 100644
--- a/packages/server/src/sdk/app/rows/external.ts
+++ b/packages/server/src/sdk/app/rows/external.ts
@@ -1,6 +1,13 @@
-import { IncludeRelationship, Operation } from "@budibase/types"
+import { IncludeRelationship, Operation, Row } from "@budibase/types"
 import { handleRequest } from "../../../api/controllers/row/external"
 import { breakRowIdField } from "../../../integrations/utils"
+import sdk from "../../../sdk"
+import {
+  inputProcessing,
+  outputProcessing,
+} from "../../../utilities/rowProcessor"
+import cloneDeep from "lodash/fp/cloneDeep"
+import isEqual from "lodash/fp/isEqual"
 
 export async function getRow(
   tableId: string,
@@ -15,3 +22,48 @@ export async function getRow(
   })
   return response ? response[0] : response
 }
+
+export async function save(
+  tableId: string,
+  inputs: Row,
+  userId: string | undefined
+) {
+  const table = await sdk.tables.getTable(tableId)
+  const { table: updatedTable, row } = await inputProcessing(
+    userId,
+    cloneDeep(table),
+    inputs
+  )
+
+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
+  const response = await handleRequest(Operation.CREATE, tableId, {
+    row,
+  })
+
+  if (!isEqual(table, updatedTable)) {
+    await sdk.tables.saveTable(updatedTable)
+  }
+
+  const rowId = response.row._id
+  if (rowId) {
+    const row = await sdk.rows.external.getRow(tableId, rowId, {
+      relationships: true,
+    })
+    return {
+      ...response,
+      row: await outputProcessing(table, row, {
+        preserveLinks: true,
+        squash: true,
+      }),
+    }
+  } else {
+    return response
+  }
+}
diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts
new file mode 100644
index 0000000000..c735d3e430
--- /dev/null
+++ b/packages/server/src/sdk/app/rows/internal.ts
@@ -0,0 +1,49 @@
+import { db } from "@budibase/backend-core"
+import { Row } from "@budibase/types"
+import sdk from "../../../sdk"
+import cloneDeep from "lodash/fp/cloneDeep"
+import { finaliseRow } from "src/api/controllers/row/staticFormula"
+import { inputProcessing } from "src/utilities/rowProcessor"
+import * as linkRows from "../../../db/linkedRows"
+
+export async function save(
+  tableId: string,
+  inputs: Row,
+  userId: string | undefined
+) {
+  inputs.tableId = tableId
+
+  if (!inputs._rev && !inputs._id) {
+    inputs._id = db.generateRowID(inputs.tableId)
+  }
+
+  // this returns the table and row incase they have been updated
+  const dbTable = await sdk.tables.getTable(inputs.tableId)
+
+  // need to copy the table so it can be differenced on way out
+  const tableClone = cloneDeep(dbTable)
+
+  let { table, row } = await inputProcessing(userId, tableClone, inputs)
+
+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    table,
+  })
+
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
+  // make sure link rows are up-to-date
+  row = (await linkRows.updateLinks({
+    eventType: linkRows.EventType.ROW_SAVE,
+    row,
+    tableId: row.tableId,
+    table,
+  })) as Row
+
+  return finaliseRow(table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
+}
diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts
index 8709180f0b..bfd84a715c 100644
--- a/packages/server/src/sdk/app/rows/rows.ts
+++ b/packages/server/src/sdk/app/rows/rows.ts
@@ -1,6 +1,9 @@
 import { db as dbCore, context } from "@budibase/backend-core"
 import { Database, Row } from "@budibase/types"
 import { getRowParams } from "../../../db/utils"
+import { isExternalTableID } from "../../../integrations/utils"
+import * as internal from "./internal"
+import * as external from "./external"
 
 export async function getAllInternalRows(appId?: string) {
   let db: Database
@@ -16,3 +19,18 @@ export async function getAllInternalRows(appId?: string) {
   )
   return response.rows.map(row => row.doc) as Row[]
 }
+
+function pickApi(tableId: any) {
+  if (isExternalTableID(tableId)) {
+    return external
+  }
+  return internal
+}
+
+export async function save(
+  tableId: string,
+  row: Row,
+  userId: string | undefined
+) {
+  return pickApi(tableId).save(tableId, row, userId)
+}

From 0452b1a307967184f02e8074ae1321d4fd3725a8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 16:25:08 +0100
Subject: [PATCH 35/68] Fix imports

---
 packages/server/src/sdk/app/rows/internal.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts
index c735d3e430..14e771b36e 100644
--- a/packages/server/src/sdk/app/rows/internal.ts
+++ b/packages/server/src/sdk/app/rows/internal.ts
@@ -2,8 +2,8 @@ import { db } from "@budibase/backend-core"
 import { Row } from "@budibase/types"
 import sdk from "../../../sdk"
 import cloneDeep from "lodash/fp/cloneDeep"
-import { finaliseRow } from "src/api/controllers/row/staticFormula"
-import { inputProcessing } from "src/utilities/rowProcessor"
+import { finaliseRow } from "../../../api/controllers/row/staticFormula"
+import { inputProcessing } from "../../../utilities/rowProcessor"
 import * as linkRows from "../../../db/linkedRows"
 
 export async function save(

From becb7bd46d3b6c5f829359da3ee0d72d5d31fab7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 17:26:51 +0100
Subject: [PATCH 36/68] Add tests

---
 .../src/sdk/app/rows/tests/internal.spec.ts   | 127 ++++++++++++++++++
 1 file changed, 127 insertions(+)
 create mode 100644 packages/server/src/sdk/app/rows/tests/internal.spec.ts

diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
new file mode 100644
index 0000000000..0d19543889
--- /dev/null
+++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
@@ -0,0 +1,127 @@
+import * as internalSdk from "../internal"
+
+import { generator } from "@budibase/backend-core/tests"
+import {
+  INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
+  FieldType,
+  Table,
+} from "@budibase/types"
+
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+
+describe("sdk >> rows >> internal", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  function makeRow() {
+    return {
+      name: generator.first(),
+      surname: generator.last(),
+      age: generator.age(),
+      address: generator.address(),
+    }
+  }
+
+  describe("save", () => {
+    const tableData: Table = {
+      name: generator.word(),
+      type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+          lastID: 0,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+          constraints: {
+            type: FieldType.STRING,
+          },
+        },
+        surname: {
+          name: "surname",
+          type: FieldType.STRING,
+          constraints: {
+            type: FieldType.STRING,
+          },
+        },
+        age: {
+          name: "age",
+          type: FieldType.NUMBER,
+          constraints: {
+            type: FieldType.NUMBER,
+          },
+        },
+        address: {
+          name: "address",
+          type: FieldType.STRING,
+          constraints: {
+            type: FieldType.STRING,
+          },
+        },
+      },
+    }
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("save will persist the row properly", async () => {
+      const table = await config.createTable(tableData)
+      const row = makeRow()
+
+      await config.doInContext(config.appId, async () => {
+        const response = await internalSdk.save(
+          table._id!,
+          row,
+          config.user._id
+        )
+
+        expect(response).toEqual({
+          table,
+          row: {
+            ...row,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+          squashed: {
+            ...row,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+        })
+
+        const persistedRow = await config.getRow(table._id!, response.row._id!)
+        expect(persistedRow).toEqual({
+          ...row,
+          type: "row",
+          _rev: expect.stringMatching("1-.*"),
+          createdAt: expect.any(String),
+          updatedAt: expect.any(String),
+        })
+      })
+    })
+
+    it.only("can persist in parallel", async () => {
+      const table = await config.createTable(tableData)
+      const rows = Array.from({ length: 100 }, () => makeRow())
+
+      await config.doInContext(config.appId, async () => {
+        await Promise.all(
+          rows.map(row => internalSdk.save(table._id!, row, config.user._id))
+        )
+
+        const persistedRows = await config.getRows(table._id!)
+        expect(persistedRows).toHaveLength(100)
+      })
+    })
+  })
+})

From 3824156ca5b90908d5a078ffb5d375194dd31ba8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 17:36:23 +0100
Subject: [PATCH 37/68] Add delete all apps script

---
 .../server/scripts/load/delete-all-apps.js    | 29 +++++++++++++++++++
 packages/server/scripts/load/utils.js         | 11 +++++++
 2 files changed, 40 insertions(+)
 create mode 100755 packages/server/scripts/load/delete-all-apps.js

diff --git a/packages/server/scripts/load/delete-all-apps.js b/packages/server/scripts/load/delete-all-apps.js
new file mode 100755
index 0000000000..5c9e974c7d
--- /dev/null
+++ b/packages/server/scripts/load/delete-all-apps.js
@@ -0,0 +1,29 @@
+#!/bin/node
+const { searchApps, deleteApp } = require("./utils")
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const apps = await searchApps(apiKey)
+  console.log(`Deleting ${apps.length} apps`)
+
+  let deletedApps = 0
+  await Promise.all(
+    apps.map(async app => {
+      await deleteApp(apiKey, app._id)
+      console.log(`App ${++deletedApps} of ${apps.length} deleted`)
+    })
+  )
+}
+
+run()
+  .then(() => {
+    console.log("Done!")
+  })
+  .catch(err => {
+    console.error(err)
+  })
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 7af867ab1d..04ebb6f3d6 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -38,6 +38,17 @@ exports.createApp = async apiKey => {
   return json.data
 }
 
+exports.searchApps = async apiKey => {
+  const res = await request(apiKey, `${URL_APP}/search`, "POST", {})
+  const json = await res.json()
+  return json.data
+}
+
+exports.deleteApp = async (apiKey, appId) => {
+  const res = await request(apiKey, `${URL_APP}/${appId}`, "DELETE")
+  return res
+}
+
 exports.getTable = async (apiKey, appId) => {
   const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId)
   const json = await res.json()

From f722ae970cdd79f7cbdd7bf7c4d1c6182932b91b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 10:30:27 +0100
Subject: [PATCH 38/68] Add autoid tests (failing)

---
 .../src/sdk/app/rows/tests/internal.spec.ts   | 112 ++++++++++++++++--
 1 file changed, 99 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
index 0d19543889..62a056bef3 100644
--- a/packages/server/src/sdk/app/rows/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
@@ -1,3 +1,4 @@
+import tk from "timekeeper"
 import * as internalSdk from "../internal"
 
 import { generator } from "@budibase/backend-core/tests"
@@ -6,9 +7,13 @@ import {
   TableSourceType,
   FieldType,
   Table,
+  AutoFieldSubTypes,
 } from "@budibase/types"
 
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { cache } from "@budibase/backend-core"
+
+tk.freeze(Date.now())
 
 describe("sdk >> rows >> internal", () => {
   const config = new TestConfiguration()
@@ -33,12 +38,6 @@ describe("sdk >> rows >> internal", () => {
       sourceId: INTERNAL_TABLE_SOURCE_ID,
       sourceType: TableSourceType.INTERNAL,
       schema: {
-        id: {
-          name: "id",
-          type: FieldType.AUTO,
-          autocolumn: true,
-          lastID: 0,
-        },
         name: {
           name: "name",
           type: FieldType.STRING,
@@ -110,18 +109,105 @@ describe("sdk >> rows >> internal", () => {
       })
     })
 
-    it.only("can persist in parallel", async () => {
-      const table = await config.createTable(tableData)
-      const rows = Array.from({ length: 100 }, () => makeRow())
+    it("auto ids will update when creating new rows", async () => {
+      const table = await config.createTable({
+        ...tableData,
+        schema: {
+          ...tableData.schema,
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            subtype: AutoFieldSubTypes.AUTO_ID,
+            autocolumn: true,
+            lastID: 0,
+          },
+        },
+      })
+      const row = makeRow()
 
       await config.doInContext(config.appId, async () => {
-        await Promise.all(
-          rows.map(row => internalSdk.save(table._id!, row, config.user._id))
+        const response = await internalSdk.save(
+          table._id!,
+          row,
+          config.user._id
         )
 
-        const persistedRows = await config.getRows(table._id!)
-        expect(persistedRows).toHaveLength(100)
+        expect(response).toEqual({
+          table: {
+            ...table,
+            schema: {
+              ...table.schema,
+              id: {
+                ...table.schema.id,
+                lastID: 1,
+              },
+            },
+          },
+          row: {
+            ...row,
+            id: 1,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+          squashed: {
+            ...row,
+            id: 1,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+        })
+
+        const persistedRow = await config.getRow(table._id!, response.row._id!)
+        expect(persistedRow).toEqual({
+          ...row,
+          type: "row",
+          id: 1,
+          _rev: expect.stringMatching("1-.*"),
+          createdAt: expect.any(String),
+          updatedAt: expect.any(String),
+        })
       })
     })
+
+    it("auto ids will update when creating new rows in parallel", async () => {
+      function makeRows(count: number) {
+        return Array.from({ length: count }, () => makeRow())
+      }
+
+      const table = await config.createTable({
+        ...tableData,
+        schema: {
+          ...tableData.schema,
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            subtype: AutoFieldSubTypes.AUTO_ID,
+            autocolumn: true,
+            lastID: 0,
+          },
+        },
+      })
+
+      await config.doInContext(config.appId, async () => {
+        for (const row of makeRows(30)) {
+          await internalSdk.save(table._id!, row, config.user._id)
+        }
+        await Promise.all(
+          makeRows(200).map(row =>
+            internalSdk.save(table._id!, row, config.user._id)
+          )
+        )
+        for (const row of makeRows(20)) {
+          await internalSdk.save(table._id!, row, config.user._id)
+        }
+      })
+
+      const persistedRows = await config.getRows(table._id!)
+      expect(persistedRows).toHaveLength(250)
+
+      const persistedTable = await config.getTable(table._id)
+      expect((table as any).schema.id.lastID).toBe(0)
+      expect(persistedTable.schema.id.lastID).toBe(250)
+    })
   })
 })

From c0a19c2a7ef5a7319997ec019caa3be7c5895dda Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 15:45:30 +0100
Subject: [PATCH 39/68] Type cache arguments

---
 packages/backend-core/src/cache/generic.ts | 23 +++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts
index 7a2be5a0f0..3ac323a8d4 100644
--- a/packages/backend-core/src/cache/generic.ts
+++ b/packages/backend-core/src/cache/generic.ts
@@ -18,14 +18,15 @@ export enum TTL {
   ONE_DAY = 86400,
 }
 
-function performExport(funcName: string) {
-  // @ts-ignore
-  return (...args: any) => GENERIC[funcName](...args)
-}
-
-export const keys = performExport("keys")
-export const get = performExport("get")
-export const store = performExport("store")
-export const destroy = performExport("delete")
-export const withCache = performExport("withCache")
-export const bustCache = performExport("bustCache")
+export const keys = (...args: Parameters<typeof GENERIC.keys>) =>
+  GENERIC.keys(...args)
+export const get = (...args: Parameters<typeof GENERIC.get>) =>
+  GENERIC.get(...args)
+export const store = (...args: Parameters<typeof GENERIC.store>) =>
+  GENERIC.store(...args)
+export const destroy = (...args: Parameters<typeof GENERIC.delete>) =>
+  GENERIC.delete(...args)
+export const withCache = (...args: Parameters<typeof GENERIC.withCache>) =>
+  GENERIC.withCache(...args)
+export const bustCache = (...args: Parameters<typeof GENERIC.bustCache>) =>
+  GENERIC.bustCache(...args)

From 0c77cf2b40d6fcec40de872934e7a90e79278ea8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:14:34 +0100
Subject: [PATCH 40/68] Fix concurrent saves

---
 .../src/api/controllers/row/staticFormula.ts  | 26 +++++++++++++------
 .../src/sdk/app/rows/tests/internal.spec.ts   | 17 ++++++++----
 packages/types/src/sdk/locks.ts               |  1 +
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index 8d52b6a05c..3cb7348ee3 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -5,8 +5,8 @@ import {
   processFormulas,
 } from "../../../utilities/rowProcessor"
 import { FieldTypes, FormulaTypes } from "../../../constants"
-import { context } from "@budibase/backend-core"
-import { Table, Row } from "@budibase/types"
+import { context, locks } from "@budibase/backend-core"
+import { Table, Row, LockType, LockName } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import sdk from "../../../sdk"
 import isEqual from "lodash/isEqual"
@@ -149,12 +149,22 @@ export async function finaliseRow(
       await db.put(table)
     } catch (err: any) {
       if (err.status === 409) {
-        const updatedTable = await sdk.tables.getTable(table._id!)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
+        // Some conflicts with the autocolumns occurred, we need to refetch the table and recalculate
+        await locks.doWithLock(
+          {
+            type: LockType.AUTO_EXTEND,
+            name: LockName.PROCESS_AUTO_COLUMNS,
+            resource: table._id,
+          },
+          async () => {
+            const latestTable = await sdk.tables.getTable(table._id!)
+            let response = processAutoColumn(null, latestTable, row, {
+              reprocessing: true,
+            })
+            await db.put(response.table)
+            row = response.row
+          }
+        )
       } else {
         throw err
       }
diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
index 62a056bef3..b60d31b226 100644
--- a/packages/server/src/sdk/app/rows/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
@@ -189,25 +189,32 @@ describe("sdk >> rows >> internal", () => {
       })
 
       await config.doInContext(config.appId, async () => {
-        for (const row of makeRows(30)) {
+        for (const row of makeRows(5)) {
           await internalSdk.save(table._id!, row, config.user._id)
         }
         await Promise.all(
-          makeRows(200).map(row =>
+          makeRows(10).map(row =>
             internalSdk.save(table._id!, row, config.user._id)
           )
         )
-        for (const row of makeRows(20)) {
+        for (const row of makeRows(5)) {
           await internalSdk.save(table._id!, row, config.user._id)
         }
       })
 
       const persistedRows = await config.getRows(table._id!)
-      expect(persistedRows).toHaveLength(250)
+      expect(persistedRows).toHaveLength(20)
+      expect(persistedRows).toEqual(
+        expect.arrayContaining(
+          Array.from({ length: 20 }).map((_, i) =>
+            expect.objectContaining({ id: i + 1 })
+          )
+        )
+      )
 
       const persistedTable = await config.getTable(table._id)
       expect((table as any).schema.id.lastID).toBe(0)
-      expect(persistedTable.schema.id.lastID).toBe(250)
+      expect(persistedTable.schema.id.lastID).toBe(20)
     })
   })
 })
diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts
index 82a7089b3f..0e6053a4db 100644
--- a/packages/types/src/sdk/locks.ts
+++ b/packages/types/src/sdk/locks.ts
@@ -21,6 +21,7 @@ export enum LockName {
   PERSIST_WRITETHROUGH = "persist_writethrough",
   QUOTA_USAGE_EVENT = "quota_usage_event",
   APP_MIGRATION = "app_migrations",
+  PROCESS_AUTO_COLUMNS = "process_auto_columns",
 }
 
 export type LockOptions = {

From 69527cd4b964ab6fffdf7009161ecf9ec3c493be Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:22:41 +0100
Subject: [PATCH 41/68] Parallel creations

---
 .../scripts/load/create-many-relationships.js | 30 +++++++------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 49d4efb28d..dba953e4c7 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -25,11 +25,10 @@ async function run() {
   }
   console.log(`Table found: ${studentsTable.name}`)
 
-  const studentsPromise = []
   let studentNumber = studentsTable.schema["Auto ID"].lastID
-  for (let i = 0; i < STUDENT_COUNT; i++) {
-    studentsPromise.push(
-      (async () => {
+  let i = 0
+  const students = await Promise.all(
+    Array.from({ length: STUDENT_COUNT }).map(async () => {
       await createRow(apiKey, app._id, studentsTable, {
         "Student Number": (++studentNumber).toString(),
         "First Name": generator.first(),
@@ -42,15 +41,12 @@ async function run() {
       })
 
     console.log(
-      `Student ${i + 1} of ${STUDENT_COUNT} created (${
+        `Student ${++i} of ${STUDENT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
         )
-      })()
+    })
     )
-  }
-
-  const students = await Promise.all(studentsPromise)
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
@@ -63,23 +59,19 @@ async function run() {
     primaryDisplay: "Name",
   })
 
-  const subjectPromises = []
-  for (let i = 0; i < SUBJECT_COUNT; i++) {
-    subjectPromises.push(
-      (async () => {
+  i = 0
+  const subjects = await Promise.all(
+    Array.from({ length: SUBJECT_COUNT }).map(async () => {
       await createRow(apiKey, app._id, subjectTable, {
         Name: generator.profession(),
       })
     console.log(
-      `Subject ${i + 1} of ${SUBJECT_COUNT} created (${
+        `Subject ${++i} of ${SUBJECT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
         )
-      })()
+    })
     )
-  }
-
-  const subjects = await Promise.all(subjectPromises)
 
   const gradesTable = await createTable(apiKey, app._id, {
     schema: {
@@ -113,7 +105,7 @@ async function run() {
     name: "Grades",
   })
 
-  let i = 0
+  i = 0
   for (const student of students) {
     for (const subject of subjects) {
       await createRow(apiKey, app._id, gradesTable, {

From b034542536b9a2b922bed014697447ed50c9b85c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:24:48 +0100
Subject: [PATCH 42/68] Fix

---
 .../scripts/load/create-many-relationships.js | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index dba953e4c7..c1e59bd586 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -29,7 +29,7 @@ async function run() {
   let i = 0
   const students = await Promise.all(
     Array.from({ length: STUDENT_COUNT }).map(async () => {
-      await createRow(apiKey, app._id, studentsTable, {
+      const row = await createRow(apiKey, app._id, studentsTable, {
         "Student Number": (++studentNumber).toString(),
         "First Name": generator.first(),
         "Last Name": generator.last(),
@@ -40,13 +40,14 @@ async function run() {
         "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
       })
 
-    console.log(
+      console.log(
         `Student ${++i} of ${STUDENT_COUNT} created (${
-        (Date.now() - start) / 1000
-      }s)`
-        )
+          (Date.now() - start) / 1000
+        }s)`
+      )
+      return row
     })
-    )
+  )
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
@@ -62,16 +63,17 @@ async function run() {
   i = 0
   const subjects = await Promise.all(
     Array.from({ length: SUBJECT_COUNT }).map(async () => {
-      await createRow(apiKey, app._id, subjectTable, {
+      const row = await createRow(apiKey, app._id, subjectTable, {
         Name: generator.profession(),
       })
-    console.log(
+      console.log(
         `Subject ${++i} of ${SUBJECT_COUNT} created (${
-        (Date.now() - start) / 1000
-      }s)`
-        )
+          (Date.now() - start) / 1000
+        }s)`
+      )
+      return row
     })
-    )
+  )
 
   const gradesTable = await createTable(apiKey, app._id, {
     schema: {

From c46509d6cb2673cb8738a63c67ea8c65e85eeeeb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:39:14 +0100
Subject: [PATCH 43/68] Use batches

---
 .../scripts/load/create-many-relationships.js | 80 +++++++++++--------
 1 file changed, 48 insertions(+), 32 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index c1e59bd586..f034cb0cc2 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -7,6 +7,8 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
+const batchSize = 100
+
 if (!process.argv[2]) {
   console.error("Please specify an API key as script argument.")
   process.exit(-1)
@@ -14,6 +16,31 @@ if (!process.argv[2]) {
 
 const start = Date.now()
 
+async function batchCreate(apiKey, appId, count, table, generator) {
+  let i = 0
+
+  async function createSingleRow() {
+    const row = await createRow(apiKey, appId, table, generator())
+    console.log(
+      `${table.name} - ${++i} of ${count} created (${
+        (Date.now() - start) / 1000
+      }s)`
+    )
+    return row
+  }
+
+  const rows = []
+  for (let j = 0; j < count; j += batchSize) {
+    const batchPromises = Array.from(
+      { length: Math.min(batchSize, count - j) },
+      createSingleRow
+    )
+    const batchRows = await Promise.all(batchPromises)
+    rows.push(...batchRows)
+  }
+  return rows
+}
+
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
@@ -26,26 +53,20 @@ async function run() {
   console.log(`Table found: ${studentsTable.name}`)
 
   let studentNumber = studentsTable.schema["Auto ID"].lastID
-  let i = 0
-  const students = await Promise.all(
-    Array.from({ length: STUDENT_COUNT }).map(async () => {
-      const row = await createRow(apiKey, app._id, studentsTable, {
-        "Student Number": (++studentNumber).toString(),
-        "First Name": generator.first(),
-        "Last Name": generator.last(),
-        Gender: generator.pickone(["M", "F"]),
-        Grade: generator.pickone(["8", "9", "10", "11"]),
-        "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
-        "Home Number": generator.phone(),
-        "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
-      })
-
-      console.log(
-        `Student ${++i} of ${STUDENT_COUNT} created (${
-          (Date.now() - start) / 1000
-        }s)`
-      )
-      return row
+  const students = await batchCreate(
+    apiKey,
+    app._id,
+    STUDENT_COUNT,
+    studentsTable,
+    () => ({
+      "Student Number": (++studentNumber).toString(),
+      "First Name": generator.first(),
+      "Last Name": generator.last(),
+      Gender: generator.pickone(["M", "F"]),
+      Grade: generator.pickone(["8", "9", "10", "11"]),
+      "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
+      "Home Number": generator.phone(),
+      "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
     })
   )
 
@@ -60,18 +81,13 @@ async function run() {
     primaryDisplay: "Name",
   })
 
-  i = 0
-  const subjects = await Promise.all(
-    Array.from({ length: SUBJECT_COUNT }).map(async () => {
-      const row = await createRow(apiKey, app._id, subjectTable, {
-        Name: generator.profession(),
-      })
-      console.log(
-        `Subject ${++i} of ${SUBJECT_COUNT} created (${
-          (Date.now() - start) / 1000
-        }s)`
-      )
-      return row
+  const subjects = await batchCreate(
+    apiKey,
+    app._id,
+    SUBJECT_COUNT,
+    subjectTable,
+    () => ({
+      Name: generator.profession(),
     })
   )
 

From dbf4d6dbdd5d76fb52b43ae4434f2e719ae9dddf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:48:39 +0100
Subject: [PATCH 44/68] Create grades in batch

---
 .../scripts/load/create-many-relationships.js | 48 ++++++++-----------
 1 file changed, 19 insertions(+), 29 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index f034cb0cc2..a90c21d250 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -7,8 +7,6 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
-const batchSize = 100
-
 if (!process.argv[2]) {
   console.error("Please specify an API key as script argument.")
   process.exit(-1)
@@ -16,13 +14,13 @@ if (!process.argv[2]) {
 
 const start = Date.now()
 
-async function batchCreate(apiKey, appId, count, table, generator) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
 
-  async function createSingleRow() {
-    const row = await createRow(apiKey, appId, table, generator())
+  async function createSingleRow(item) {
+    const row = await createRow(apiKey, appId, table, item)
     console.log(
-      `${table.name} - ${++i} of ${count} created (${
+      `${table.name} - ${++i} of ${items.length} created (${
         (Date.now() - start) / 1000
       }s)`
     )
@@ -30,11 +28,8 @@ async function batchCreate(apiKey, appId, count, table, generator) {
   }
 
   const rows = []
-  for (let j = 0; j < count; j += batchSize) {
-    const batchPromises = Array.from(
-      { length: Math.min(batchSize, count - j) },
-      createSingleRow
-    )
+  for (let j = 0; j < items.length; j += batchSize) {
+    const batchPromises = items.slice(j, j + batchSize).map(createSingleRow)
     const batchRows = await Promise.all(batchPromises)
     rows.push(...batchRows)
   }
@@ -56,9 +51,8 @@ async function run() {
   const students = await batchCreate(
     apiKey,
     app._id,
-    STUDENT_COUNT,
     studentsTable,
-    () => ({
+    Array.from({ length: SUBJECT_COUNT }).map(() => ({
       "Student Number": (++studentNumber).toString(),
       "First Name": generator.first(),
       "Last Name": generator.last(),
@@ -67,7 +61,7 @@ async function run() {
       "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
       "Home Number": generator.phone(),
       "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
-    })
+    }))
   )
 
   const subjectTable = await createTable(apiKey, app._id, {
@@ -84,11 +78,10 @@ async function run() {
   const subjects = await batchCreate(
     apiKey,
     app._id,
-    SUBJECT_COUNT,
     subjectTable,
-    () => ({
+    Array.from({ length: SUBJECT_COUNT }).map(() => ({
       Name: generator.profession(),
-    })
+    }))
   )
 
   const gradesTable = await createTable(apiKey, app._id, {
@@ -123,21 +116,18 @@ async function run() {
     name: "Grades",
   })
 
-  i = 0
-  for (const student of students) {
-    for (const subject of subjects) {
-      await createRow(apiKey, app._id, gradesTable, {
+  await batchCreate(
+    apiKey,
+    app._id,
+    gradesTable,
+    students.flatMap(student =>
+      subjects.map(subject => ({
         Score: generator.integer({ min: 0, max: 100 }),
         Student: [student],
         Subject: [subject],
-      })
-      console.log(
-        `Grade ${++i} of ${students.length * subjects.length} created (${
-          (Date.now() - start) / 1000
-        }s)`
-      )
-    }
-  }
+      }))
+    )
+  )
 }
 
 run()

From 3eee77c5498328547c5b7c5ff9368eda5757718d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:52:04 +0100
Subject: [PATCH 45/68] Create 500 students

---
 packages/server/scripts/load/create-many-relationships.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index a90c21d250..4626b8fe24 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -52,7 +52,7 @@ async function run() {
     apiKey,
     app._id,
     studentsTable,
-    Array.from({ length: SUBJECT_COUNT }).map(() => ({
+    Array.from({ length: STUDENT_COUNT }).map(() => ({
       "Student Number": (++studentNumber).toString(),
       "First Name": generator.first(),
       "Last Name": generator.last(),

From c518d8a7fad730d9c7798080840dcf0abfb3bbf1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 18:02:44 +0100
Subject: [PATCH 46/68] Add logs

---
 packages/server/scripts/load/create-many-relationships.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 4626b8fe24..92db2f83a1 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -128,6 +128,10 @@ async function run() {
       }))
     )
   )
+
+  console.log(
+    `Access the app here: http://localhost:10000/builder/app/${app._id}`
+  )
 }
 
 run()

From 92f1107e6caf686bc24820f0a11527299de3fbb1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 18:28:10 +0100
Subject: [PATCH 47/68] Prevent one single request to stop the process

---
 .../scripts/load/create-many-relationships.js  | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 92db2f83a1..d7526a70ff 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -18,13 +18,17 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
 
   async function createSingleRow(item) {
-    const row = await createRow(apiKey, appId, table, item)
-    console.log(
-      `${table.name} - ${++i} of ${items.length} created (${
-        (Date.now() - start) / 1000
-      }s)`
-    )
-    return row
+    try {
+      const row = await createRow(apiKey, appId, table, item)
+      console.log(
+        `${table.name} - ${++i} of ${items.length} created (${
+          (Date.now() - start) / 1000
+        }s)`
+      )
+      return row
+    } catch {
+      console.error("Error creating row", item)
+    }
   }
 
   const rows = []

From 494a2ff91f5b87e85658f1ffcd06c1da9887de89 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 18:29:57 +0100
Subject: [PATCH 48/68] Errors

---
 packages/server/scripts/load/create-many-relationships.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index d7526a70ff..15ef9bcb1e 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -17,6 +17,7 @@ const start = Date.now()
 async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
 
+  let errors = 0
   async function createSingleRow(item) {
     try {
       const row = await createRow(apiKey, appId, table, item)
@@ -27,7 +28,7 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
       )
       return row
     } catch {
-      console.error("Error creating row", item)
+      errors++
     }
   }
 
@@ -37,6 +38,9 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
     const batchRows = await Promise.all(batchPromises)
     rows.push(...batchRows)
   }
+  if (errors) {
+    console.error(`Error creating ${errors} row`)
+  }
   return rows
 }
 

From abf025b3f7e5fbe2458b53995f362268323cb376 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 19:06:26 +0100
Subject: [PATCH 49/68] Use inflight max instead of batch wait

---
 .../scripts/load/create-many-relationships.js | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 15ef9bcb1e..0f4fe3470c 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -13,11 +13,10 @@ if (!process.argv[2]) {
 }
 
 const start = Date.now()
-
-async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
   let i = 0
-
   let errors = 0
+
   async function createSingleRow(item) {
     try {
       const row = await createRow(apiKey, appId, table, item)
@@ -33,14 +32,25 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   }
 
   const rows = []
-  for (let j = 0; j < items.length; j += batchSize) {
-    const batchPromises = items.slice(j, j + batchSize).map(createSingleRow)
-    const batchRows = await Promise.all(batchPromises)
-    rows.push(...batchRows)
-  }
-  if (errors) {
-    console.error(`Error creating ${errors} row`)
+  const maxConcurrency = Math.min(batchSize, items.length)
+  const inFlight = {}
+
+  for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+    const item = items[itemIndex]
+    const promise = createSingleRow(item).then(result => {
+      rows.push(result)
+      delete inFlight[itemIndex]
+    })
+
+    inFlight[itemIndex] = promise
+
+    if (Object.keys(inFlight).length >= maxConcurrency) {
+      await Promise.race(Object.values(inFlight))
+    }
   }
+
+  await Promise.all(Object.values(inFlight))
+
   return rows
 }
 

From 31c01e0a6b8338aa6c58772ff572e08c16bf6620 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 10:05:49 +0100
Subject: [PATCH 50/68] Display errors

---
 .../server/scripts/load/create-many-relationships.js  | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 0f4fe3470c..14eaccd99c 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -2,6 +2,7 @@
 const { createApp, getTable, createRow, createTable } = require("./utils")
 
 const Chance = require("chance")
+
 const generator = new Chance()
 
 const STUDENT_COUNT = 500
@@ -13,7 +14,7 @@ if (!process.argv[2]) {
 }
 
 const start = Date.now()
-async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
   let i = 0
   let errors = 0
 
@@ -51,6 +52,14 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
 
   await Promise.all(Object.values(inFlight))
 
+  if (errors) {
+    console.error(
+      `${table.name} - ${errors} creation errored (${
+        (Date.now() - start) / 1000
+      }s)`
+    )
+  }
+
   return rows
 }
 

From 3e991cc2f13ba80aca8fc85e63b1bf087a1da412 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 11:36:29 +0100
Subject: [PATCH 51/68] Use yargs

---
 .../server/scripts/load/create-many-relationships.js   | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 14eaccd99c..84d1f94dda 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -1,5 +1,5 @@
 #!/bin/node
-const { createApp, getTable, createRow, createTable } = require("./utils")
+const yargs = require("yargs")
 
 const Chance = require("chance")
 
@@ -8,10 +8,9 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
-if (!process.argv[2]) {
-  console.error("Please specify an API key as script argument.")
-  process.exit(-1)
-}
+const { apiKey, appId } = require("yargs")
+  .demandOption(["apiKey"])
+  .option("appId").argv
 
 const start = Date.now()
 async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
@@ -64,7 +63,6 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
 }
 
 async function run() {
-  const apiKey = process.argv[2]
   const app = await createApp(apiKey)
   console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
 

From c5a50a911f64b7094c8941b12b5eb6b34e70b9a0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 11:46:49 +0100
Subject: [PATCH 52/68] Allow reusing app

---
 .../scripts/load/create-many-relationships.js | 54 ++++++++++++-------
 packages/server/scripts/load/utils.js         | 13 ++++-
 2 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 84d1f94dda..f683d3aed4 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -1,5 +1,11 @@
 #!/bin/node
-const yargs = require("yargs")
+const {
+  createApp,
+  getTable,
+  createRow,
+  createTable,
+  getApp,
+} = require("./utils")
 
 const Chance = require("chance")
 
@@ -8,12 +14,12 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
-const { apiKey, appId } = require("yargs")
+let { apiKey, appId } = require("yargs")
   .demandOption(["apiKey"])
   .option("appId").argv
 
 const start = Date.now()
-async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
   let errors = 0
 
@@ -62,20 +68,34 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
   return rows
 }
 
-async function run() {
-  const app = await createApp(apiKey)
-  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
+const useExistingApp = !!appId
 
-  const studentsTable = await getTable(apiKey, app._id)
-  if (studentsTable.name !== "Students") {
-    throw 'Fetched table should be "Students"'
+async function upsertTable(appId, tableName, tableData) {
+  if (useExistingApp) {
+    return await getTable(apiKey, appId, tableName)
   }
-  console.log(`Table found: ${studentsTable.name}`)
+
+  const table = await createTable(apiKey, appId, {
+    ...tableData,
+    name: tableName,
+  })
+  return table
+}
+
+async function run() {
+  if (!appId) {
+    const app = appId ? await getApp(apiKey, appId) : await createApp(apiKey)
+    appId = app._id
+  }
+
+  console.log(`App created: http://localhost:10000/builder/app/${appId}`)
+
+  const studentsTable = await getTable(apiKey, appId, "Students")
 
   let studentNumber = studentsTable.schema["Auto ID"].lastID
   const students = await batchCreate(
     apiKey,
-    app._id,
+    appId,
     studentsTable,
     Array.from({ length: STUDENT_COUNT }).map(() => ({
       "Student Number": (++studentNumber).toString(),
@@ -89,27 +109,26 @@ async function run() {
     }))
   )
 
-  const subjectTable = await createTable(apiKey, app._id, {
+  const subjectTable = await upsertTable(appId, "Subjects", {
     schema: {
       Name: {
         name: "Name",
         type: "string",
       },
     },
-    name: "Subjects",
     primaryDisplay: "Name",
   })
 
   const subjects = await batchCreate(
     apiKey,
-    app._id,
+    appId,
     subjectTable,
     Array.from({ length: SUBJECT_COUNT }).map(() => ({
       Name: generator.profession(),
     }))
   )
 
-  const gradesTable = await createTable(apiKey, app._id, {
+  const gradesTable = await upsertTable(appId, "Grades", {
     schema: {
       Score: {
         name: "Score",
@@ -138,12 +157,11 @@ async function run() {
         type: "link",
       },
     },
-    name: "Grades",
   })
 
   await batchCreate(
     apiKey,
-    app._id,
+    appId,
     gradesTable,
     students.flatMap(student =>
       subjects.map(subject => ({
@@ -155,7 +173,7 @@ async function run() {
   )
 
   console.log(
-    `Access the app here: http://localhost:10000/builder/app/${app._id}`
+    `Access the app here: http://localhost:10000/builder/app/${appId}`
   )
 }
 
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 04ebb6f3d6..2b789cde52 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -38,6 +38,11 @@ exports.createApp = async apiKey => {
   return json.data
 }
 
+exports.getApp = async (apiKey, appId) => {
+  const res = await request(apiKey, `${URL_APP}/${appId}`, "GET")
+  const json = await res.json()
+  return json.data
+}
 exports.searchApps = async apiKey => {
   const res = await request(apiKey, `${URL_APP}/search`, "POST", {})
   const json = await res.json()
@@ -49,10 +54,14 @@ exports.deleteApp = async (apiKey, appId) => {
   return res
 }
 
-exports.getTable = async (apiKey, appId) => {
+exports.getTable = async (apiKey, appId, tableName) => {
   const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId)
   const json = await res.json()
-  return json.data[0]
+  const table = json.data.find(t => t.name === tableName)
+  if (!table) {
+    throw `Table '${tableName} not found`
+  }
+  return table
 }
 
 exports.createRow = async (apiKey, appId, table, body) => {

From bd221f27270a56bb6b9d363866dfc3c07a5dfa9a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 11:52:48 +0100
Subject: [PATCH 53/68] Prevent locks

---
 .../server/scripts/load/create-many-relationships.js  | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index f683d3aed4..07861cc632 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -43,10 +43,13 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
 
   for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
     const item = items[itemIndex]
-    const promise = createSingleRow(item).then(result => {
-      rows.push(result)
-      delete inFlight[itemIndex]
-    })
+    const promise = createSingleRow(item)
+      .then(result => {
+        rows.push(result)
+      })
+      .finally(() => {
+        delete inFlight[itemIndex]
+      })
 
     inFlight[itemIndex] = promise
 

From 5a45249ad4f802d5b6a0d724d99207218ae7a025 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 12:18:24 +0100
Subject: [PATCH 54/68] Reuse subjects

---
 .../scripts/load/create-many-relationships.js | 19 +++++++++++--------
 packages/server/scripts/load/utils.js         |  6 ++++++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 07861cc632..13f861ec0b 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -5,6 +5,7 @@ const {
   createRow,
   createTable,
   getApp,
+  getRows,
 } = require("./utils")
 
 const Chance = require("chance")
@@ -122,14 +123,16 @@ async function run() {
     primaryDisplay: "Name",
   })
 
-  const subjects = await batchCreate(
-    apiKey,
-    appId,
-    subjectTable,
-    Array.from({ length: SUBJECT_COUNT }).map(() => ({
-      Name: generator.profession(),
-    }))
-  )
+  const subjects = useExistingApp
+    ? await getRows(apiKey, appId, subjectTable._id)
+    : await batchCreate(
+        apiKey,
+        appId,
+        subjectTable,
+        Array.from({ length: SUBJECT_COUNT }).map(() => ({
+          Name: generator.profession(),
+        }))
+      )
 
   const gradesTable = await upsertTable(appId, "Grades", {
     schema: {
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 2b789cde52..1dabdcec9a 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -88,6 +88,12 @@ exports.createRow = async (apiKey, appId, table, body) => {
   return (await res.json()).data
 }
 
+exports.getRows = async (apiKey, appId, tableId) => {
+  const url = `${URL_TABLE}/${tableId}/rows/search`
+  const res = await request(apiKey, url, "POST", {}, appId)
+  return (await res.json()).data
+}
+
 exports.createTable = async (apiKey, appId, config) => {
   const res = await request(apiKey, URL_TABLE, "POST", config, appId)
   const json = await res.json()

From 3cfe641486af117b1db5468e7054909c720b3427 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 22 Dec 2023 18:28:07 +0000
Subject: [PATCH 55/68] Fixing issue with Redis disconnection - this should
 correctly reconnect the service when Redis service becomes available again.

---
 packages/backend-core/src/queue/queue.ts |  2 +-
 packages/backend-core/src/redis/redis.ts | 21 +++++++--------------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index 0657437a3b..b95dace5b2 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -47,7 +47,7 @@ export function createQueue<T>(
     cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
     // fire off an initial cleanup
     cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
+      console.error(`Unable to cleanup ${jobQueue} initially - ${err}`)
     })
   }
   return queue
diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index 701e262091..d15453ba62 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -18,6 +18,7 @@ import {
   SelectableDatabase,
   getRedisConnectionDetails,
 } from "./utils"
+import { logAlert } from "../logging"
 import * as timers from "../timers"
 
 const RETRY_PERIOD_MS = 2000
@@ -39,21 +40,16 @@ function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
 }
 
-function connectionError(
-  selectDb: number,
-  timeout: NodeJS.Timeout,
-  err: Error | string
-) {
+function connectionError(timeout: NodeJS.Timeout, err: Error | string) {
   // manually shut down, ignore errors
   if (CLOSED) {
     return
   }
-  pickClient(selectDb).disconnect()
   CLOSED = true
   // always clear this on error
   clearTimeout(timeout)
   CONNECTED = false
-  console.error("Redis connection failed - " + err)
+  logAlert("Redis connection failed", err)
   setTimeout(() => {
     init()
   }, RETRY_PERIOD_MS)
@@ -79,11 +75,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   // start the timer - only allowed 5 seconds to connect
   timeout = setTimeout(() => {
     if (!CONNECTED) {
-      connectionError(
-        selectDb,
-        timeout,
-        "Did not successfully connect in timeout"
-      )
+      connectionError(timeout, "Did not successfully connect in timeout")
     }
   }, STARTUP_TIMEOUT_MS)
 
@@ -106,12 +98,13 @@ function init(selectDb = DEFAULT_SELECT_DB) {
       // allow the process to exit
       return
     }
-    connectionError(selectDb, timeout, err)
+    connectionError(timeout, err)
   })
   client.on("error", (err: Error) => {
-    connectionError(selectDb, timeout, err)
+    connectionError(timeout, err)
   })
   client.on("connect", () => {
+    console.log(`Connected to Redis DB: ${selectDb}`)
     clearTimeout(timeout)
     CONNECTED = true
   })

From a6537e66c3085e70e89ac76aeaea4ee33b22e9d6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 10:26:03 +0100
Subject: [PATCH 56/68] Set scim config on beforeall

---
 packages/worker/src/api/routes/global/tests/scim.spec.ts | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts
index 884625805c..3614328d91 100644
--- a/packages/worker/src/api/routes/global/tests/scim.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts
@@ -14,9 +14,14 @@ import { events } from "@budibase/backend-core"
 jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)
 
-mocks.licenses.useScimIntegration()
-
 describe("scim", () => {
+  beforeAll(async () => {
+    tk.freeze(mocks.date.MOCK_DATE)
+    mocks.licenses.useScimIntegration()
+
+    await config.setSCIMConfig(true)
+  })
+
   beforeEach(async () => {
     jest.resetAllMocks()
     tk.freeze(mocks.date.MOCK_DATE)

From fede6dc3e4136925946d97c4a45eceef7a988d5f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 11:12:31 +0100
Subject: [PATCH 57/68] Fix flaky scim test

---
 .../worker/src/api/routes/global/tests/scim.spec.ts | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts
index 3614328d91..56b7ca9f40 100644
--- a/packages/worker/src/api/routes/global/tests/scim.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts
@@ -1,6 +1,6 @@
 import tk from "timekeeper"
 import _ from "lodash"
-import { mocks, structures } from "@budibase/backend-core/tests"
+import { generator, mocks, structures } from "@budibase/backend-core/tests"
 import {
   ScimCreateUserRequest,
   ScimGroupResponse,
@@ -575,8 +575,15 @@ describe("scim", () => {
         beforeAll(async () => {
           groups = []
 
-          for (let i = 0; i < groupCount; i++) {
-            const body = structures.scim.createGroupRequest()
+          const groupNames = generator.unique(
+            () => generator.word(),
+            groupCount
+          )
+
+          for (const groupName of groupNames) {
+            const body = structures.scim.createGroupRequest({
+              displayName: groupName,
+            })
             groups.push(await config.api.scimGroupsAPI.post({ body }))
           }
 

From f722f9e2d63997f1b2da6a89b23614d8bc19b7c2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 15:06:04 +0100
Subject: [PATCH 58/68] Invalidate reset code once used

---
 packages/backend-core/src/cache/passwordReset.ts | 11 ++++++++++-
 packages/worker/src/sdk/auth/auth.ts             |  2 ++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts
index 7f5a93f149..a19e99745a 100644
--- a/packages/backend-core/src/cache/passwordReset.ts
+++ b/packages/backend-core/src/cache/passwordReset.ts
@@ -1,6 +1,6 @@
 import * as redis from "../redis/init"
 import * as utils from "../utils"
-import { Duration, DurationType } from "../utils"
+import { Duration } from "../utils"
 
 const TTL_SECONDS = Duration.fromHours(1).toSeconds()
 
@@ -36,3 +36,12 @@ export async function getCode(code: string): Promise<PasswordReset> {
   }
   return value
 }
+
+/**
+ * Given a reset code this will invalidate it.
+ * @param code The code provided via the email link.
+ */
+export async function invalidateCode(code: string): Promise<void> {
+  const client = await redis.getPasswordResetClient()
+  await client.delete(code)
+}
diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index 1f9da8a260..bdc5fc2366 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -79,6 +79,8 @@ export const resetUpdate = async (resetCode: string, password: string) => {
   user.password = password
   user = await userSdk.db.save(user)
 
+  await cache.passwordReset.invalidateCode(resetCode)
+
   // remove password from the user before sending events
   delete user.password
   await events.user.passwordReset(user)

From dcacd6bf17a2491f64708b6c4b88f428c5fdc6d3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 16:07:26 +0100
Subject: [PATCH 59/68] Add basic test

---
 packages/backend-core/src/users/db.ts         |  2 +-
 .../worker/src/sdk/auth/tests/auth.spec.ts    | 28 +++++++++++++++++++
 .../worker/src/sdk/users/tests/users.spec.ts  |  1 -
 3 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 packages/worker/src/sdk/auth/tests/auth.spec.ts

diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index 326bed3cc5..01fa4899d1 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -2,7 +2,7 @@ import env from "../environment"
 import * as eventHelpers from "./events"
 import * as accountSdk from "../accounts"
 import * as cache from "../cache"
-import { doInTenant, getGlobalDB, getIdentity, getTenantId } from "../context"
+import { getGlobalDB, getIdentity, getTenantId } from "../context"
 import * as dbUtils from "../db"
 import { EmailUnavailableError, HTTPError } from "../errors"
 import * as platform from "../platform"
diff --git a/packages/worker/src/sdk/auth/tests/auth.spec.ts b/packages/worker/src/sdk/auth/tests/auth.spec.ts
new file mode 100644
index 0000000000..b1758e79c6
--- /dev/null
+++ b/packages/worker/src/sdk/auth/tests/auth.spec.ts
@@ -0,0 +1,28 @@
+import { cache, context, utils } from "@budibase/backend-core"
+import { resetUpdate } from "../auth"
+import { generator, structures } from "@budibase/backend-core/tests"
+import { TestConfiguration } from "../../../tests"
+
+describe("auth", () => {
+  const config = new TestConfiguration()
+
+  describe("resetUpdate", () => {
+    it("providing a valid code will update the password", async () => {
+      await context.doInTenant(structures.tenant.id(), async () => {
+        const user = await config.createUser()
+        const previousPassword = user.password
+
+        const code = await cache.passwordReset.createCode(user._id!, {})
+        const newPassword = generator.hash()
+
+        await resetUpdate(code, newPassword)
+
+        const persistedUser = await config.getUser(user.email)
+        expect(persistedUser.password).not.toBe(previousPassword)
+        expect(
+          await utils.compare(newPassword, persistedUser.password!)
+        ).toBeTruthy()
+      })
+    })
+  })
+})
diff --git a/packages/worker/src/sdk/users/tests/users.spec.ts b/packages/worker/src/sdk/users/tests/users.spec.ts
index df1aa74200..a02ca42c2f 100644
--- a/packages/worker/src/sdk/users/tests/users.spec.ts
+++ b/packages/worker/src/sdk/users/tests/users.spec.ts
@@ -1,6 +1,5 @@
 import { structures, mocks } from "../../../tests"
 import { env, context } from "@budibase/backend-core"
-import * as users from "../users"
 import { db as userDb } from "../"
 import { CloudAccount } from "@budibase/types"
 

From f74264c1c82dce62fdbbe45c3e94369d9e857a74 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 16:37:34 +0100
Subject: [PATCH 60/68] Add tests

---
 .../backend-core/src/cache/passwordReset.ts   |  4 ++-
 .../worker/src/sdk/auth/tests/auth.spec.ts    | 25 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts
index a19e99745a..db32b520f7 100644
--- a/packages/backend-core/src/cache/passwordReset.ts
+++ b/packages/backend-core/src/cache/passwordReset.ts
@@ -32,7 +32,9 @@ export async function getCode(code: string): Promise<PasswordReset> {
   const client = await redis.getPasswordResetClient()
   const value = (await client.get(code)) as PasswordReset | undefined
   if (!value) {
-    throw "Provided information is not valid, cannot reset password - please try again."
+    throw new Error(
+      "Provided information is not valid, cannot reset password - please try again."
+    )
   }
   return value
 }
diff --git a/packages/worker/src/sdk/auth/tests/auth.spec.ts b/packages/worker/src/sdk/auth/tests/auth.spec.ts
index b1758e79c6..0d05a3fbb3 100644
--- a/packages/worker/src/sdk/auth/tests/auth.spec.ts
+++ b/packages/worker/src/sdk/auth/tests/auth.spec.ts
@@ -24,5 +24,30 @@ describe("auth", () => {
         ).toBeTruthy()
       })
     })
+
+    it("wrong code will not allow to reset the password", async () => {
+      await context.doInTenant(structures.tenant.id(), async () => {
+        const code = generator.hash()
+        const newPassword = generator.hash()
+
+        await expect(resetUpdate(code, newPassword)).rejects.toThrow(
+          "Provided information is not valid, cannot reset password - please try again."
+        )
+      })
+    })
+
+    it("the same code cannot be used twice", async () => {
+      await context.doInTenant(structures.tenant.id(), async () => {
+        const user = await config.createUser()
+
+        const code = await cache.passwordReset.createCode(user._id!, {})
+        const newPassword = generator.hash()
+
+        await resetUpdate(code, newPassword)
+        await expect(resetUpdate(code, newPassword)).rejects.toThrow(
+          "Provided information is not valid, cannot reset password - please try again."
+        )
+      })
+    })
   })
 })

From d1ffe242692570e4f04d147178710c319a2bfd8c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 16:54:47 +0100
Subject: [PATCH 61/68] Invalidate session on password update

---
 packages/worker/src/sdk/auth/auth.ts          |  1 +
 .../worker/src/sdk/auth/tests/auth.spec.ts    | 21 +++++++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts
index bdc5fc2366..3f24de440a 100644
--- a/packages/worker/src/sdk/auth/auth.ts
+++ b/packages/worker/src/sdk/auth/auth.ts
@@ -80,6 +80,7 @@ export const resetUpdate = async (resetCode: string, password: string) => {
   user = await userSdk.db.save(user)
 
   await cache.passwordReset.invalidateCode(resetCode)
+  await sessions.invalidateSessions(userId)
 
   // remove password from the user before sending events
   delete user.password
diff --git a/packages/worker/src/sdk/auth/tests/auth.spec.ts b/packages/worker/src/sdk/auth/tests/auth.spec.ts
index 0d05a3fbb3..e9f348f7c7 100644
--- a/packages/worker/src/sdk/auth/tests/auth.spec.ts
+++ b/packages/worker/src/sdk/auth/tests/auth.spec.ts
@@ -1,5 +1,5 @@
-import { cache, context, utils } from "@budibase/backend-core"
-import { resetUpdate } from "../auth"
+import { cache, context, sessions, utils } from "@budibase/backend-core"
+import { loginUser, resetUpdate } from "../auth"
 import { generator, structures } from "@budibase/backend-core/tests"
 import { TestConfiguration } from "../../../tests"
 
@@ -49,5 +49,22 @@ describe("auth", () => {
         )
       })
     })
+
+    it("updating the password will invalidate all the sessions", async () => {
+      await context.doInTenant(structures.tenant.id(), async () => {
+        const user = await config.createUser()
+
+        await loginUser(user)
+
+        expect(await sessions.getSessionsForUser(user._id!)).toHaveLength(1)
+
+        const code = await cache.passwordReset.createCode(user._id!, {})
+        const newPassword = generator.hash()
+
+        await resetUpdate(code, newPassword)
+
+        expect(await sessions.getSessionsForUser(user._id!)).toHaveLength(0)
+      })
+    })
   })
 })

From 865038d02f42bc6b23567359fdb3dc152a28a213 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 29 Dec 2023 17:24:51 +0000
Subject: [PATCH 62/68] Bump version to 2.13.51

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 671935c34b..bbe4da4264 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.50",
+  "version": "2.13.51",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From d277832f47b9c4bcb769e6f5c90dfbc7d6592256 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 2 Jan 2024 12:01:17 +0100
Subject: [PATCH 63/68] Fix enterprise license color

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index e46a352a63..ebdd4aa056 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit e46a352a6326a838faa00f912de069aee95d7300
+Subproject commit ebdd4aa0561dfcc3de3a906eead3979b680d039a

From e38624161ecc486fb49c862a8808c6c4701a842e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 2 Jan 2024 11:02:25 +0000
Subject: [PATCH 64/68] Update account-portal submodule ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index e46a352a63..af1ae16221 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit e46a352a6326a838faa00f912de069aee95d7300
+Subproject commit af1ae16221a4e6ababe8cc1f14803cbc849b2e85

From debf2b3d1e127f236bb09be0e3e910f9a7668263 Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Tue, 2 Jan 2024 12:06:18 +0100
Subject: [PATCH 65/68] Update account portal submodule

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index ebdd4aa056..d6a1f89aa5 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit ebdd4aa0561dfcc3de3a906eead3979b680d039a
+Subproject commit d6a1f89aa543bdce7acde5fbe4ce650a1344e2fe

From e26c39677dde005f5133d0401322d8474393f541 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jan 2024 12:31:08 +0100
Subject: [PATCH 66/68] Update messages

---
 packages/server/scripts/load/create-many-relationships.js | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 13f861ec0b..b81aed3d5d 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -90,9 +90,13 @@ async function run() {
   if (!appId) {
     const app = appId ? await getApp(apiKey, appId) : await createApp(apiKey)
     appId = app._id
-  }
 
-  console.log(`App created: http://localhost:10000/builder/app/${appId}`)
+    console.log(`App created. Url: http://localhost:10000/builder/app/${appId}`)
+  } else {
+    console.log(
+      `App retrieved. Url: http://localhost:10000/builder/app/${appId}`
+    )
+  }
 
   const studentsTable = await getTable(apiKey, appId, "Students")
 

From 642b75e0aeae5e50e9a38d847bd9efee091129fd Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 2 Jan 2024 11:36:32 +0000
Subject: [PATCH 67/68] Revert "Remove all custom tracing to see if it's the
 cause of the memory leak."

---
 .../backend-core/src/db/couch/DatabaseImpl.ts |   4 +-
 packages/backend-core/src/db/db.ts            |   3 +-
 .../backend-core/src/db/instrumentation.ts    | 156 +++++
 .../backend-core/src/logging/pino/logger.ts   |   6 +
 packages/server/src/automations/utils.ts      |  70 ++-
 packages/server/src/jsRunner.ts               |  53 +-
 packages/server/src/middleware/currentapp.ts  |  14 +
 packages/server/src/threads/automation.ts     | 583 ++++++++++--------
 .../src/utilities/rowProcessor/utils.ts       |  56 +-
 9 files changed, 622 insertions(+), 323 deletions(-)
 create mode 100644 packages/backend-core/src/db/instrumentation.ts

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index c2c0b6b21d..3fec573bb9 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -17,6 +17,7 @@ import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
+import { DDInstrumentedDatabase } from "../instrumentation"
 
 function buildNano(couchInfo: { url: string; cookie: string }) {
   return Nano({
@@ -35,7 +36,8 @@ export function DatabaseWithConnection(
   connection: string,
   opts?: DatabaseOpts
 ) {
-  return new DatabaseImpl(dbName, opts, connection)
+  const db = new DatabaseImpl(dbName, opts, connection)
+  return new DDInstrumentedDatabase(db)
 }
 
 export class DatabaseImpl implements Database {
diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts
index 3e69d49f0e..197770298e 100644
--- a/packages/backend-core/src/db/db.ts
+++ b/packages/backend-core/src/db/db.ts
@@ -1,8 +1,9 @@
 import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types"
+import { DDInstrumentedDatabase } from "./instrumentation"
 
 export function getDB(dbName: string, opts?: DatabaseOpts): Database {
-  return new DatabaseImpl(dbName, opts)
+  return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts))
 }
 
 // we have to use a callback for this so that we can close
diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
new file mode 100644
index 0000000000..ba5febcba6
--- /dev/null
+++ b/packages/backend-core/src/db/instrumentation.ts
@@ -0,0 +1,156 @@
+import {
+  DocumentScope,
+  DocumentDestroyResponse,
+  DocumentInsertResponse,
+  DocumentBulkResponse,
+  OkResponse,
+} from "@budibase/nano"
+import {
+  AllDocsResponse,
+  AnyDocument,
+  Database,
+  DatabaseDumpOpts,
+  DatabasePutOpts,
+  DatabaseQueryOpts,
+  Document,
+} from "@budibase/types"
+import tracer from "dd-trace"
+import { Writable } from "stream"
+
+export class DDInstrumentedDatabase implements Database {
+  constructor(private readonly db: Database) {}
+
+  get name(): string {
+    return this.db.name
+  }
+
+  exists(): Promise<boolean> {
+    return tracer.trace("db.exists", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.exists()
+    })
+  }
+
+  checkSetup(): Promise<DocumentScope<any>> {
+    return tracer.trace("db.checkSetup", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.checkSetup()
+    })
+  }
+
+  get<T extends Document>(id?: string | undefined): Promise<T> {
+    return tracer.trace("db.get", span => {
+      span?.addTags({ db_name: this.name, doc_id: id })
+      return this.db.get(id)
+    })
+  }
+
+  getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean | undefined } | undefined
+  ): Promise<T[]> {
+    return tracer.trace("db.getMultiple", span => {
+      span?.addTags({
+        db_name: this.name,
+        num_docs: ids.length,
+        allow_missing: opts?.allowMissing,
+      })
+      return this.db.getMultiple(ids, opts)
+    })
+  }
+
+  remove(
+    id: string | Document,
+    rev?: string | undefined
+  ): Promise<DocumentDestroyResponse> {
+    return tracer.trace("db.remove", span => {
+      span?.addTags({ db_name: this.name, doc_id: id })
+      return this.db.remove(id, rev)
+    })
+  }
+
+  put(
+    document: AnyDocument,
+    opts?: DatabasePutOpts | undefined
+  ): Promise<DocumentInsertResponse> {
+    return tracer.trace("db.put", span => {
+      span?.addTags({ db_name: this.name, doc_id: document._id })
+      return this.db.put(document, opts)
+    })
+  }
+
+  bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
+    return tracer.trace("db.bulkDocs", span => {
+      span?.addTags({ db_name: this.name, num_docs: documents.length })
+      return this.db.bulkDocs(documents)
+    })
+  }
+
+  allDocs<T extends Document>(
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
+    return tracer.trace("db.allDocs", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.allDocs(params)
+    })
+  }
+
+  query<T extends Document>(
+    viewName: string,
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
+    return tracer.trace("db.query", span => {
+      span?.addTags({ db_name: this.name, view_name: viewName })
+      return this.db.query(viewName, params)
+    })
+  }
+
+  destroy(): Promise<void | OkResponse> {
+    return tracer.trace("db.destroy", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.destroy()
+    })
+  }
+
+  compact(): Promise<void | OkResponse> {
+    return tracer.trace("db.compact", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.compact()
+    })
+  }
+
+  dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
+    return tracer.trace("db.dump", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.dump(stream, opts)
+    })
+  }
+
+  load(...args: any[]): Promise<any> {
+    return tracer.trace("db.load", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.load(...args)
+    })
+  }
+
+  createIndex(...args: any[]): Promise<any> {
+    return tracer.trace("db.createIndex", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.createIndex(...args)
+    })
+  }
+
+  deleteIndex(...args: any[]): Promise<any> {
+    return tracer.trace("db.deleteIndex", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.deleteIndex(...args)
+    })
+  }
+
+  getIndexes(...args: any[]): Promise<any> {
+    return tracer.trace("db.getIndexes", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.getIndexes(...args)
+    })
+  }
+}
diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts
index ad68bd300d..7a051e7f12 100644
--- a/packages/backend-core/src/logging/pino/logger.ts
+++ b/packages/backend-core/src/logging/pino/logger.ts
@@ -5,6 +5,7 @@ import { IdentityType } from "@budibase/types"
 import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
+import tracer from "dd-trace"
 import { formats } from "dd-trace/ext"
 
 import { localFileDestination } from "../system"
@@ -116,6 +117,11 @@ if (!env.DISABLE_PINO_LOGGER) {
       correlationId: correlation.getId(),
     }
 
+    const span = tracer.scope().active()
+    if (span) {
+      tracer.inject(span.context(), formats.LOG, contextObject)
+    }
+
     const mergingObject: any = {
       err: error,
       pid: process.pid,
diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index 04fd36e3d1..0c28787f67 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -16,6 +16,7 @@ import {
 } from "@budibase/types"
 import sdk from "../sdk"
 import { automationsEnabled } from "../features"
+import tracer from "dd-trace"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -39,27 +40,62 @@ function loggingArgs(job: AutomationJob) {
 }
 
 export async function processEvent(job: AutomationJob) {
-  const appId = job.data.event.appId!
-  const automationId = job.data.automation._id!
+  return tracer.trace(
+    "processEvent",
+    { resource: "automation" },
+    async span => {
+      const appId = job.data.event.appId!
+      const automationId = job.data.automation._id!
 
-  const task = async () => {
-    try {
-      // need to actually await these so that an error can be captured properly
-      console.log("automation running", ...loggingArgs(job))
-
-      const runFn = () => Runner.run(job)
-      const result = await quotas.addAutomation(runFn, {
+      span?.addTags({
+        appId,
         automationId,
+        job: {
+          id: job.id,
+          name: job.name,
+          attemptsMade: job.attemptsMade,
+          opts: {
+            attempts: job.opts.attempts,
+            priority: job.opts.priority,
+            delay: job.opts.delay,
+            repeat: job.opts.repeat,
+            backoff: job.opts.backoff,
+            lifo: job.opts.lifo,
+            timeout: job.opts.timeout,
+            jobId: job.opts.jobId,
+            removeOnComplete: job.opts.removeOnComplete,
+            removeOnFail: job.opts.removeOnFail,
+            stackTraceLimit: job.opts.stackTraceLimit,
+            preventParsingData: job.opts.preventParsingData,
+          },
+        },
       })
-      console.log("automation completed", ...loggingArgs(job))
-      return result
-    } catch (err) {
-      console.error(`automation was unable to run`, err, ...loggingArgs(job))
-      return { err }
-    }
-  }
 
-  return await context.doInAutomationContext({ appId, automationId, task })
+      const task = async () => {
+        try {
+          // need to actually await these so that an error can be captured properly
+          console.log("automation running", ...loggingArgs(job))
+
+          const runFn = () => Runner.run(job)
+          const result = await quotas.addAutomation(runFn, {
+            automationId,
+          })
+          console.log("automation completed", ...loggingArgs(job))
+          return result
+        } catch (err) {
+          span?.addTags({ error: true })
+          console.error(
+            `automation was unable to run`,
+            err,
+            ...loggingArgs(job)
+          )
+          return { err }
+        }
+      }
+
+      return await context.doInAutomationContext({ appId, automationId, task })
+    }
+  )
 }
 
 export async function updateTestHistory(
diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts
index a9301feb60..ab0381a399 100644
--- a/packages/server/src/jsRunner.ts
+++ b/packages/server/src/jsRunner.ts
@@ -2,35 +2,44 @@ import vm from "vm"
 import env from "./environment"
 import { setJSRunner } from "@budibase/string-templates"
 import { context, timers } from "@budibase/backend-core"
+import tracer from "dd-trace"
 
 type TrackerFn = <T>(f: () => T) => T
 
 export function init() {
   setJSRunner((js: string, ctx: vm.Context) => {
-    const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
-    let track: TrackerFn = f => f()
-    if (perRequestLimit) {
-      const bbCtx = context.getCurrentContext()
-      if (bbCtx) {
-        if (!bbCtx.jsExecutionTracker) {
-          bbCtx.jsExecutionTracker =
-            timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+    return tracer.trace("runJS", {}, span => {
+      const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
+      let track: TrackerFn = f => f()
+      if (perRequestLimit) {
+        const bbCtx = context.getCurrentContext()
+        if (bbCtx) {
+          if (!bbCtx.jsExecutionTracker) {
+            bbCtx.jsExecutionTracker =
+              timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+          }
+          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
+          span?.addTags({
+            js: {
+              limitMS: bbCtx.jsExecutionTracker.limitMs,
+              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+            },
+          })
         }
-        track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
       }
-    }
 
-    ctx = {
-      ...ctx,
-      alert: undefined,
-      setInterval: undefined,
-      setTimeout: undefined,
-    }
-    vm.createContext(ctx)
-    return track(() =>
-      vm.runInNewContext(js, ctx, {
-        timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-      })
-    )
+      ctx = {
+        ...ctx,
+        alert: undefined,
+        setInterval: undefined,
+        setTimeout: undefined,
+      }
+      vm.createContext(ctx)
+      return track(() =>
+        vm.runInNewContext(js, ctx, {
+          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+        })
+      )
+    })
   })
 }
diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts
index 984dd8e5e9..ad6f2afa18 100644
--- a/packages/server/src/middleware/currentapp.ts
+++ b/packages/server/src/middleware/currentapp.ts
@@ -12,6 +12,7 @@ import { getCachedSelf } from "../utilities/global"
 import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
 import { UserCtx, ContextUser } from "@budibase/types"
+import tracer from "dd-trace"
 
 export default async (ctx: UserCtx, next: any) => {
   // try to get the appID from the request
@@ -20,6 +21,11 @@ export default async (ctx: UserCtx, next: any) => {
     return next()
   }
 
+  if (requestAppId) {
+    const span = tracer.scope().active()
+    span?.setTag("appId", requestAppId)
+  }
+
   // deny access to application preview
   if (!env.isTest()) {
     if (
@@ -70,6 +76,14 @@ export default async (ctx: UserCtx, next: any) => {
     return next()
   }
 
+  if (ctx.user) {
+    const span = tracer.scope().active()
+    if (ctx.user._id) {
+      span?.setTag("userId", ctx.user._id)
+    }
+    span?.setTag("tenantId", ctx.user.tenantId)
+  }
+
   const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined
 
   // if the user is not in the right tenant then make sure to wipe their cookie
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 9bb1717f3c..4447899f96 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -34,6 +34,7 @@ import { cloneDeep } from "lodash/fp"
 import { performance } from "perf_hooks"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
+import tracer from "dd-trace"
 
 threadUtils.threadSetup()
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
@@ -242,281 +243,347 @@ class Orchestrator {
   }
 
   async execute(): Promise<any> {
-    // this will retrieve from context created at start of thread
-    this._context.env = await sdkUtils.getEnvironmentVariables()
-    let automation = this._automation
-    let stopped = false
-    let loopStep: AutomationStep | undefined = undefined
+    return tracer.trace(
+      "Orchestrator.execute",
+      { resource: "automation" },
+      async span => {
+        span?.addTags({
+          appId: this._appId,
+          automationId: this._automation._id,
+        })
 
-    let stepCount = 0
-    let loopStepNumber: any = undefined
-    let loopSteps: LoopStep[] | undefined = []
-    let metadata
-    let timeoutFlag = false
-    let wasLoopStep = false
-    let timeout = this._job.data.event.timeout
-    // check if this is a recurring automation,
-    if (isProdAppID(this._appId) && isRecurring(automation)) {
-      metadata = await this.getMetadata()
-      const shouldStop = await this.checkIfShouldStop(metadata)
-      if (shouldStop) {
-        return
-      }
-    }
-    const start = performance.now()
-    for (let step of automation.definition.steps) {
-      let input: any,
-        iterations = 1,
-        iterationCount = 0
+        // this will retrieve from context created at start of thread
+        this._context.env = await sdkUtils.getEnvironmentVariables()
+        let automation = this._automation
+        let stopped = false
+        let loopStep: AutomationStep | undefined = undefined
 
-      if (timeoutFlag) {
-        break
-      }
-
-      if (timeout) {
-        setTimeout(() => {
-          timeoutFlag = true
-        }, timeout || 12000)
-      }
-
-      stepCount++
-      if (step.stepId === LOOP_STEP_ID) {
-        loopStep = step
-        loopStepNumber = stepCount
-        continue
-      }
-
-      if (loopStep) {
-        input = await processObject(loopStep.inputs, this._context)
-        iterations = getLoopIterations(loopStep as LoopStep)
-      }
-      for (let index = 0; index < iterations; index++) {
-        let originalStepInput = cloneDeep(step.inputs)
-        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
-        if (loopStep && input.binding) {
-          let tempOutput = {
-            items: loopSteps,
-            iterations: iterationCount,
-          }
-          try {
-            loopStep.inputs.binding = automationUtils.typecastForLooping(
-              loopStep as LoopStep,
-              loopStep.inputs as LoopInput
-            )
-          } catch (err) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.INCORRECT_TYPE,
-              success: false,
-            })
-            loopSteps = undefined
-            loopStep = undefined
-            break
-          }
-          let item = []
-          if (
-            typeof loopStep.inputs.binding === "string" &&
-            loopStep.inputs.option === "String"
-          ) {
-            item = automationUtils.stringSplit(loopStep.inputs.binding)
-          } else if (Array.isArray(loopStep.inputs.binding)) {
-            item = loopStep.inputs.binding
-          }
-          this._context.steps[loopStepNumber] = {
-            currentItem: item[index],
-          }
-
-          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
-          // Pretty hacky because we need to account for the row object
-          for (let [key, value] of Object.entries(originalStepInput)) {
-            if (typeof value === "object") {
-              for (let [innerKey, innerValue] of Object.entries(
-                originalStepInput[key]
-              )) {
-                if (typeof innerValue === "string") {
-                  originalStepInput[key][innerKey] =
-                    automationUtils.substituteLoopStep(
-                      innerValue,
-                      `steps.${loopStepNumber}`
-                    )
-                } else if (typeof value === "object") {
-                  for (let [innerObject, innerValue] of Object.entries(
-                    originalStepInput[key][innerKey]
-                  )) {
-                    originalStepInput[key][innerKey][innerObject] =
-                      automationUtils.substituteLoopStep(
-                        innerValue as string,
-                        `steps.${loopStepNumber}`
-                      )
-                  }
-                }
-              }
-            } else {
-              if (typeof value === "string") {
-                originalStepInput[key] = automationUtils.substituteLoopStep(
-                  value,
-                  `steps.${loopStepNumber}`
-                )
-              }
-            }
-          }
-
-          if (
-            index === env.AUTOMATION_MAX_ITERATIONS ||
-            index === parseInt(loopStep.inputs.iterations)
-          ) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.MAX_ITERATIONS,
-              success: true,
-            })
-            loopSteps = undefined
-            loopStep = undefined
-            break
-          }
-
-          let isFailure = false
-          const currentItem = this._context.steps[loopStepNumber]?.currentItem
-          if (currentItem && typeof currentItem === "object") {
-            isFailure = Object.keys(currentItem).some(value => {
-              return currentItem[value] === loopStep?.inputs.failure
-            })
-          } else {
-            isFailure = currentItem && currentItem === loopStep.inputs.failure
-          }
-
-          if (isFailure) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.FAILURE_CONDITION,
-              success: false,
-            })
-            loopSteps = undefined
-            loopStep = undefined
-            break
+        let stepCount = 0
+        let loopStepNumber: any = undefined
+        let loopSteps: LoopStep[] | undefined = []
+        let metadata
+        let timeoutFlag = false
+        let wasLoopStep = false
+        let timeout = this._job.data.event.timeout
+        // check if this is a recurring automation,
+        if (isProdAppID(this._appId) && isRecurring(automation)) {
+          span?.addTags({ recurring: true })
+          metadata = await this.getMetadata()
+          const shouldStop = await this.checkIfShouldStop(metadata)
+          if (shouldStop) {
+            span?.addTags({ shouldStop: true })
+            return
           }
         }
-
-        // execution stopped, record state for that
-        if (stopped) {
-          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
-          continue
-        }
-
-        // If it's a loop step, we need to manually add the bindings to the context
-        let stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await processObject(originalStepInput, this._context)
-        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
-
-        try {
-          // appId is always passed
-          const outputs = await stepFn({
-            inputs: inputs,
-            appId: this._appId,
-            emitter: this._emitter,
-            context: this._context,
+        const start = performance.now()
+        for (let step of automation.definition.steps) {
+          const stepSpan = tracer.startSpan("Orchestrator.execute.step", {
+            childOf: span,
+          })
+          stepSpan.addTags({
+            resource: "automation",
+            step: {
+              stepId: step.stepId,
+              id: step.id,
+              name: step.name,
+              type: step.type,
+              title: step.stepTitle,
+              internal: step.internal,
+              deprecated: step.deprecated,
+            },
           })
 
-          this._context.steps[stepCount] = outputs
-          // if filter causes us to stop execution don't break the loop, set a var
-          // so that we can finish iterating through the steps and record that it stopped
-          if (step.stepId === FILTER_STEP_ID && !outputs.result) {
-            stopped = true
-            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-              ...outputs,
-              ...STOPPED_STATUS,
-            })
-            continue
-          }
-          if (loopStep && loopSteps) {
-            loopSteps.push(outputs)
-          } else {
-            this.updateExecutionOutput(
-              step.id,
-              step.stepId,
-              step.inputs,
-              outputs
-            )
-          }
-        } catch (err) {
-          console.error(`Automation error - ${step.stepId} - ${err}`)
-          return err
-        }
+          let input: any,
+            iterations = 1,
+            iterationCount = 0
 
-        if (loopStep) {
-          iterationCount++
-          if (index === iterations - 1) {
+          try {
+            if (timeoutFlag) {
+              span?.addTags({ timedOut: true })
+              break
+            }
+
+            if (timeout) {
+              setTimeout(() => {
+                timeoutFlag = true
+              }, timeout || 12000)
+            }
+
+            stepCount++
+            if (step.stepId === LOOP_STEP_ID) {
+              loopStep = step
+              loopStepNumber = stepCount
+              continue
+            }
+
+            if (loopStep) {
+              input = await processObject(loopStep.inputs, this._context)
+              iterations = getLoopIterations(loopStep as LoopStep)
+              stepSpan?.addTags({ step: { iterations } })
+            }
+            for (let index = 0; index < iterations; index++) {
+              let originalStepInput = cloneDeep(step.inputs)
+              // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+              if (loopStep && input.binding) {
+                let tempOutput = {
+                  items: loopSteps,
+                  iterations: iterationCount,
+                }
+                try {
+                  loopStep.inputs.binding = automationUtils.typecastForLooping(
+                    loopStep as LoopStep,
+                    loopStep.inputs as LoopInput
+                  )
+                } catch (err) {
+                  this.updateContextAndOutput(
+                    loopStepNumber,
+                    step,
+                    tempOutput,
+                    {
+                      status: AutomationErrors.INCORRECT_TYPE,
+                      success: false,
+                    }
+                  )
+                  loopSteps = undefined
+                  loopStep = undefined
+                  break
+                }
+                let item = []
+                if (
+                  typeof loopStep.inputs.binding === "string" &&
+                  loopStep.inputs.option === "String"
+                ) {
+                  item = automationUtils.stringSplit(loopStep.inputs.binding)
+                } else if (Array.isArray(loopStep.inputs.binding)) {
+                  item = loopStep.inputs.binding
+                }
+                this._context.steps[loopStepNumber] = {
+                  currentItem: item[index],
+                }
+
+                // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+                // Pretty hacky because we need to account for the row object
+                for (let [key, value] of Object.entries(originalStepInput)) {
+                  if (typeof value === "object") {
+                    for (let [innerKey, innerValue] of Object.entries(
+                      originalStepInput[key]
+                    )) {
+                      if (typeof innerValue === "string") {
+                        originalStepInput[key][innerKey] =
+                          automationUtils.substituteLoopStep(
+                            innerValue,
+                            `steps.${loopStepNumber}`
+                          )
+                      } else if (typeof value === "object") {
+                        for (let [innerObject, innerValue] of Object.entries(
+                          originalStepInput[key][innerKey]
+                        )) {
+                          originalStepInput[key][innerKey][innerObject] =
+                            automationUtils.substituteLoopStep(
+                              innerValue as string,
+                              `steps.${loopStepNumber}`
+                            )
+                        }
+                      }
+                    }
+                  } else {
+                    if (typeof value === "string") {
+                      originalStepInput[key] =
+                        automationUtils.substituteLoopStep(
+                          value,
+                          `steps.${loopStepNumber}`
+                        )
+                    }
+                  }
+                }
+
+                if (
+                  index === env.AUTOMATION_MAX_ITERATIONS ||
+                  index === parseInt(loopStep.inputs.iterations)
+                ) {
+                  this.updateContextAndOutput(
+                    loopStepNumber,
+                    step,
+                    tempOutput,
+                    {
+                      status: AutomationErrors.MAX_ITERATIONS,
+                      success: true,
+                    }
+                  )
+                  loopSteps = undefined
+                  loopStep = undefined
+                  break
+                }
+
+                let isFailure = false
+                const currentItem =
+                  this._context.steps[loopStepNumber]?.currentItem
+                if (currentItem && typeof currentItem === "object") {
+                  isFailure = Object.keys(currentItem).some(value => {
+                    return currentItem[value] === loopStep?.inputs.failure
+                  })
+                } else {
+                  isFailure =
+                    currentItem && currentItem === loopStep.inputs.failure
+                }
+
+                if (isFailure) {
+                  this.updateContextAndOutput(
+                    loopStepNumber,
+                    step,
+                    tempOutput,
+                    {
+                      status: AutomationErrors.FAILURE_CONDITION,
+                      success: false,
+                    }
+                  )
+                  loopSteps = undefined
+                  loopStep = undefined
+                  break
+                }
+              }
+
+              // execution stopped, record state for that
+              if (stopped) {
+                this.updateExecutionOutput(
+                  step.id,
+                  step.stepId,
+                  {},
+                  STOPPED_STATUS
+                )
+                continue
+              }
+
+              // If it's a loop step, we need to manually add the bindings to the context
+              let stepFn = await this.getStepFunctionality(step.stepId)
+              let inputs = await processObject(originalStepInput, this._context)
+              inputs = automationUtils.cleanInputValues(
+                inputs,
+                step.schema.inputs
+              )
+
+              try {
+                // appId is always passed
+                const outputs = await stepFn({
+                  inputs: inputs,
+                  appId: this._appId,
+                  emitter: this._emitter,
+                  context: this._context,
+                })
+
+                this._context.steps[stepCount] = outputs
+                // if filter causes us to stop execution don't break the loop, set a var
+                // so that we can finish iterating through the steps and record that it stopped
+                if (step.stepId === FILTER_STEP_ID && !outputs.result) {
+                  stopped = true
+                  this.updateExecutionOutput(
+                    step.id,
+                    step.stepId,
+                    step.inputs,
+                    {
+                      ...outputs,
+                      ...STOPPED_STATUS,
+                    }
+                  )
+                  continue
+                }
+                if (loopStep && loopSteps) {
+                  loopSteps.push(outputs)
+                } else {
+                  this.updateExecutionOutput(
+                    step.id,
+                    step.stepId,
+                    step.inputs,
+                    outputs
+                  )
+                }
+              } catch (err) {
+                console.error(`Automation error - ${step.stepId} - ${err}`)
+                return err
+              }
+
+              if (loopStep) {
+                iterationCount++
+                if (index === iterations - 1) {
+                  loopStep = undefined
+                  this._context.steps.splice(loopStepNumber, 1)
+                  break
+                }
+              }
+            }
+          } finally {
+            stepSpan?.finish()
+          }
+
+          if (loopStep && iterations === 0) {
             loopStep = undefined
+            this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+              id: step.id,
+              stepId: step.stepId,
+              outputs: {
+                status: AutomationStepStatus.NO_ITERATIONS,
+                success: true,
+              },
+              inputs: {},
+            })
+
             this._context.steps.splice(loopStepNumber, 1)
-            break
+            iterations = 1
+          }
+
+          // Delete the step after the loop step as it's irrelevant, since information is included
+          // in the loop step
+          if (wasLoopStep && !loopStep) {
+            this._context.steps.splice(loopStepNumber + 1, 1)
+            wasLoopStep = false
+          }
+          if (loopSteps && loopSteps.length) {
+            let tempOutput = {
+              success: true,
+              items: loopSteps,
+              iterations: iterationCount,
+            }
+            this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+              id: step.id,
+              stepId: step.stepId,
+              outputs: tempOutput,
+              inputs: step.inputs,
+            })
+            this._context.steps[loopStepNumber] = tempOutput
+
+            wasLoopStep = true
+            loopSteps = []
           }
         }
-      }
 
-      if (loopStep && iterations === 0) {
-        loopStep = undefined
-        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-          id: step.id,
-          stepId: step.stepId,
-          outputs: {
-            status: AutomationStepStatus.NO_ITERATIONS,
-            success: true,
-          },
-          inputs: {},
-        })
+        const end = performance.now()
+        const executionTime = end - start
 
-        this._context.steps.splice(loopStepNumber, 1)
-        iterations = 1
-      }
+        console.info(
+          `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
+          {
+            _logKey: "automation",
+            executionTime,
+          }
+        )
 
-      // Delete the step after the loop step as it's irrelevant, since information is included
-      // in the loop step
-      if (wasLoopStep && !loopStep) {
-        this._context.steps.splice(loopStepNumber + 1, 1)
-        wasLoopStep = false
-      }
-      if (loopSteps && loopSteps.length) {
-        let tempOutput = {
-          success: true,
-          items: loopSteps,
-          iterations: iterationCount,
+        // store the logs for the automation run
+        try {
+          await storeLog(this._automation, this.executionOutput)
+        } catch (e: any) {
+          if (e.status === 413 && e.request?.data) {
+            // if content is too large we shouldn't log it
+            delete e.request.data
+            e.request.data = { message: "removed due to large size" }
+          }
+          logging.logAlert("Error writing automation log", e)
         }
-        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-          id: step.id,
-          stepId: step.stepId,
-          outputs: tempOutput,
-          inputs: step.inputs,
-        })
-        this._context.steps[loopStepNumber] = tempOutput
-
-        wasLoopStep = true
-        loopSteps = []
-      }
-    }
-
-    const end = performance.now()
-    const executionTime = end - start
-
-    console.info(
-      `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
-      {
-        _logKey: "automation",
-        executionTime,
+        if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
+          await this.updateMetadata(metadata)
+        }
+        return this.executionOutput
       }
     )
-
-    // store the logs for the automation run
-    try {
-      await storeLog(this._automation, this.executionOutput)
-    } catch (e: any) {
-      if (e.status === 413 && e.request?.data) {
-        // if content is too large we shouldn't log it
-        delete e.request.data
-        e.request.data = { message: "removed due to large size" }
-      }
-      logging.logAlert("Error writing automation log", e)
-    }
-    if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
-      await this.updateMetadata(metadata)
-    }
-    return this.executionOutput
   }
 }
 
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 22c099c58c..cafd366cae 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -11,6 +11,7 @@ import {
   Row,
   Table,
 } from "@budibase/types"
+import tracer from "dd-trace"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -50,35 +51,42 @@ export function processFormulas<T extends Row | Row[]>(
   inputRows: T,
   { dynamic, contextRows }: FormulaOpts = { dynamic: true }
 ): T {
-  const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
-  if (rows) {
-    for (let [column, schema] of Object.entries(table.schema)) {
-      if (schema.type !== FieldTypes.FORMULA) {
-        continue
-      }
+  return tracer.trace("processFormulas", {}, span => {
+    const numRows = Array.isArray(inputRows) ? inputRows.length : 1
+    span?.addTags({ table_id: table._id, dynamic, numRows })
+    const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+    if (rows) {
+      for (let [column, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.FORMULA) {
+          continue
+        }
 
-      const isStatic = schema.formulaType === FormulaTypes.STATIC
+        const isStatic = schema.formulaType === FormulaTypes.STATIC
 
-      if (
-        schema.formula == null ||
-        (dynamic && isStatic) ||
-        (!dynamic && !isStatic)
-      ) {
-        continue
-      }
-      // iterate through rows and process formula
-      for (let i = 0; i < rows.length; i++) {
-        let row = rows[i]
-        let context = contextRows ? contextRows[i] : row
-        let formula = schema.formula
-        rows[i] = {
-          ...row,
-          [column]: processStringSync(formula, context),
+        if (
+          schema.formula == null ||
+          (dynamic && isStatic) ||
+          (!dynamic && !isStatic)
+        ) {
+          continue
+        }
+        // iterate through rows and process formula
+        for (let i = 0; i < rows.length; i++) {
+          let row = rows[i]
+          let context = contextRows ? contextRows[i] : row
+          let formula = schema.formula
+          rows[i] = {
+            ...row,
+            [column]: tracer.trace("processStringSync", {}, span => {
+              span?.addTags({ table_id: table._id, column, static: isStatic })
+              return processStringSync(formula, context)
+            }),
+          }
         }
       }
     }
-  }
-  return Array.isArray(inputRows) ? rows : rows[0]
+    return Array.isArray(inputRows) ? rows : rows[0]
+  })
 }
 
 /**

From 5392d89fde6e1cc920d79c251e443d8ea61dde82 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jan 2024 12:38:50 +0100
Subject: [PATCH 68/68] Update account-portal ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index d6a1f89aa5..c1a53bb2f4 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit d6a1f89aa543bdce7acde5fbe4ce650a1344e2fe
+Subproject commit c1a53bb2f4cafcb4c55ad7181146617b449907f2