From 75c2392b601d70931971b1368d27041679dc31bb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 8 Feb 2024 15:32:38 +0000
Subject: [PATCH 01/34] Flesh out Postgres tests, add MySQL tests.

---
 .../api/routes/tests/queries/mysql.spec.ts    | 239 ++++++++++++++++++
 .../api/routes/tests/queries/postgres.spec.ts |  73 ++++++
 .../src/integrations/tests/utils/index.ts     |   3 +-
 .../src/integrations/tests/utils/mysql.ts     |  52 ++++
 .../src/integrations/tests/utils/postgres.ts  |   4 +-
 5 files changed, 367 insertions(+), 4 deletions(-)
 create mode 100644 packages/server/src/api/routes/tests/queries/mysql.spec.ts
 create mode 100644 packages/server/src/integrations/tests/utils/mysql.ts

diff --git a/packages/server/src/api/routes/tests/queries/mysql.spec.ts b/packages/server/src/api/routes/tests/queries/mysql.spec.ts
new file mode 100644
index 0000000000..1c9c1d3865
--- /dev/null
+++ b/packages/server/src/api/routes/tests/queries/mysql.spec.ts
@@ -0,0 +1,239 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import mysql from "mysql2/promise"
+
+jest.unmock("mysql2")
+jest.unmock("mysql2/promise")
+
+const createTableSQL = `
+CREATE TABLE test_table (
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    name VARCHAR(50) NOT NULL
+)
+`
+
+const insertSQL = `
+INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')
+`
+
+const dropTableSQL = `
+DROP TABLE test_table
+`
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    return await config.api.query.create({ ...defaultQuery, ...query })
+  }
+
+  async function withConnection(
+    callback: (client: mysql.Connection) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.mysql.datasource()
+    const con = await mysql.createConnection(ds.config!)
+    try {
+      await callback(con)
+    } finally {
+      con.end()
+    }
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.mysql.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.mysql.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withConnection(async connection => {
+      const resp = await connection.query(createTableSQL)
+      await connection.query(insertSQL)
+    })
+  })
+
+  afterEach(async () => {
+    await withConnection(async connection => {
+      await connection.query(dropTableSQL)
+    })
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table ORDER BY id",
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 1,
+        name: "one",
+      },
+      {
+        id: 2,
+        name: "two",
+      },
+      {
+        id: 3,
+        name: "three",
+      },
+      {
+        id: 4,
+        name: "four",
+      },
+      {
+        id: 5,
+        name: "five",
+      },
+    ])
+  })
+
+  it("should be able to transform a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table WHERE id = 1",
+      },
+      transformer: `
+        data[0].id = data[0].id + 1;
+        return data;
+      `,
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 2,
+        name: "one",
+      },
+    ])
+  })
+
+  it("should be able to insert with bindings", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+      },
+      parameters: [
+        {
+          name: "foo",
+          default: "bar",
+        },
+      ],
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        foo: "baz",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        created: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE name = 'baz'"
+      )
+      expect(rows).toHaveLength(1)
+    })
+  })
+
+  it("should be able to update rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+        {
+          name: "name",
+          default: "updated",
+        },
+      ],
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+        name: "foo",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        updated: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toEqual([{ id: 1, name: "foo" }])
+    })
+  })
+
+  it("should be able to delete rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "DELETE FROM test_table WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+      ],
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        deleted: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toHaveLength(0)
+    })
+  })
+})
diff --git a/packages/server/src/api/routes/tests/queries/postgres.spec.ts b/packages/server/src/api/routes/tests/queries/postgres.spec.ts
index 487644e787..fd6a2b7d3c 100644
--- a/packages/server/src/api/routes/tests/queries/postgres.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/postgres.spec.ts
@@ -167,4 +167,77 @@ describe("/queries", () => {
       expect(rows).toHaveLength(1)
     })
   })
+
+  it("should be able to update rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+        {
+          name: "name",
+          default: "updated",
+        },
+      ],
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+        name: "foo",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        updated: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toEqual([{ id: 1, name: "foo" }])
+    })
+  })
+
+  it("should be able to delete rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "DELETE FROM test_table WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+      ],
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        deleted: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toHaveLength(0)
+    })
+  })
 })
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 77fb5d7128..b6e4e43e7a 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -3,6 +3,7 @@ jest.unmock("pg")
 import { Datasource } from "@budibase/types"
 import * as postgres from "./postgres"
 import * as mongodb from "./mongodb"
+import * as mysql from "./mysql"
 import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
@@ -13,4 +14,4 @@ export interface DatabaseProvider {
   datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = { postgres, mongodb }
+export const databaseTestProviders = { postgres, mongodb, mysql }
diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts
new file mode 100644
index 0000000000..2c44dd1373
--- /dev/null
+++ b/packages/server/src/integrations/tests/utils/mysql.ts
@@ -0,0 +1,52 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer("mysql:8.3")
+    .withExposedPorts(3306)
+    .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
+    .withWaitStrategy(
+      Wait.forSuccessfulCommand(
+        // Because MySQL first starts itself up, runs an init script, then restarts,
+        // it's possible for the mysqladmin ping to succeed early and then tests to
+        // run against a MySQL that's mid-restart and fail. To avoid this, we run
+        // the ping command three times with a small delay between each.
+        `
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 0.5 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 0.5 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword
+        `
+      )
+    )
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(3306)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.MYSQL,
+    plus: true,
+    config: {
+      host,
+      port,
+      user: "root",
+      password: "password",
+      database: "mysql",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 82a62e3916..4bf42c7f88 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -8,9 +8,7 @@ export async function start(): Promise<StartedTestContainer> {
     .withExposedPorts(5432)
     .withEnvironment({ POSTGRES_PASSWORD: "password" })
     .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        "pg_isready -h localhost -p 5432"
-      ).withStartupTimeout(10000)
+      Wait.forSuccessfulCommand("pg_isready -h localhost -p 5432")
     )
     .start()
 }

From 169fec29c6e23ecc5b5c26711406d58f5c6de282 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 13 Feb 2024 18:21:41 +0000
Subject: [PATCH 02/34] Some quick fixes, making sure that automation queries
 respect timeout, they will timeout within the usual range.

---
 packages/backend-core/src/utils/utils.ts          | 15 +++++++++++++--
 .../components/integration/RestQueryViewer.svelte |  2 +-
 packages/server/src/automations/bullboard.ts      |  2 +-
 .../server/src/automations/steps/executeQuery.ts  |  7 ++++++-
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts
index 30cf55b149..046f8aaf94 100644
--- a/packages/backend-core/src/utils/utils.ts
+++ b/packages/backend-core/src/utils/utils.ts
@@ -226,8 +226,19 @@ export function isClient(ctx: Ctx) {
   return ctx.headers[Header.TYPE] === "client"
 }
 
-export function timeout(timeMs: number) {
-  return new Promise(resolve => setTimeout(resolve, timeMs))
+export function timeout(
+  timeMs: number,
+  opts?: { reject?: boolean }
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    setTimeout(() => {
+      if (opts?.reject) {
+        reject(new Error(`timed out - ${timeMs}ms`))
+      } else {
+        resolve()
+      }
+    }, timeMs)
+  })
 }
 
 export function isAudited(event: Event) {
diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte
index d6a8fe6fc3..74aaa5386f 100644
--- a/packages/builder/src/components/integration/RestQueryViewer.svelte
+++ b/packages/builder/src/components/integration/RestQueryViewer.svelte
@@ -159,7 +159,7 @@
     newQuery.fields.queryString = queryString
     newQuery.fields.authConfigId = authConfigId
     newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
-    newQuery.schema = schema
+    newQuery.schema = schema || {}
 
     return newQuery
   }
diff --git a/packages/server/src/automations/bullboard.ts b/packages/server/src/automations/bullboard.ts
index df784eacff..34f18754a2 100644
--- a/packages/server/src/automations/bullboard.ts
+++ b/packages/server/src/automations/bullboard.ts
@@ -15,7 +15,7 @@ const PATH_PREFIX = "/bulladmin"
 
 export async function init() {
   // Set up queues for bull board admin
-  const backupQueue = await backups.getBackupQueue()
+  const backupQueue = backups.getBackupQueue()
   const queues = [automationQueue]
   if (backupQueue) {
     queues.push(backupQueue)
diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts
index a9517b01a0..d3dc6a219c 100644
--- a/packages/server/src/automations/steps/executeQuery.ts
+++ b/packages/server/src/automations/steps/executeQuery.ts
@@ -10,6 +10,8 @@ import {
   AutomationStepSchema,
   AutomationStepType,
 } from "@budibase/types"
+import { utils } from "@budibase/backend-core"
+import env from "../../environment"
 
 export const definition: AutomationStepSchema = {
   name: "External Data Connector",
@@ -84,7 +86,10 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
   })
 
   try {
-    await queryController.executeV2(ctx, { isAutomation: true })
+    await Promise.race([
+      queryController.executeV2(ctx, { isAutomation: true }),
+      utils.timeout(env.QUERY_THREAD_TIMEOUT, { reject: true }),
+    ])
     const { data, ...rest } = ctx.body
 
     return {

From cfd1c98c8571cdb1301a236dd0210ad63dd10ebb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 14 Feb 2024 11:44:07 +0000
Subject: [PATCH 03/34] Updating timeouts for automations, making it easier to
 manage/find the various timeout limits. Also adding a new environment
 variable AUTOMATION_THREAD_TIMEOUT which can be used to control how long
 automations can run for.

---
 .../server/src/api/controllers/automation.ts  | 31 +++++++++----------
 .../server/src/api/controllers/query/index.ts |  2 +-
 packages/server/src/automations/steps/bash.ts |  2 +-
 .../src/automations/steps/executeQuery.ts     |  5 +--
 .../automations/steps/triggerAutomationRun.ts |  5 +--
 .../tests/triggerAutomationRun.spec.ts        | 13 ++++++--
 packages/server/src/constants/automations.ts  |  4 +++
 packages/server/src/constants/index.ts        |  2 ++
 packages/server/src/environment.ts            |  4 ++-
 packages/server/src/threads/automation.ts     |  4 +--
 10 files changed, 43 insertions(+), 29 deletions(-)
 create mode 100644 packages/server/src/constants/automations.ts

diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts
index 6feba9fb2d..212ef5668f 100644
--- a/packages/server/src/api/controllers/automation.ts
+++ b/packages/server/src/api/controllers/automation.ts
@@ -1,4 +1,3 @@
-import * as actions from "../../automations/actions"
 import * as triggers from "../../automations/triggers"
 import {
   getAutomationParams,
@@ -11,7 +10,7 @@ import {
   removeDeprecated,
 } from "../../automations/utils"
 import { deleteEntityMetadata } from "../../utilities"
-import { MetadataTypes } from "../../constants"
+import { MetadataTypes, AUTOMATION_SYNC_TIMEOUT } from "../../constants"
 import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events, db as dbCore } from "@budibase/backend-core"
 import { automations, features } from "@budibase/pro"
@@ -20,7 +19,7 @@ import {
   Automation,
   AutomationActionStepId,
   AutomationResults,
-  BBContext,
+  UserCtx,
 } from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
 import sdk from "../../sdk"
@@ -72,7 +71,7 @@ function cleanAutomationInputs(automation: Automation) {
   return automation
 }
 
-export async function create(ctx: BBContext) {
+export async function create(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -141,7 +140,7 @@ export async function handleStepEvents(
   }
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -192,7 +191,7 @@ export async function update(ctx: BBContext) {
   builderSocket?.emitAutomationUpdate(ctx, automation)
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const db = context.getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
@@ -202,12 +201,12 @@ export async function fetch(ctx: BBContext) {
   ctx.body = response.rows.map(row => row.doc)
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const db = context.getAppDB()
   ctx.body = await db.get(ctx.params.id)
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get<Automation>(automationId)
@@ -221,11 +220,11 @@ export async function destroy(ctx: BBContext) {
   builderSocket?.emitAutomationDeletion(ctx, automationId)
 }
 
-export async function logSearch(ctx: BBContext) {
+export async function logSearch(ctx: UserCtx) {
   ctx.body = await automations.logs.logSearch(ctx.request.body)
 }
 
-export async function clearLogError(ctx: BBContext) {
+export async function clearLogError(ctx: UserCtx) {
   const { automationId, appId } = ctx.request.body
   await context.doInAppContext(appId, async () => {
     const db = context.getProdAppDB()
@@ -244,15 +243,15 @@ export async function clearLogError(ctx: BBContext) {
   })
 }
 
-export async function getActionList(ctx: BBContext) {
+export async function getActionList(ctx: UserCtx) {
   ctx.body = await getActionDefinitions()
 }
 
-export async function getTriggerList(ctx: BBContext) {
+export async function getTriggerList(ctx: UserCtx) {
   ctx.body = getTriggerDefinitions()
 }
 
-export async function getDefinitionList(ctx: BBContext) {
+export async function getDefinitionList(ctx: UserCtx) {
   ctx.body = {
     trigger: getTriggerDefinitions(),
     action: await getActionDefinitions(),
@@ -265,7 +264,7 @@ export async function getDefinitionList(ctx: BBContext) {
  *                   *
  *********************/
 
-export async function trigger(ctx: BBContext) {
+export async function trigger(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
 
@@ -275,7 +274,7 @@ export async function trigger(ctx: BBContext) {
       automation,
       {
         fields: ctx.request.body.fields,
-        timeout: ctx.request.body.timeout * 1000 || 120000,
+        timeout: ctx.request.body.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT,
       },
       { getResponses: true }
     )
@@ -310,7 +309,7 @@ function prepareTestInput(input: any) {
   return input
 }
 
-export async function test(ctx: BBContext) {
+export async function test(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
   await setTestFlag(automation._id!)
diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts
index 1be836b169..8dabe5b3cc 100644
--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@@ -22,7 +22,7 @@ import {
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
-  timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
+  timeoutMs: env.QUERY_THREAD_TIMEOUT,
 })
 
 // simple function to append "readable" to all read queries
diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts
index 61d446f12c..1a13f651ec 100644
--- a/packages/server/src/automations/steps/bash.ts
+++ b/packages/server/src/automations/steps/bash.ts
@@ -65,7 +65,7 @@ export async function run({ inputs, context }: AutomationStepInput) {
       success = true
     try {
       stdout = execSync(command, {
-        timeout: environment.QUERY_THREAD_TIMEOUT || 500,
+        timeout: environment.QUERY_THREAD_TIMEOUT,
       }).toString()
     } catch (err: any) {
       stdout = err.message
diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts
index d3dc6a219c..ea0737c86a 100644
--- a/packages/server/src/automations/steps/executeQuery.ts
+++ b/packages/server/src/automations/steps/executeQuery.ts
@@ -86,10 +86,7 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
   })
 
   try {
-    await Promise.race([
-      queryController.executeV2(ctx, { isAutomation: true }),
-      utils.timeout(env.QUERY_THREAD_TIMEOUT, { reject: true }),
-    ])
+    await queryController.executeV2(ctx, { isAutomation: true })
     const { data, ...rest } = ctx.body
 
     return {
diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts
index cb6126ca01..73c9b3adf1 100644
--- a/packages/server/src/automations/steps/triggerAutomationRun.ts
+++ b/packages/server/src/automations/steps/triggerAutomationRun.ts
@@ -9,8 +9,9 @@ import {
   AutomationCustomIOType,
 } from "@budibase/types"
 import * as triggers from "../triggers"
-import { db as dbCore, context } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
+import { AUTOMATION_SYNC_TIMEOUT } from "../../constants"
 
 export const definition: AutomationStepSchema = {
   name: "Trigger an automation",
@@ -76,7 +77,7 @@ export async function run({ inputs }: AutomationStepInput) {
         automation,
         {
           fields: { ...fieldParams },
-          timeout: inputs.timeout * 1000 || 120000,
+          timeout: inputs.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT,
         },
         { getResponses: true }
       )
diff --git a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts
index f8cf647e79..83e3c20bbb 100644
--- a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts
+++ b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts
@@ -3,6 +3,7 @@ jest.spyOn(global.console, "error")
 import * as setup from "./utilities"
 import * as automation from "../index"
 import { serverLogAutomation } from "../../tests/utilities/structures"
+import { AUTOMATION_ASYNC_TIMEOUT } from "../../constants"
 
 describe("Test triggering an automation from another automation", () => {
   let config = setup.getConfig()
@@ -22,7 +23,10 @@ describe("Test triggering an automation from another automation", () => {
     let newAutomation = await config.createAutomation(automation)
 
     const inputs: any = {
-      automation: { automationId: newAutomation._id, timeout: 12000 },
+      automation: {
+        automationId: newAutomation._id,
+        timeout: AUTOMATION_ASYNC_TIMEOUT,
+      },
     }
     const res = await setup.runStep(
       setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
@@ -33,7 +37,12 @@ describe("Test triggering an automation from another automation", () => {
   })
 
   it("should fail gracefully if the automation id is incorrect", async () => {
-    const inputs: any = { automation: { automationId: null, timeout: 12000 } }
+    const inputs: any = {
+      automation: {
+        automationId: null,
+        timeout: AUTOMATION_ASYNC_TIMEOUT,
+      },
+    }
     const res = await setup.runStep(
       setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
       inputs
diff --git a/packages/server/src/constants/automations.ts b/packages/server/src/constants/automations.ts
new file mode 100644
index 0000000000..5d83369f87
--- /dev/null
+++ b/packages/server/src/constants/automations.ts
@@ -0,0 +1,4 @@
+import { Duration } from "@budibase/backend-core"
+
+export const AUTOMATION_SYNC_TIMEOUT = Duration.fromMinutes(2).toMs()
+export const AUTOMATION_ASYNC_TIMEOUT = Duration.fromSeconds(12).toMs()
diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index 49f1d01afb..530cc536ca 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -6,6 +6,8 @@ import {
   TableSourceType,
 } from "@budibase/types"
 
+export * from "./automations"
+
 export enum FilterTypes {
   STRING = "string",
   FUZZY = "fuzzy",
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 8e6866d5e4..b3d75070a6 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -53,7 +53,9 @@ const environment = {
     parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
-  QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
+  QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) || 10000,
+  AUTOMATION_THREAD_TIMEOUT:
+    parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) || 12000,
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index a828af5d19..a4938bb138 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -303,7 +303,7 @@ class Orchestrator {
             if (timeout) {
               setTimeout(() => {
                 timeoutFlag = true
-              }, timeout || 12000)
+              }, timeout || env.AUTOMATION_THREAD_TIMEOUT)
             }
 
             stepCount++
@@ -621,7 +621,7 @@ export async function executeInThread(job: Job<AutomationData>) {
   const timeoutPromise = new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error("Timeout exceeded"))
-    }, job.data.event.timeout || 12000)
+    }, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT)
   })
 
   return await context.doInAppContext(appId, async () => {

From 77225e6eb9c76982c2d07b79462dc0985c759db9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 14 Feb 2024 15:04:08 +0000
Subject: [PATCH 04/34] Updating how default environment variables are handled,
 so that the defaults are easier to access.

---
 packages/server/src/constants/automations.ts |  4 +-
 packages/server/src/environment.ts           | 51 +++++++++++++++-----
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/constants/automations.ts b/packages/server/src/constants/automations.ts
index 5d83369f87..6688cbedb7 100644
--- a/packages/server/src/constants/automations.ts
+++ b/packages/server/src/constants/automations.ts
@@ -1,4 +1,6 @@
 import { Duration } from "@budibase/backend-core"
+import env from "../environment"
 
+const defaults = env.getDefaults()
 export const AUTOMATION_SYNC_TIMEOUT = Duration.fromMinutes(2).toMs()
-export const AUTOMATION_ASYNC_TIMEOUT = Duration.fromSeconds(12).toMs()
+export const AUTOMATION_ASYNC_TIMEOUT = defaults.QUERY_THREAD_TIMEOUT
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index b3d75070a6..ec70ced66c 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -18,6 +18,20 @@ function parseIntSafe(number?: string) {
   }
 }
 
+const DEFAULTS = {
+  QUERY_THREAD_TIMEOUT: 10000,
+  AUTOMATION_THREAD_TIMEOUT: 12000,
+  AUTOMATION_MAX_ITERATIONS: 200,
+  JS_PER_EXECUTION_TIME_LIMIT_MS: 1000,
+  TEMPLATE_REPOSITORY: "app",
+  PLUGINS_DIR: "/plugins",
+  FORKED_PROCESS_NAME: "main",
+  JS_RUNNER_MEMORY_LIMIT: 64,
+}
+
+const QUERY_THREAD_TIMEOUT =
+  parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) ||
+  DEFAULTS.QUERY_THREAD_TIMEOUT
 const environment = {
   // features
   APP_FEATURES: process.env.APP_FEATURES,
@@ -42,7 +56,8 @@ const environment = {
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app",
+  TEMPLATE_REPOSITORY:
+    process.env.TEMPLATE_REPOSITORY || DEFAULTS.TEMPLATE_REPOSITORY,
   DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   // minor
@@ -50,16 +65,20 @@ const environment = {
   LOGGER: process.env.LOGGER,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   AUTOMATION_MAX_ITERATIONS:
-    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
+    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) ||
+    DEFAULTS.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
-  QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) || 10000,
+  QUERY_THREAD_TIMEOUT: QUERY_THREAD_TIMEOUT,
   AUTOMATION_THREAD_TIMEOUT:
-    parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) || 12000,
+    parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) ||
+    DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT
+      ? DEFAULTS.AUTOMATION_THREAD_TIMEOUT
+      : QUERY_THREAD_TIMEOUT,
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
-  PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
+  PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
@@ -72,12 +91,21 @@ const environment = {
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
-  FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
+  FORKED_PROCESS_NAME:
+    process.env.FORKED_PROCESS_NAME || DEFAULTS.FORKED_PROCESS_NAME,
   JS_PER_INVOCATION_TIMEOUT_MS:
-    parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
+    parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) ||
+    DEFAULTS.JS_PER_EXECUTION_TIME_LIMIT_MS,
   JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
     process.env.JS_PER_REQUEST_TIME_LIMIT_MS
   ),
+  TOP_LEVEL_PATH:
+    process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
+  APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
+  JS_RUNNER_MEMORY_LIMIT:
+    parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
+    DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
+  LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
   // old
   CLIENT_ID: process.env.CLIENT_ID,
   _set(key: string, value: any) {
@@ -94,12 +122,9 @@ const environment = {
   isInThread: () => {
     return process.env.FORKED_PROCESS
   },
-  TOP_LEVEL_PATH:
-    process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
-  APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
-  JS_RUNNER_MEMORY_LIMIT:
-    parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
-  LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
+  getDefaults: () => {
+    return DEFAULTS
+  },
 }
 
 // clean up any environment variable edge cases

From fa585fe69df421c0f0b90c697f0376267e7d0795 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 14 Feb 2024 15:05:08 +0000
Subject: [PATCH 05/34] Removing timeout reject option.

---
 packages/backend-core/src/utils/utils.ts | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts
index 046f8aaf94..30cf55b149 100644
--- a/packages/backend-core/src/utils/utils.ts
+++ b/packages/backend-core/src/utils/utils.ts
@@ -226,19 +226,8 @@ export function isClient(ctx: Ctx) {
   return ctx.headers[Header.TYPE] === "client"
 }
 
-export function timeout(
-  timeMs: number,
-  opts?: { reject?: boolean }
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    setTimeout(() => {
-      if (opts?.reject) {
-        reject(new Error(`timed out - ${timeMs}ms`))
-      } else {
-        resolve()
-      }
-    }, timeMs)
-  })
+export function timeout(timeMs: number) {
+  return new Promise(resolve => setTimeout(resolve, timeMs))
 }
 
 export function isAudited(event: Event) {

From bcabfd02a6852df8e5df28785db7cce038de3d5e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 14 Feb 2024 15:41:35 +0000
Subject: [PATCH 06/34] Add nested flag to button group settings

---
 packages/client/manifest.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 0fee0e7478..7faf7a02e2 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -270,6 +270,7 @@
           {
             "type": "buttonConfiguration",
             "key": "buttons",
+            "nested": true,
             "defaultValue": [
               {
                 "type": "cta",

From ea5d04e1d3dbaf22337402c7b736684ff44ad5e7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 14 Feb 2024 17:15:42 +0000
Subject: [PATCH 07/34] Addressing PR comments.

---
 packages/server/src/api/controllers/automation.ts          | 7 +++++--
 .../server/src/automations/steps/triggerAutomationRun.ts   | 5 +++--
 .../src/automations/tests/triggerAutomationRun.spec.ts     | 6 +++---
 packages/server/src/constants/automations.ts               | 6 ------
 packages/server/src/constants/index.ts                     | 2 --
 packages/server/src/environment.ts                         | 1 +
 6 files changed, 12 insertions(+), 15 deletions(-)
 delete mode 100644 packages/server/src/constants/automations.ts

diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts
index 212ef5668f..186b68f3b7 100644
--- a/packages/server/src/api/controllers/automation.ts
+++ b/packages/server/src/api/controllers/automation.ts
@@ -10,7 +10,7 @@ import {
   removeDeprecated,
 } from "../../automations/utils"
 import { deleteEntityMetadata } from "../../utilities"
-import { MetadataTypes, AUTOMATION_SYNC_TIMEOUT } from "../../constants"
+import { MetadataTypes } from "../../constants"
 import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events, db as dbCore } from "@budibase/backend-core"
 import { automations, features } from "@budibase/pro"
@@ -24,6 +24,7 @@ import {
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
+import env from "../../environment"
 
 async function getActionDefinitions() {
   return removeDeprecated(await actionDefs())
@@ -274,7 +275,9 @@ export async function trigger(ctx: UserCtx) {
       automation,
       {
         fields: ctx.request.body.fields,
-        timeout: ctx.request.body.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT,
+        timeout:
+          ctx.request.body.timeout * 1000 ||
+          env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
       },
       { getResponses: true }
     )
diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts
index 73c9b3adf1..83e1722877 100644
--- a/packages/server/src/automations/steps/triggerAutomationRun.ts
+++ b/packages/server/src/automations/steps/triggerAutomationRun.ts
@@ -11,7 +11,7 @@ import {
 import * as triggers from "../triggers"
 import { context } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
-import { AUTOMATION_SYNC_TIMEOUT } from "../../constants"
+import env from "../../environment"
 
 export const definition: AutomationStepSchema = {
   name: "Trigger an automation",
@@ -77,7 +77,8 @@ export async function run({ inputs }: AutomationStepInput) {
         automation,
         {
           fields: { ...fieldParams },
-          timeout: inputs.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT,
+          timeout:
+            inputs.timeout * 1000 || env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
         },
         { getResponses: true }
       )
diff --git a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts
index 83e3c20bbb..9d699e15fa 100644
--- a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts
+++ b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts
@@ -3,7 +3,7 @@ jest.spyOn(global.console, "error")
 import * as setup from "./utilities"
 import * as automation from "../index"
 import { serverLogAutomation } from "../../tests/utilities/structures"
-import { AUTOMATION_ASYNC_TIMEOUT } from "../../constants"
+import env from "../../environment"
 
 describe("Test triggering an automation from another automation", () => {
   let config = setup.getConfig()
@@ -25,7 +25,7 @@ describe("Test triggering an automation from another automation", () => {
     const inputs: any = {
       automation: {
         automationId: newAutomation._id,
-        timeout: AUTOMATION_ASYNC_TIMEOUT,
+        timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
       },
     }
     const res = await setup.runStep(
@@ -40,7 +40,7 @@ describe("Test triggering an automation from another automation", () => {
     const inputs: any = {
       automation: {
         automationId: null,
-        timeout: AUTOMATION_ASYNC_TIMEOUT,
+        timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
       },
     }
     const res = await setup.runStep(
diff --git a/packages/server/src/constants/automations.ts b/packages/server/src/constants/automations.ts
deleted file mode 100644
index 6688cbedb7..0000000000
--- a/packages/server/src/constants/automations.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import { Duration } from "@budibase/backend-core"
-import env from "../environment"
-
-const defaults = env.getDefaults()
-export const AUTOMATION_SYNC_TIMEOUT = Duration.fromMinutes(2).toMs()
-export const AUTOMATION_ASYNC_TIMEOUT = defaults.QUERY_THREAD_TIMEOUT
diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index 530cc536ca..49f1d01afb 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -6,8 +6,6 @@ import {
   TableSourceType,
 } from "@budibase/types"
 
-export * from "./automations"
-
 export enum FilterTypes {
   STRING = "string",
   FUZZY = "fuzzy",
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index ec70ced66c..20142776b8 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -21,6 +21,7 @@ function parseIntSafe(number?: string) {
 const DEFAULTS = {
   QUERY_THREAD_TIMEOUT: 10000,
   AUTOMATION_THREAD_TIMEOUT: 12000,
+  AUTOMATION_SYNC_TIMEOUT: 120000,
   AUTOMATION_MAX_ITERATIONS: 200,
   JS_PER_EXECUTION_TIME_LIMIT_MS: 1000,
   TEMPLATE_REPOSITORY: "app",

From 5148cb88c6e665bb8b59c73def66a5d66a612f05 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 15 Feb 2024 09:14:43 +0000
Subject: [PATCH 08/34] Bump account portal

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 1ba8414bed..8c446c4ba3 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 1ba8414bed14439512153cf851086146a80560f5
+Subproject commit 8c446c4ba385592127fa31755d3b64467b291882

From ad8d1e25988c46f2c0f0d2e061b72892e3d24962 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 15 Feb 2024 09:44:01 +0000
Subject: [PATCH 09/34] Make the MySQL healthcheck stricter.

---
 packages/server/src/integrations/tests/utils/mysql.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts
index 2c44dd1373..474819287e 100644
--- a/packages/server/src/integrations/tests/utils/mysql.ts
+++ b/packages/server/src/integrations/tests/utils/mysql.ts
@@ -14,8 +14,9 @@ export async function start(): Promise<StartedTestContainer> {
         // run against a MySQL that's mid-restart and fail. To avoid this, we run
         // the ping command three times with a small delay between each.
         `
-          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 0.5 &&
-          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 0.5 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
           mysqladmin ping -h localhost -P 3306 -u root -ppassword
         `
       )

From b12aa639d3098c2a79c04871d50e23d3e6503e6d Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Thu, 15 Feb 2024 10:53:58 +0000
Subject: [PATCH 10/34] Allow Collapsing Selected Components, Add Hotkeys for
 Collapsing Components (#12764)

* wip

* fix spelling

* wip

* linting

* change order of fix version of linting

* lint fix

* linting
---
 package.json                                  |  2 +-
 .../src/builderStore/store/frontend.js        | 19 ++++++--
 .../ComponentDropdownMenu.svelte              | 34 ++++++++++++++
 .../ComponentList/ComponentKeyHandler.svelte  | 35 ++++++++++++++
 .../ComponentList/ComponentTree.svelte        | 32 ++++---------
 .../stores/portal/componentTreeNodesStore.js  | 36 +++++++++++++++
 packages/frontend-core/src/stores/index.js    |  1 +
 .../src/stores/sessionStorage.js              | 46 +++++++++++++++++++
 8 files changed, 178 insertions(+), 27 deletions(-)
 create mode 100644 packages/builder/src/stores/portal/componentTreeNodesStore.js
 create mode 100644 packages/frontend-core/src/stores/sessionStorage.js

diff --git a/package.json b/package.json
index 499952a441..4407fd33f3 100644
--- a/package.json
+++ b/package.json
@@ -58,7 +58,7 @@
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
-    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
+    "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
     "build:specs": "lerna run --stream specs",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js
index 55208bb97e..fd492cca0b 100644
--- a/packages/builder/src/builderStore/store/frontend.js
+++ b/packages/builder/src/builderStore/store/frontend.js
@@ -39,6 +39,7 @@ import { makePropSafe as safe } from "@budibase/string-templates"
 import { getComponentFieldOptions } from "helpers/formFields"
 import { createBuilderWebsocket } from "builderStore/websocket"
 import { BuilderSocketEvent } from "@budibase/shared-core"
+import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
 const INITIAL_FRONTEND_STATE = {
   initialised: false,
@@ -1053,6 +1054,7 @@ export const getFrontendStore = () => {
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
         const index = parent?._children.findIndex(x => x._id === componentId)
+        const componentTreeNodes = get(componentTreeNodesStore)
 
         // Check for screen and navigation component edge cases
         const screenComponentId = `${screen._id}-screen`
@@ -1071,9 +1073,15 @@ export const getFrontendStore = () => {
         if (index > 0) {
           // If sibling before us accepts children, select a descendant
           const previousSibling = parent._children[index - 1]
-          if (previousSibling._children?.length) {
+          if (
+            previousSibling._children?.length &&
+            componentTreeNodes[`nodeOpen-${previousSibling._id}`]
+          ) {
             let target = previousSibling
-            while (target._children?.length) {
+            while (
+              target._children?.length &&
+              componentTreeNodes[`nodeOpen-${target._id}`]
+            ) {
               target = target._children[target._children.length - 1]
             }
             return target._id
@@ -1093,6 +1101,7 @@ export const getFrontendStore = () => {
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
         const index = parent?._children.findIndex(x => x._id === componentId)
+        const componentTreeNodes = get(componentTreeNodesStore)
 
         // Check for screen and navigation component edge cases
         const screenComponentId = `${screen._id}-screen`
@@ -1102,7 +1111,11 @@ export const getFrontendStore = () => {
         }
 
         // If we have children, select first child
-        if (component._children?.length) {
+        if (
+          component._children?.length &&
+          (state.selectedComponentId === navComponentId ||
+            componentTreeNodes[`nodeOpen-${component._id}`])
+        ) {
           return component._children[0]._id
         } else if (!parent) {
           return null
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte
index 4645ee0d41..baaa561679 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte
@@ -3,6 +3,7 @@
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
 
   export let component
+  export let opened
 
   $: definition = componentStore.getDefinition(component?._component)
   $: noPaste = !$componentStore.componentToPaste
@@ -85,6 +86,39 @@
   >
     Paste
   </MenuItem>
+
+  {#if component?._children?.length}
+    <MenuItem
+      icon="TreeExpand"
+      keyBind="!ArrowRight"
+      on:click={() => keyboardEvent("ArrowRight", false)}
+      disabled={opened}
+    >
+      Expand
+    </MenuItem>
+    <MenuItem
+      icon="TreeCollapse"
+      keyBind="!ArrowLeft"
+      on:click={() => keyboardEvent("ArrowLeft", false)}
+      disabled={!opened}
+    >
+      Collapse
+    </MenuItem>
+    <MenuItem
+      icon="TreeExpandAll"
+      keyBind="Ctrl+!ArrowRight"
+      on:click={() => keyboardEvent("ArrowRight", true)}
+    >
+      Expand All
+    </MenuItem>
+    <MenuItem
+      icon="TreeCollapseAll"
+      keyBind="Ctrl+!ArrowLeft"
+      on:click={() => keyboardEvent("ArrowLeft", true)}
+    >
+      Collapse All
+    </MenuItem>
+  {/if}
 </ActionMenu>
 
 <style>
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte
index 406650f715..f6bbac39a5 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte
@@ -9,6 +9,7 @@
   import { goto, isActive } from "@roxi/routify"
   import { notifications } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
   let confirmDeleteDialog
   let confirmEjectDialog
@@ -61,6 +62,40 @@
     ["ArrowDown"]: () => {
       componentStore.selectNext()
     },
+    ["ArrowRight"]: component => {
+      componentTreeNodesStore.expandNode(component._id)
+    },
+    ["ArrowLeft"]: component => {
+      componentTreeNodesStore.collapseNode(component._id)
+    },
+    ["Ctrl+ArrowRight"]: component => {
+      componentTreeNodesStore.expandNode(component._id)
+
+      const expandChildren = component => {
+        const children = component._children ?? []
+
+        children.forEach(child => {
+          componentTreeNodesStore.expandNode(child._id)
+          expandChildren(child)
+        })
+      }
+
+      expandChildren(component)
+    },
+    ["Ctrl+ArrowLeft"]: component => {
+      componentTreeNodesStore.collapseNode(component._id)
+
+      const collapseChildren = component => {
+        const children = component._children ?? []
+
+        children.forEach(child => {
+          componentTreeNodesStore.collapseNode(child._id)
+          collapseChildren(child)
+        })
+      }
+
+      collapseChildren(component)
+    },
     ["Escape"]: () => {
       if ($isActive(`./:componentId/new`)) {
         $goto(`./${$componentStore.selectedComponentId}`)
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte
index bf1b7f21d7..f94e86382b 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte
@@ -17,11 +17,12 @@
   } from "helpers/components"
   import { get } from "svelte/store"
   import { dndStore } from "./dndStore"
+  import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
   export let components = []
   export let level = 0
 
-  let closedNodes = {}
+  $: openNodes = $componentTreeNodesStore
 
   $: filteredComponents = components?.filter(component => {
     return (
@@ -54,15 +55,6 @@
     return componentSupportsChildren(component) && component._children?.length
   }
 
-  function toggleNodeOpen(componentId) {
-    if (closedNodes[componentId]) {
-      delete closedNodes[componentId]
-    } else {
-      closedNodes[componentId] = true
-    }
-    closedNodes = closedNodes
-  }
-
   const onDrop = async e => {
     e.stopPropagation()
     try {
@@ -72,14 +64,14 @@
     }
   }
 
-  const isOpen = (component, selectedComponentPath, closedNodes) => {
+  const isOpen = (component, selectedComponentPath, openNodes) => {
     if (!component?._children?.length) {
       return false
     }
-    if (selectedComponentPath.includes(component._id)) {
+    if (selectedComponentPath.slice(0, -1).includes(component._id)) {
       return true
     }
-    return !closedNodes[component._id]
+    return openNodes[`nodeOpen-${component._id}`]
   }
 
   const isChildOfSelectedComponent = component => {
@@ -96,7 +88,7 @@
 
 <ul>
   {#each filteredComponents || [] as component, index (component._id)}
-    {@const opened = isOpen(component, $selectedComponentPath, closedNodes)}
+    {@const opened = isOpen(component, $selectedComponentPath, openNodes)}
     <li
       on:click|stopPropagation={() => {
         componentStore.select(component._id)
@@ -110,7 +102,7 @@
         on:dragend={dndStore.actions.reset}
         on:dragstart={() => dndStore.actions.dragstart(component)}
         on:dragover={dragover(component, index)}
-        on:iconClick={() => toggleNodeOpen(component._id)}
+        on:iconClick={() => componentTreeNodesStore.toggleNode(component._id)}
         on:drop={onDrop}
         hovering={$hoverStore.componentId === component._id}
         on:mouseenter={() => hover(component._id)}
@@ -125,8 +117,9 @@
         highlighted={isChildOfSelectedComponent(component)}
         selectedBy={$userSelectedResourceMap[component._id]}
       >
-        <ComponentDropdownMenu {component} />
+        <ComponentDropdownMenu {opened} {component} />
       </NavItem>
+
       {#if opened}
         <svelte:self
           components={component._children}
@@ -144,13 +137,6 @@
     padding-left: 0;
     margin: 0;
   }
-  ul :global(.icon.arrow) {
-    transition: opacity 130ms ease-out;
-    opacity: 0;
-  }
-  ul:hover :global(.icon.arrow) {
-    opacity: 1;
-  }
   ul,
   li {
     min-width: max-content;
diff --git a/packages/builder/src/stores/portal/componentTreeNodesStore.js b/packages/builder/src/stores/portal/componentTreeNodesStore.js
new file mode 100644
index 0000000000..a6c07ae03b
--- /dev/null
+++ b/packages/builder/src/stores/portal/componentTreeNodesStore.js
@@ -0,0 +1,36 @@
+import { createSessionStorageStore } from "@budibase/frontend-core"
+
+const baseStore = createSessionStorageStore("openNodes", {})
+
+const toggleNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = !openNodes[`nodeOpen-${componentId}`]
+
+    return openNodes
+  })
+}
+
+const expandNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = true
+
+    return openNodes
+  })
+}
+
+const collapseNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = false
+
+    return openNodes
+  })
+}
+
+const store = {
+  subscribe: baseStore.subscribe,
+  toggleNode,
+  expandNode,
+  collapseNode,
+}
+
+export default store
diff --git a/packages/frontend-core/src/stores/index.js b/packages/frontend-core/src/stores/index.js
index 9766da43e3..6e801e5f63 100644
--- a/packages/frontend-core/src/stores/index.js
+++ b/packages/frontend-core/src/stores/index.js
@@ -1 +1,2 @@
 export { createLocalStorageStore } from "./localStorage"
+export { createSessionStorageStore } from "./sessionStorage"
diff --git a/packages/frontend-core/src/stores/sessionStorage.js b/packages/frontend-core/src/stores/sessionStorage.js
new file mode 100644
index 0000000000..3d632b6919
--- /dev/null
+++ b/packages/frontend-core/src/stores/sessionStorage.js
@@ -0,0 +1,46 @@
+import { get, writable } from "svelte/store"
+
+export const createSessionStorageStore = (sessionStorageKey, initialValue) => {
+  const store = writable(initialValue, () => {
+    // Hydrate from session storage when we get a new subscriber
+    hydrate()
+
+    // Listen for session storage changes and keep store in sync
+    const storageListener = ({ key }) => {
+      return key === sessionStorageKey && hydrate()
+    }
+
+    window.addEventListener("storage", storageListener)
+    return () => window.removeEventListener("storage", storageListener)
+  })
+
+  // New store setter which updates the store and sessionstorage
+  const set = value => {
+    store.set(value)
+    sessionStorage.setItem(sessionStorageKey, JSON.stringify(value))
+  }
+
+  // New store updater which updates the store and sessionstorage
+  const update = updaterFn => set(updaterFn(get(store)))
+
+  // Hydrates the store from sessionstorage
+  const hydrate = () => {
+    const sessionValue = sessionStorage.getItem(sessionStorageKey)
+    if (sessionValue == null) {
+      set(initialValue)
+    } else {
+      try {
+        store.set(JSON.parse(sessionValue))
+      } catch {
+        set(initialValue)
+      }
+    }
+  }
+
+  // Patch the default svelte store functions with our overrides
+  return {
+    ...store,
+    set,
+    update,
+  }
+}

From 2fbbd295fc753fee95b23950ebf8d9c7c8838b66 Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Thu, 15 Feb 2024 11:16:00 +0000
Subject: [PATCH 11/34] fix arrow navigation behavior (#13040)

---
 .../builder/src/stores/builder/components.js  | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js
index b742331343..d7572dca60 100644
--- a/packages/builder/src/stores/builder/components.js
+++ b/packages/builder/src/stores/builder/components.js
@@ -29,6 +29,7 @@ import {
 } from "constants/backend"
 import BudiStore from "./BudiStore"
 import { Utils } from "@budibase/frontend-core"
+import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -662,6 +663,7 @@ export class ComponentStore extends BudiStore {
     const screen = get(selectedScreen)
     const parent = findComponentParent(screen.props, componentId)
     const index = parent?._children.findIndex(x => x._id === componentId)
+    const componentTreeNodes = get(componentTreeNodesStore)
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -680,9 +682,15 @@ export class ComponentStore extends BudiStore {
     if (index > 0) {
       // If sibling before us accepts children, select a descendant
       const previousSibling = parent._children[index - 1]
-      if (previousSibling._children?.length) {
+      if (
+        previousSibling._children?.length &&
+        componentTreeNodes[`nodeOpen-${previousSibling._id}`]
+      ) {
         let target = previousSibling
-        while (target._children?.length) {
+        while (
+          target._children?.length &&
+          componentTreeNodes[`nodeOpen-${target._id}`]
+        ) {
           target = target._children[target._children.length - 1]
         }
         return target._id
@@ -703,6 +711,7 @@ export class ComponentStore extends BudiStore {
     const screen = get(selectedScreen)
     const parent = findComponentParent(screen.props, componentId)
     const index = parent?._children.findIndex(x => x._id === componentId)
+    const componentTreeNodes = get(componentTreeNodesStore)
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -712,7 +721,11 @@ export class ComponentStore extends BudiStore {
     }
 
     // If we have children, select first child
-    if (component._children?.length) {
+    if (
+      component._children?.length &&
+      (state.selectedComponentId === navComponentId ||
+        componentTreeNodes[`nodeOpen-${component._id}`])
+    ) {
       return component._children[0]._id
     } else if (!parent) {
       return null

From dea72b5818e19c22bee754d98d7af63670484252 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 15 Feb 2024 11:27:41 +0000
Subject: [PATCH 12/34] Bump version to 2.19.4

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 0e026aefc5..4940336b29 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.19.3",
+  "version": "2.19.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From e9bb17dd8acb5675c0df62e1dcf8dfb87cafbab9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 12:47:27 +0000
Subject: [PATCH 13/34] Fixing cluster mode in docker-compose setups, we
 weren't copying across the pm2 config that was needed to achieve this.

---
 packages/server/Dockerfile | 1 +
 packages/worker/Dockerfile | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile
index f63481c373..3aae395bc5 100644
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@@ -66,6 +66,7 @@ COPY packages/server/dist/ dist/
 COPY packages/server/docker_run.sh .
 COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
+COPY packages/server/pm2.config.js .
 
 ARG BUDIBASE_VERSION
 ARG GIT_COMMIT_SHA
diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile
index 52fd3677b1..77869c48f1 100644
--- a/packages/worker/Dockerfile
+++ b/packages/worker/Dockerfile
@@ -37,6 +37,7 @@ RUN apk del .gyp \
 
 COPY packages/worker/dist/ dist/
 COPY packages/worker/docker_run.sh .
+COPY packages/server/pm2.config.js .
 
 EXPOSE 4001
 

From d3bb8b2fa462382b556f4a613bf8955cef4cd07b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 12:49:39 +0000
Subject: [PATCH 14/34] There was an issue where extra headers were being
 carried over to the worker instance when performing cross-service comms -
 these headers were causing the request to be rejected without consideration.
 Cleaning up to only include the headers Budibase really cares about in
 request, let fetch work out the generic ones.

---
 .../server/src/utilities/workerRequests.ts    | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 69ce58c8a9..7528ff10c7 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -14,12 +14,23 @@ export function request(ctx?: Ctx, request?: any) {
   if (!request.headers) {
     request.headers = {}
   }
+
   if (!ctx) {
     request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
-    if (tenancy.isTenantIdSet()) {
-      request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  } else {
+    // copy all Budibase utilised headers over - copying everything can have
+    // side effects like requests being rejected due to odd content types etc
+    for (let header of Object.values(constants.Header)) {
+      if (ctx.headers[header]) {
+        request.headers[header] = ctx.headers[header]
+      }
     }
   }
+
+  // apply tenancy if its available
+  if (tenancy.isTenantIdSet()) {
+    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"
     request.body =
@@ -29,9 +40,6 @@ export function request(ctx?: Ctx, request?: any) {
   } else {
     delete request.body
   }
-  if (ctx && ctx.headers) {
-    request.headers = ctx.headers
-  }
 
   // add x-budibase-correlation-id header
   logging.correlation.setHeader(request.headers)
@@ -54,7 +62,7 @@ async function checkResponse(
     }
     const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
     if (ctx) {
-      ctx.throw(msg, response.status)
+      ctx.throw(response.status || 500, msg)
     } else {
       throw msg
     }

From 05d19c92c747f946b1b8a1fae2ab28042287dd46 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 12:59:41 +0000
Subject: [PATCH 15/34] Fixing bug found by test.

---
 packages/server/src/utilities/workerRequests.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 7528ff10c7..5612084216 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -17,7 +17,7 @@ export function request(ctx?: Ctx, request?: any) {
 
   if (!ctx) {
     request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
-  } else {
+  } else if (ctx.headers) {
     // copy all Budibase utilised headers over - copying everything can have
     // side effects like requests being rejected due to odd content types etc
     for (let header of Object.values(constants.Header)) {

From 649dafba47c8cc248b76bd2049ceb5c426c58e37 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Thu, 15 Feb 2024 13:05:03 +0000
Subject: [PATCH 16/34] n8n automation action integration  (#12992)

* Add n8n automation action

* Add authorization header support

* add unit tests

* Replace test.com with example.com

* Add HttpMethod enum to types

* fix unit test

* Add required field label asterisk
---
 packages/builder/assets/n8n_square.png        | Bin 0 -> 12155 bytes
 .../FlowChart/ActionModal.svelte              |   6 +-
 .../FlowChart/ExternalActions.js              |   2 +
 .../SetupPanel/AutomationBlockSetup.svelte    |   8 +-
 .../src/constants/backend/automations.js      |   1 +
 packages/server/src/automations/actions.ts    |   3 +
 packages/server/src/automations/steps/make.ts |  22 +--
 packages/server/src/automations/steps/n8n.ts  | 125 ++++++++++++++++++
 .../server/src/automations/steps/zapier.ts    |  20 ---
 .../server/src/automations/tests/n8n.spec.ts  |  68 ++++++++++
 packages/server/src/integrations/rest.ts      |  23 ++--
 .../types/src/documents/app/automation.ts     |   1 +
 packages/types/src/documents/app/query.ts     |   9 ++
 13 files changed, 232 insertions(+), 56 deletions(-)
 create mode 100644 packages/builder/assets/n8n_square.png
 create mode 100644 packages/server/src/automations/steps/n8n.ts
 create mode 100644 packages/server/src/automations/tests/n8n.spec.ts

diff --git a/packages/builder/assets/n8n_square.png b/packages/builder/assets/n8n_square.png
new file mode 100644
index 0000000000000000000000000000000000000000..23b75ee68842219fd2e11989be8067ac4aeee507
GIT binary patch
literal 12155
zcmeHtXH-+$)^;e;dzTslA}tU|2qg60J0isbgwR6^9qAw)1f}<m3P?wa6zLu5AV^UV
z3sM9@6!@a&+<VV?zx(Ho@xK4=9@)v>YtH%1IiETA8d+oQSOYzEN-|b5002O#si9(o
z`#t`3k`Um&yWEs-0sw4n0VZZ%M%KPScTYDvM;A2E%ikRhMEg0~0RVo>yE*2m><Ay#
z%Q`YJ9yK&Q;+2l{)S<TsNW=R^!>r9ow2qw(6Eq@)^UJ>E%0|y<-sS1Zr;9}SvW!im
z8eLJD5F>qAiaW5&@6b2CUq1wegt%T@OkA9H{`el>ZJAPV@O9F(bMVq{EjWMuD{5cx
zay)kV?84^afLS(Za_z#acrU8hl)<mA>-xtv`u_dzQ9=3;xt1+$w_`Uq>-@>dz4!9g
z>(6Fd%+^{%^50Ef>ZR@Mj^&6&(t`s!2TyM+%&xuL^b<XL;ZfM%_ladRup;=Tl0q<J
zML|egcy(F%UCyKEs-O4JUBPST2up!`)UhFCp5MQJ&DfJ3?V1=NKVQ3rXxe+aKX{_w
zaAE$cp!iwW$rqnBSu8UTqtfk8$>X=CZ-kO^ZYgEBCY3SX{hotR4Z556Oznnfee?O@
z!)NGK{10vEF0X5y-=E)VHnSlcj2eljF^$<9XL8=L?$^IKt!H!k>@le3x}Y5xxAETM
z^X=vzTW{uAZ@w8Q>XFJiNLjn^?D`QFI7epJd7i-diA=G_?&?TxCzj=E-49BqV7!Zi
z<M;9-&b$%khNkge_|ozfnM)Vsq?_7a@4Ck(_1n!>*XU_LB6;Qss^SXs!gfO2&LndQ
z3PvoYy^^f_I^tG{=y8y8F?NVDudnR8Mw73@P*@Ac9)X=LXy$I(%8`_YsaDq6<Hv@|
zJcGHO6W112JSR)~Qnf_NW<dt*67%nzYRcx9ot~BSr(3v|&COf5l|*ymjjw1wdvW|`
za6LN$N-ZZVWtDK%F4xgca<r;pTyC)Vo~ea@+Kro*L5UnbE9-+I?O&RIa1K(Hc})*j
z<OVE%u)n3XtL8g1-m((#de`^o*s^4G$jLFAd4t>h%h5+vRq_o3?M)X_QKWkHo@y_{
zwbcfm8Yw6eZ;eT63-Dt{vi8DihQd_swp?<=)re;59y~iA$|EmG5X4>@y}6|uGGuvb
z4}JfrZEb92?fvde^Vh{i$Ku~UC|#LTZ%bY)fGe?q+Pi*yxcJE2_2cWM{!v#zaqxK@
z_cJeyD)}Sp&^e0|GJo&VPGKEC!|2_#0YXYlo2uyC3&)rT{Jcmfw~_3I#}isZ=i9Rj
zU*6_wOKd^zAFqDO;xBF=Wq*HWaXY@2!_8pn-uJ5`3Z<)W-oG0}7}O*}CvN$EhCAm8
z<g@7$j|SdqvQKyY+UDt&AUo;j2w%jO_iY5Kn69$QE(Ypg4nW@?{={ec)(H2B9UR!s
zSMGTD<!;>?OnbWie9<v-xajfZ3;UI7MmEYyMevrT@@M#a4Ej^5SCf?gu6Vld_3<>t
zl-Zp&Ips|7-pJg<)j(bc3k}$b@KV6@`P0fK0?rQYXz>%8I;n`q%tDm&XFI0v?_5uQ
z4JGI5;2=F?KX^1@A$ha<_!QwRyp?1aaMG=>Ggp1(`_~_fOrAXxD}9W&mOMgj5lxg8
zQLB-Vg+2B_{&exn6w5CsFZi;)48~vf=tJBiwcW96-Kd@0i}`~}FM-MWN#hkp74jut
z5WCy26c-#?ePCmiKZ%EG1pGd&JVz%9GkhU=YM&Dg`XsVSqrK&l|Jje)Xh4pih>8tr
z$bK%uzpZ8M`(ihCi|&QDppEO{s@>IDT62y_w*7Yh7X+l^F~>E5ckI4&&dRu`)?S<)
zC4C))a>vKHmwHWRGkn=ASUk7cD4>6O=el6|LXe<@MReEf)VCK?V(<HOSSp{GSZ_me
zcemz*N(KoQM3{WOF8R;vSBHqiJP)~PD_bDsDGSOsn_FZIV$F-X0k5@woZ+4xjl%C!
z;zLb=B8f2zS1j#BO;=wm8C9$037$Z>OZ-<JQE*&=k(!M4eOKjwP(1lUa%ylw<DOj&
zR=+X|kpAh00S(YS)kvM{q1?^4QG0GocJGw$IU^zH1=dehCSD-2$PBfb4>kR=#aZCo
z+ooZmtqn^bJs5&&Ayx+(lCPs|dfz-{QmEE>umR8X{}GqO-(V1a?jy)Tzfcpdm%+R|
zwL9xDDd(AGn-h0`Rhva3H?^10{w*hCRakk&b??CuD*o8Ii6!Irv78u1#Q+hmN!M>X
zL17x?^OBJSV?hQrfdEjYGeqKmxhdr<qBV@3-Czp#$*zsdD!C<Rbsj|P6r`#$EU;UJ
z<hjU6!y9dXuLB*k<Pl1Dt0IU<ibI$vvWtN3>QPmfRRv1>MlMp-lGXzwSozIUoS$?i
zdbOt$JjiU3&vDXp#0+mYtPYgJ;`+!RTGPjE8a3U|$G>lsU72SDcgSH{Pntr0&Buqx
zL=F=_2k55G$fF}JU+JeZ3nGSp_TBJU6W?SIozns9sgLl^lhwyMDPfwtO_c*v=(_Ke
zxzA`GlT4&Cu!>jtc9@j&R@}qGxI=a0DcU<1wtM9kn@ztET8FJ6yRla_mdCy*7|cIM
z&%Vq}P7ld6G{e4i)Yoe4;U|va38$I*uz!P+e`-@z-+k|vRge@fAECJB&JSLCzRMQR
zD~rI7p)%!?gg(Bs>xt&F-U}3O#w;Uc{dt^6LnGln4b7DI3aRaOcnq`XXw`S#Jzn4*
zh>m~n5MaOdU4vc+!{t}<bFI$(<SKRh$0aU`oDyW5r$GzPoHnHe8hm{P8tcZ0JtYn_
z!>5=f&N$pQ6ZDel6MwFHvJo7_^X=`=0piMrNdeYVb^6b$=T*MTTq%Htklcv)&IW#;
z4tjOrhZm(?(p+0!kD_%0STs`UnTHHxMt*WI;BB*nOuf8y8p|O?ss&P57L<rBjQR-V
zeyAYf7`W4Hev5<EI1Y%MFj??e5l&kw3%TP!j`k2L58o#sX8LY`2o?_4w<yB5@%_LL
zd~zQd#?>YO+vj&ypcyS34ixfM{BiYzK|Xuw;sIgs3ak;tj=u<H@KvD(6ZCPfRH<Kg
zK;*7fPWdOyWaNJ9`;bXelBM*@x>ZZ1`=UA9iE5pE8t(CA#gWw)psK(hEt7Vml)3|u
z8?JYvM8dCFt)=ov3sCr-fiPo+r$XWIpwSGmAnVR2ui_ygFs`9gla|0Vc9=A)7){(v
z>pc-Ct9*5WZ5Vm)8i!8ITaO-MhA6`a<|95q2Sp0d3i{#39@53K;uz{KS)dZWyE)aj
zr!(7FN&dy6uH^xD0^+<{Jk%$kQ%YUS8U5V--U3oh5CYhpmEuK=*ev2yK`3;u9>3M%
zD2rR1(A#%WdS!*-8G4a{`N`YSt=VG_s1c!~+AI|OsOCgk+Pd+`($n}w^n<7qD{}4y
zu@uQO7S$Jf029Mz#Z>oVI|tf6ldbuIu@wq|frr_?q=eD(1Ugpx=h~P?yulHY-fdFi
zq(d32;*2l-Peh;CR1nY>Xgt30_-;iOMKqx<mi)QuDB%$a(RxgrVIQO6t;Z@fA$5TI
z=d+#KX9vLQ6V-yuk{zKu*!6584a!hTJf_k+{DEOd7%pABsgEoyKsKdfm66h0Md9Zv
zAHy99(h0A$z8&7=X=l_ud?(sJcK+g}^R@CaR>!8w%^}%1Y;?W0h6ziW?)|B4)2hOm
z{ewU&j^a4H#=%;83NE^FI4tFbmztLQx(<xRzh~i7Jwqv<S44XRpwhSmFT9+07Oql|
zZi+ABUm8piC9VodI}M!xB~i@ZKUC0OAg45(Tt8`GuE{hgWe}F5{v4A(%_r{_QVJ$x
zA2067${cxvsatg1OpdgwxM~#gS^5mMCZFQ+hFY>H-^u!6Vw*ShjhN4M0WExy#cw%W
zU(45Cd!b=cqxnilocAF4JHQN9+{qxM^65LVc~T5t6&`O)eyPUow?6?I<u1U8dp4VP
zG?-cr2mAvy0HD1>F?P?x4M>N-C9eJA<bHiw4!dn}KO)2Wo=3iV{uO8S>BB9PsN!fM
zq#$qN!GRW`a&U$#Rk%QHp$l<3ZLZqWXXbUz-Cl9z5hSRnCm)eJ5ZK!C>ClZ~#*qi`
zk6!YK2uVZq9J>)-Tol~~JC`DO{HfM4e+HJVI<Y*dQ!(ReafNfmj&>_;BauvFqo!`s
z+)HaOm~4lI(vQad{d>nA=RH4}O|={gIb*<xWOq8^C_Zqs%6y_v#}JVYJ*fI1E3lPd
zb1sn47EiOtc^Xz|@6O&@e0NgYn_E)y0scE57Hl^tWm~Jo_B0~(fD7~uCb%ViPcJ!0
zVA;>yGFWSA#>RA5H5HJXPuE*h@)Uj^zNUOs;^BA+A-1^6%uN9%?HE_EZYHQ$iAY+a
zH{g5ni1(VFrCliI>vez(ZC0doUfcQbIKL^(u99VWS%i04;F@dgQR!ti@63ju6e+<w
zgV8HNh8}lF5{i4CI<%lxC?JKbY<cPC!9;ZW!Y151%ZBE&*D@2VI0bGC{@CTv;6i;%
zQ8pKe>Nk9zc{^+5N_ljgtHQy|-YLCM9#HKu7gGzYz-xi-sinci%OAJ#X?TtZt7y&e
z`xz^yZ#BA$hT1yb4&=zzQn;*}8A2cl&-**3QW)b&FH+a*xQ2~JX{3ipj;OP|bJ@2%
zNC}_c(VWoAhBuK#N>_b@8&_P(j@sXfeS)-L48qG&l5HF>x1krm?m$H*2P~BLxgQtP
z@tTHqMFyhKqG#G(h>669)%MF)(x_a?H_W$p*e|h87!xPDF-ipZPV`y&0rURBku<o~
zTDbMi>`Re#?v<4kg2J%qLFz5Sdxfk;+Wa4EZaCUJVq^bY=`0Qmn=LEq)=kxLL@^Oa
z%?w??;$Kf|*Zn9-oL{UPbBilfF-=E!z>5|ygdE9z;^T3;6%$|>B;4VfUZ_?$kuk(M
z$h7ANYM5vhONFqKe0uMhk<i3<Czs_1@Fc!n&BMe1CDUkOCZ@k@D4F@3%CiBvJH*u6
zTZBpI&FwBG`-&7+w`G2AdX1s-?Fy<asr5*9SDA>0`O7-9>(~os>#3{RCXH91o}&dE
znqMBwmYkb{`VKTl1^S~NmWFHyO3}X}UB2%7_U9(&T%-d^H)<Nc@;kq#h{KYeVnluJ
zHtOu$KVmd-QZD<wVhac5Qt0+;rtzEZMbn2DVpn6LA<t9W_Q@+_tcdd148kb5Ak!cQ
z8Ug`@I&dqu7DtOV{>%GP14|h-Lqnw~kaMhK){sdwg?J*Nxr3bAHu+0p-Im**QnI50
z6006m1l!g-YG%V%co8FH5ASD8QL%zHuaE?68^PK-%*fZQn6)_FB3}9XZ<taiKGM8x
zHNI%c`f$qTxLY%zZ#gqkfWVlPFB2&07s;)D=h`kr1rycHDn-y{6v#r(|1z(VL+B;@
zWnlYfP+>$}ivFwoqjY;u25G=M$DH@=H)U$sl<3*Zc_Elu6#!>^wqQ&#Er1~IS$6#h
zaX9~wn>v2WX&uD}`rY`>vpyb!g70{AraoHd#!3+*8DB0~IG2N+Vk$!rx?LjIUgo6|
zCElw{;alL`$McFlg2fvJFm*RBLkL|{Sovd@$t77Aw-n^odE;9g5W9kgSGmbY_?-5M
zY9(0TY&@Ng`7|6uOI1Rl(420S6aX3Aau~Y*QHrw_-|iMS8+()>jlp#MPtGD4g;)^}
zm2dl8VCQ{!q3(2Q-u%Ir%($wxAIDukpul9Y58qcLtM8Tcp5K?~rAa&Vx<>WDNz{=u
zvp_+D5HFjdRBTqZ_^C2HC49`p`F(yJ9)Sp`j~~xm*fzH?YuX@|P%x4p-i)f+6T%Mk
z^k1=4$b6{R>l|K5oSeKX1>tWdL&f!tWM>^oNC(~RKE5_;L)S)^XC|FT>!e^mW5gFX
z5L+a-ZpJ#~miQRP3UzIXuS$GVf!tsreH*5SoMwEG`9g8YC%XmjMmE57Z1>TE0<HB~
zW0@4mO~%Ci*QD@|vG}>q)ig_M{G?8Y^arD?_R81U3g)Rs=lM&xdAN6ktO50#)?)5N
z4eCcDem}0gCDSgfDPHYeV%Jf;bSouJyH<!_gU4qDE&e&FCT)b&oY=LZ@HEet(jB%@
zk*~D8@<Dd$ysDr|4x1}8@Ga3Hs1)-o$m!-8))y0<%3y`Z*cnBb$(|y8*o<Zt6cZvO
z&FpQw`D{GR|9vr0t>?RF20-<@ywp3|4O@CEzWI;Xr?eb-K9hH-wIL%{YvrL#>nbH<
z8q(a9-6BIkV4RwYZBZR@;bc?g3ZdrRHQ3=G3K@I*qtrSx<D^;iAQ#^k&Nefnhswqz
zUBD4-K%KHlo@ikv1&fx6zsRfDPOAQC;gIGRg=!jiQEQ3UYuK0!ZWL~|A4aPU(K#D%
zS4Nx^K6g0WGQCgsWc$N66L3Ea<|UI&P8OX0+)A?$Pzv!{SCQ-d03BA|=h&ytSk)0J
z?|CU<o<ErEg{5yT&}Cw6^RO9>;-8c%gw8uosyraM+;Hy5mLAF~=T25rG-6@36qsfJ
zGo95Ch?^YQ6C_FKsM;0(G#nJ3<~?fai8|0u+v;OY91$O%jo#i|ZXG-+By#EWc;r*X
zm&)YGO-VYUbu+$%3_>9xl^q4eXPD9dsu@IPIXtkRAw8g+QnrwFiY_GD&6vwK)8w@$
zI*i6ci1<@K?>&L}sI$|G-_ubaPZN+9Q<Qn~(><<we<L-_Zd_}MK^N6}U!P=_mUi&)
zNlfh}i{8j*Or7EwNfCUKo=bmBKS9&xdp(=RK?PW?NIZNXS^z7nrA*oR*gZzbxFdHH
znk;qPwd5Vh=K2l^T)fG$GPJmC#c(5{lGN)hpWY+>P_<Akvz+B8N^wtENV<%S{T|OM
z_Dk<d1EYbSAE^_wts)aj1fK6My`Gv63u>GNz24LY>Bheu(9oCya4|03R$0aCb)&k{
zCED~*Q`c6m*n^EuebQ{_WY@NotZAlasd8S3WW)9)gBSe5W&;|t3phkW#fsC&%9uB^
zufj-U4pcD_*MxU$^mrL*`%}7o3J1$)IzQn%cns)|f%x&4a|jcN<yaV9+3FI@r*ay}
zNjX_l!CbF)cM{artQzh?WzWLYx^}Wyfd%n2c#IwJ0OD~O+_dgqoQ>3zH74{**e~}1
zA-yjZuTD}QQz1E&sd^m0MsDv=W>oux3OxSCyM*PD7}nMzlTo-UlC|KZH*?EmRAp6}
zz2pRZ9aH8Rd>A&O{L_29il&(fO4-(=s0r<1S^rt-0gSKvv_GlE$j5#|c>-Rrm>J*)
zNQ^{g>%P{>|GE6N_>r(vK-45nL3i=Xc<9GEA90<sLS8N{CK8bzIc@#fLT@;5!b181
zR>>jD0dUdC@FWAsEi+Q~gL{Ed;&zJ40PLmO&DvhFav#$k8o%q#uN)oDbn3cw-Jqk!
z!<y|$uN1?1C|Mpp>>+prUnnxAx=*G_4+~(?uihYN0vk-vf1B-GCzHItBSUITI)l%8
zll0q8gmAMP8$sIfI-T|tJh$WuHgyF;?J(+7H6Q~z-mG^kp@lqbKE>dAtyEn6i-#Oa
z*Y@%eWLM$#^HT(GiDa@wNZU^H;f0ZVgnZ`u+DR9DJ`7kEaS?($0L@qLc7`b)5O{A{
zeLZdz;*H&zCW_9F9C53Od0zT9G==h$MWJCn$qZ-nJ)}=)6o_tTLE~L%V<=?&MBr8q
zJq4+FRdJl+Rr`vz{+h<bKDH^r={jZCoAs2^SA3`RmNiO?Sj9FLH29P$r~ET{Oji?~
z3V~=r)8)Zg+x+y>8^<ddw2lVOJXpPc<~hmJ8NTXoh3hnlQTLtbuQqTDDda~qF9?a;
zcJ1F^VvO=v#71E@>v%wwUYVvPn*meowq}+P-D~+@z1OFyUB0~Ma#OwNpB#&GZHNPO
zCvF&@JhIDL3>`sA2YXT%GfNz+m;oO@;VaS5ay*dXJ^vs!`?MrU9Z)!Q`s%*eX)g5k
zvJ!wKiO(q&Ln0WpGV<WglEZN5jedpc0;l7RC-BD}w>wVqnGuK6_RgC+r4n?+&Ks?|
zFA4?bce06^r6V4N6vdMte!bPw@L3e>g%EdACwrszv+z#%u9nv1R0H0r+YCyP>yom<
zX@Q!UVAM0GNIDX|W<}~QCb3ze3uIoo@=b4Y6O19=98cA(0*ELK(w)`IkaBdDbvLx8
z!vfO$;*&M9U1IR@NEk8YwNgqJn4YxMjpf<UTTZ2$$5MmA(h742{z<QtrO(4HuFl<Q
zA_d%GrYS5kcUT*tjC(`T)YlpcawkV<8kPt=fm1WJC^k=g<WeI>e=s-M$O%a*eILYo
zr|L)T_3tmQF+LlY6=>x~;jQ+7!r}&wt{}q#By+*_H1y;y_*9&4ueBuDeytZ?FC&uT
zK;74yLnQ_5h~0;;esLH(F@Wo}j3)F-Zkkk?D}2D3lTbeyd8As9%Lq?gtuagO`igly
z9`mF+wK7+>>)f-@G8wBQyI12=v)}5$HT8WO#Y;jZ{Mt7{d;|Y1nqhU1ZSnF9(GY1z
z^ufp78c$v$pONXbC|Y(S_rjku?uk-8`%A)5VCQD3%C*^0H>INM`bR@?`n`f9T<dLB
zT(*0gmo0d55;dIxTEVci{iex>aAviV39+*s&Ds}iTD=cewo~DOPw(qeM#CN?Lkc2G
zn|+-sIv-!hxH0fVDF2AK`8@Gh)#T+T4q6}D5?~D;+q(IToz}|N-2jbsby)$Mx0nKS
zZQk<y<-IVPb%jcd-f(l8fSi0@erSY=WcZl7pr{PYB-BOn#K{q^Wrlh{<udZzEss60
zj9bGR-2tuSv5;lZf!fTB)@HjVv?&isBtCp>Vr#tN{pz7=T_y{uPIOo8D`^KCz8vqM
zNAr@H*C33?w2iC@!JQuR1&MEKyd_m>Wj%xGqRY$W$DT4fVpSc8#_@SnVt}orqrJ5P
zmEsT3vYx?1yCs8l%5*3R+d{aZC>w|FOpGqnMef%8m$kG6dJFBSFt_{${qfcP<+e%K
z9d7lHmSIZVV7YpHd!lUht3e1%q^OdobBb?WC1RE}+bt`#m7i1gjpflTaerO@VUDt!
z((q#5bXDWL^`k};1s4_B<|u`pZ%R7una?d}ZbwF4MALrRmxZ}^Y|}lZ8&7~{?h^%9
z*k@oJm<=pHLDi5>D<Q?_EAmoSA)Ma;3=vw5arfxtxW0#;64ezxm42<e=uMePUn_qk
z{HQ~8krnp7EREOph%gVC?$$_c5^3s0HyDtw`Y^_1D{0D?;b_i}<;C^g#JPezGmua~
zw9J%bMonSp`gh^i1Qehz8K2!B#5*tweY_qXKlN}@Z_A*vf+7=YQ|Nx<x}#r;U|;Zd
zVEwh=0++0U=&Ll4fbNK+g{O*D{I=Wk?>42bdyWM(v9SUc>&0&#?M$&X%R8SP4T0Ip
zJ6VOKdmkGEmY3hM?>aR?A8n-Y_FMUx2$etjNEjy5-0sw{i7&Zm!$RcIam?^3AX#9p
zVmO6F9a|u>;bsl~l8{5GnS;k8*~;g{`gAE)pnhCCuaoV_^`qiq3^$%rtAN)0QI*WA
zO}cD=&FP5{B(Zt<uH3-qi{tl&m(TJ^WE<Om=y%m+8=oU>d;tKwNJnL515IV+f9&Gn
z_U>|nQe-uH71;al8ux>45KG5M5mxCjsi?9yk*QYFI7OLq=!l$$^HFNX#h0vGxbLIt
zKCW+8J&kw@Bnb|cl9m!^(}=y)CIj2ZNPQV>4@_?N4K)yjj(=n!hONSlW1nXZ*Tmg^
zMnj_{dY|?|7J5w6h_3VPCr-T1H~oI;m#y;Z$=MAoG&HgHS=P&aY<)91%{~4FQZM7`
zweIQ3*iB!F<!Y_*`#I3MUf&Jnu|rDevvV_%W2zd(gtA!=7(7hoyhfPCIiKdXNI7o_
zI=^|?1<9Scnck|0B_S2IA*EB{uPlxfcj<GO)2w??eEvK+Wa?P^ozc(@e4ask!6{&S
z@V@-%YIRyc?tbb^kHWA09&&1-HMa{5vAKJwLC0J`kNvuq+jk*vtL=wJ7TWQgctNkb
z`@OY|zlDx+Sgj8~&|OLkheh4v^ph`f8^J$sQ8>w7u*7mzP%C7&X}tjuM|V4f)h`6S
z;c(j9U4xEa1Ov!aJ%d!2;^6=Q6~Ga<t7@jJgG9MuM67MyY|tWp7<b&RD*zxP@8@of
zaz=XrZP50PuCk!L_AU_6(N-38Q$iP_>#mG;aMTF!L>mX_nV<rkQ3zX*yd0T~9})+E
zL3>#P{V*=BSfrmU=r=ABcl~Qv6a@UO;^iz0GSf8xD!X~2fiMx62n4L^=jbC2k|P7k
zc-q<_ja1bBguva&f*icO+>xT9zP`R9zEBZ2PkT`@1Og!n5f>E~2jeurSbtY9Yd^3n
zmgg759~dfVEXvc--OJI<75EF&+Q!Y>OBMve$$|ff596+@`%idR?4K;)_z?B8b{7>B
zfrw%-qJOo(da3&0K>iHq-&$ZzaGTSjMrf>?w<ij%>VtOm;`u9tE$W~4?%tj*ztgcr
ziK1Q57@R5==T+=)F4Z-44gP8IivoK`jQejZ9NB-<^m4TO7g>Lc?U&?tI)4oWXZ}yz
zziIy?_utAmEnQutiW|!NS9qE#vY=o7k+yCqM_c6YOBf7S4HViAY$GNm28KyWNrDjw
zDG4w{0xf}p+d=JY5aNG<(sadoS-Yapzo2m7B91s56dVS#5l2aYttG_mz%Ya@8f+~k
z4gpJ|#o=giad8`4H1sbJ`ks!suC#XfYgE6WY;jO9aXUMlQ){pd8j6FG5`%&fa0Ci$
z3z0xTt<ex0Gy?S-$`*xGbMwSl<I3rXv9?Exy1Ux{7W^U{sc4`n3lbNB{A<L(#oEgb
zXCMpGwMGH;jsGPvam1jFy{v!HDJBV#gh9j+a48rRCIJ=umy#*k6N_ubU!Y<T5$GSl
zzv_a-WrJhZ`d6dk0DfEGl0hnaqOHB$JWbr(Tx3DNVgmlM{N3L`nLnzc;fTd)`2TA8
zKbzhd?eWLmAJ2e`<8KiV_`7eB)~G+6V6A=7w!a<W?EVm;9IRdK(YOcv&yM=XxZ}U6
zmXw%{l#Q4e9E`HHh2yFvg$7GOB&EQT5;nieC<%dCOZ*)j>t^TWYwd|vw8v$N%Ldm$
zzq0}I|G}le-_?B`(7%X+h{M1TaWF*6L=1+6LXqN9LJ$ZN0s)Es-{**sw1FWca7p2o
zm?$s|itB6}lmxD+;Wlt6LPE+~ObY(@p#L{=gd-vUBS#t0Uvt+#$|@uJ{}uiZg}+R8
zIBxzJ!%d>NIa&0dlk%UW{pywf!{g6}`#<!6gZ{6R|A^oJ()C}u{v!tdBjx{g*MI5y
zj~Muml>ggZ|7&!S{cBl=cE$bG^TjRN+H*el;ud@e10569-%CHhCGNmj{6`-rp5R5C
zF`k!^o-sgSdlC0qE&$I)TU`YZbA>|#cSGu~VU7g=$mxHbc!2C&7Th4Qm!_^N@eV!-
z6)(v>R*E|S0D-xtilPbbjo!S~q<jd)6>#23$on@YPfTuD0)2i;ViR9dS!;UQoQ~nF
z|IJYdI~zxC$P1friq;gE3CPY`QZMev=vi7+Rx?*U;w%A!XVaOSoJ*P-4K3N4%A%HE
zp4~N~Z{oVTTYGVDQTRg$m(=iC|8W*-*F9|8tBF!eZ!OkjHl28W*y2uy%s7d?IkU6>
zbiajUb-aT%d3Dp2#X=dm84eF_p?!6Ecz)RdT-7o*Udp~};(X2A`HA<cNsm}s$ec!h
z4QGB%E7757wh`hJdE(<0O3o0<2H$(RlNX=4ivs312*M{bH>4LMDK9Q_*lu+wp6@m6
zTSUb?O_!L#I?7HW8;Jy-WR?2Au<R{h)v_4(YtFI7a8DmAkC=SuSQos36m8W?)6=OD
zFI!W)xe)O<p$Ou$ip97d)_+)^dy{Cs^JC!c!gJb(+s3;><UeK@Q+)-4z!Uqo$uCzm
zZhYFY+)DI`KfAFPj&0ig7(p=C{FM47tf@(kB|1mr_0nhMf&za`X^717wFjMYMnTmC
z?mDIP)mXar<@xy=XP<k;Z<p0kwKONrH@{kjTjac2_<8p$g?)#e+Q`R}HLZp&Z<qd?
zbC9jUhzg6#lSTM)=au(~(m(ur{9oS=9bAQm48E^i@7M(go4CJeZ4mmre0>(ld()wG
zzWLN_^aHh^)p>c3v(SbECcdr{Q7vk0=<ka#E(!Q{CRn3X=Jj$@qDxQ?{VM;QNX5u>
z!&fPp**0!<>n>}XT%441h|6`W0RukprAr={azHMhoE^1UGJC<MlH$!VvZKoH<YIyS
zjhhg=5R2zhf_5RdGS(#R3XZr>OX{)xGAOM0kn=)?-`K?=?NmbpS0^+jEB8+JW?qtq
zvJZ1A>d;B7{WX=cqO-FrM!H)0PEyum*=}SZQyo^f+uixKBW@5=;hik1z2!8S0`I!L
zd}n^U@%q}tGCLV$_SIR+4+G~(;r?<nx6`E|Ij(T(OHnS-Ad~d9Q|4sqkB69S@4o2u
z_Jp<VV~$f&#EhMDL*jC?;cFbj;90n3C8pWnwM$X6u1#4Z%KSpgts!U`o|ZVl!xQ;z
fo9KT$98=ME6B8oy@^tT5|B`5`>Z#NyS%v>U(Vh6k

literal 0
HcmV?d00001

diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte
index a8711d220b..72cedb2b21 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte
@@ -128,10 +128,10 @@
         >
           <div class="item-body">
             <img
-              width="20"
-              height="20"
+              width={20}
+              height={20}
               src={externalActions[action.stepId].icon}
-              alt="zapier"
+              alt={externalActions[action.stepId].name}
             />
             <span class="icon-spacing">
               <Body size="XS">
diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js
index c6f8d25640..d5d382485c 100644
--- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js
+++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js
@@ -1,5 +1,6 @@
 import DiscordLogo from "assets/discord.svg"
 import ZapierLogo from "assets/zapier.png"
+import n8nLogo from "assets/n8n_square.png"
 import MakeLogo from "assets/make.svg"
 import SlackLogo from "assets/slack.svg"
 
@@ -8,4 +9,5 @@ export const externalActions = {
   discord: { name: "discord", icon: DiscordLogo },
   slack: { name: "slack", icon: SlackLogo },
   integromat: { name: "integromat", icon: MakeLogo },
+  n8n: { name: "n8n", icon: n8nLogo },
 }
diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
index 7ba1c8a4b1..707317f9e6 100644
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@@ -79,6 +79,7 @@
     disableWrapping: true,
   })
   $: editingJs = codeMode === EditorModes.JS
+  $: requiredProperties = block.schema.inputs.required || []
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -359,6 +360,11 @@
     )
   }
 
+  function getFieldLabel(key, value) {
+    const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
+    return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
+  }
+
   onMount(async () => {
     try {
       await environment.loadVariables()
@@ -376,7 +382,7 @@
           <Label
             tooltip={value.title === "Binding / Value"
               ? "If using the String input type, please use a comma or newline separated string"
-              : null}>{value.title || (key === "row" ? "Table" : key)}</Label
+              : null}>{getFieldLabel(key, value)}</Label
           >
         {/if}
         <div class:field-width={shouldRenderField(value)}>
diff --git a/packages/builder/src/constants/backend/automations.js b/packages/builder/src/constants/backend/automations.js
index 6981418fa7..7c3e17e225 100644
--- a/packages/builder/src/constants/backend/automations.js
+++ b/packages/builder/src/constants/backend/automations.js
@@ -27,6 +27,7 @@ export const ActionStepID = {
   slack: "slack",
   zapier: "zapier",
   integromat: "integromat",
+  n8n: "n8n",
 }
 
 export const Features = {
diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts
index ac8a340e82..eee8ab4a7b 100644
--- a/packages/server/src/automations/actions.ts
+++ b/packages/server/src/automations/actions.ts
@@ -9,6 +9,7 @@ import * as serverLog from "./steps/serverLog"
 import * as discord from "./steps/discord"
 import * as slack from "./steps/slack"
 import * as zapier from "./steps/zapier"
+import * as n8n from "./steps/n8n"
 import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
@@ -48,6 +49,7 @@ const ACTION_IMPLS: Record<
   slack: slack.run,
   zapier: zapier.run,
   integromat: make.run,
+  n8n: n8n.run,
 }
 export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
   {
@@ -70,6 +72,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
     slack: slack.definition,
     zapier: zapier.definition,
     integromat: make.definition,
+    n8n: n8n.definition,
   }
 
 // don't add the bash script/definitions unless in self host
diff --git a/packages/server/src/automations/steps/make.ts b/packages/server/src/automations/steps/make.ts
index 06e96907d9..555df8308a 100644
--- a/packages/server/src/automations/steps/make.ts
+++ b/packages/server/src/automations/steps/make.ts
@@ -34,28 +34,8 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.JSON,
           title: "Payload",
         },
-        value1: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 1",
-        },
-        value2: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 2",
-        },
-        value3: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 3",
-        },
-        value4: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 4",
-        },
-        value5: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 5",
-        },
       },
-      required: ["url", "value1", "value2", "value3", "value4", "value5"],
+      required: ["url", "body"],
     },
     outputs: {
       properties: {
diff --git a/packages/server/src/automations/steps/n8n.ts b/packages/server/src/automations/steps/n8n.ts
new file mode 100644
index 0000000000..c400c7037a
--- /dev/null
+++ b/packages/server/src/automations/steps/n8n.ts
@@ -0,0 +1,125 @@
+import fetch, { HeadersInit } from "node-fetch"
+import { getFetchResponse } from "./utils"
+import {
+  AutomationActionStepId,
+  AutomationStepSchema,
+  AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
+  AutomationFeature,
+  HttpMethod,
+} from "@budibase/types"
+
+export const definition: AutomationStepSchema = {
+  name: "n8n Integration",
+  stepTitle: "n8n",
+  tagline: "Trigger an n8n workflow",
+  description:
+    "Performs a webhook call to n8n and gets the response (if configured)",
+  icon: "ri-shut-down-line",
+  stepId: AutomationActionStepId.n8n,
+  type: AutomationStepType.ACTION,
+  internal: false,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        url: {
+          type: AutomationIOType.STRING,
+          title: "Webhook URL",
+        },
+        method: {
+          type: AutomationIOType.STRING,
+          title: "Method",
+          enum: Object.values(HttpMethod),
+        },
+        authorization: {
+          type: AutomationIOType.STRING,
+          title: "Authorization",
+        },
+        body: {
+          type: AutomationIOType.JSON,
+          title: "Payload",
+        },
+      },
+      required: ["url", "method"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether call was successful",
+        },
+        httpStatus: {
+          type: AutomationIOType.NUMBER,
+          description: "The HTTP status code returned",
+        },
+        response: {
+          type: AutomationIOType.OBJECT,
+          description: "The webhook response - this can have properties",
+        },
+      },
+      required: ["success", "response"],
+    },
+  },
+}
+
+export async function run({ inputs }: AutomationStepInput) {
+  const { url, body, method, authorization } = inputs
+
+  let payload = {}
+  try {
+    payload = body?.value ? JSON.parse(body?.value) : {}
+  } catch (err) {
+    return {
+      httpStatus: 400,
+      response: "Invalid payload JSON",
+      success: false,
+    }
+  }
+
+  if (!url?.trim()?.length) {
+    return {
+      httpStatus: 400,
+      response: "Missing Webhook URL",
+      success: false,
+    }
+  }
+  let response
+  let request: {
+    method: string
+    headers: HeadersInit
+    body?: string
+  } = {
+    method: method || HttpMethod.GET,
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: authorization,
+    },
+  }
+  if (!["GET", "HEAD"].includes(request.method)) {
+    request.body = JSON.stringify({
+      ...payload,
+    })
+  }
+
+  try {
+    response = await fetch(url, request)
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
+
+  const { status, message } = await getFetchResponse(response)
+  return {
+    httpStatus: status,
+    success: status === 200,
+    response: message,
+  }
+}
diff --git a/packages/server/src/automations/steps/zapier.ts b/packages/server/src/automations/steps/zapier.ts
index eeff0c2c7d..e48d677228 100644
--- a/packages/server/src/automations/steps/zapier.ts
+++ b/packages/server/src/automations/steps/zapier.ts
@@ -32,26 +32,6 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.JSON,
           title: "Payload",
         },
-        value1: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 1",
-        },
-        value2: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 2",
-        },
-        value3: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 3",
-        },
-        value4: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 4",
-        },
-        value5: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 5",
-        },
       },
       required: ["url"],
     },
diff --git a/packages/server/src/automations/tests/n8n.spec.ts b/packages/server/src/automations/tests/n8n.spec.ts
new file mode 100644
index 0000000000..d60a08b53b
--- /dev/null
+++ b/packages/server/src/automations/tests/n8n.spec.ts
@@ -0,0 +1,68 @@
+import { getConfig, afterAll, runStep, actions } from "./utilities"
+
+describe("test the outgoing webhook action", () => {
+  let config = getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll()
+
+  it("should be able to run the action and default to 'get'", async () => {
+    const res = await runStep(actions.n8n.stepId, {
+      url: "http://www.example.com",
+      body: {
+        test: "IGNORE_ME",
+      },
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("GET")
+    expect(res.response.body).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+
+  it("should add the payload props when a JSON string is provided", async () => {
+    const payload = `{ "name": "Adam", "age": 9 }`
+    const res = await runStep(actions.n8n.stepId, {
+      body: {
+        value: payload,
+      },
+      method: "POST",
+      url: "http://www.example.com",
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("POST")
+    expect(res.response.body).toEqual(`{"name":"Adam","age":9}`)
+    expect(res.success).toEqual(true)
+  })
+
+  it("should return a 400 if the JSON payload string is malformed", async () => {
+    const payload = `{ value1 1 }`
+    const res = await runStep(actions.n8n.stepId, {
+      value1: "ONE",
+      body: {
+        value: payload,
+      },
+      method: "POST",
+      url: "http://www.example.com",
+    })
+    expect(res.httpStatus).toEqual(400)
+    expect(res.response).toEqual("Invalid payload JSON")
+    expect(res.success).toEqual(false)
+  })
+
+  it("should not append the body if the method is HEAD", async () => {
+    const res = await runStep(actions.n8n.stepId, {
+      url: "http://www.example.com",
+      method: "HEAD",
+      body: {
+        test: "IGNORE_ME",
+      },
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("HEAD")
+    expect(res.response.body).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+})
diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts
index 9cb8f8e2c1..44c62f60b7 100644
--- a/packages/server/src/integrations/rest.ts
+++ b/packages/server/src/integrations/rest.ts
@@ -10,6 +10,7 @@ import {
   RestAuthType,
   RestBasicAuthConfig,
   RestBearerAuthConfig,
+  HttpMethod,
 } from "@budibase/types"
 import get from "lodash/get"
 import * as https from "https"
@@ -86,30 +87,30 @@ const SCHEMA: Integration = {
   query: {
     create: {
       readable: true,
-      displayName: "POST",
+      displayName: HttpMethod.POST,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     read: {
-      displayName: "GET",
+      displayName: HttpMethod.GET,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     update: {
-      displayName: "PUT",
+      displayName: HttpMethod.PUT,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     patch: {
-      displayName: "PATCH",
+      displayName: HttpMethod.PATCH,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     delete: {
-      displayName: "DELETE",
+      displayName: HttpMethod.DELETE,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
@@ -358,7 +359,7 @@ class RestIntegration implements IntegrationBase {
       path = "",
       queryString = "",
       headers = {},
-      method = "GET",
+      method = HttpMethod.GET,
       disabledHeaders,
       bodyType,
       requestBody,
@@ -413,23 +414,23 @@ class RestIntegration implements IntegrationBase {
   }
 
   async create(opts: RestQuery) {
-    return this._req({ ...opts, method: "POST" })
+    return this._req({ ...opts, method: HttpMethod.POST })
   }
 
   async read(opts: RestQuery) {
-    return this._req({ ...opts, method: "GET" })
+    return this._req({ ...opts, method: HttpMethod.GET })
   }
 
   async update(opts: RestQuery) {
-    return this._req({ ...opts, method: "PUT" })
+    return this._req({ ...opts, method: HttpMethod.PUT })
   }
 
   async patch(opts: RestQuery) {
-    return this._req({ ...opts, method: "PATCH" })
+    return this._req({ ...opts, method: HttpMethod.PATCH })
   }
 
   async delete(opts: RestQuery) {
-    return this._req({ ...opts, method: "DELETE" })
+    return this._req({ ...opts, method: HttpMethod.DELETE })
   }
 }
 
diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts
index 91a1a2ab68..fef72b78a9 100644
--- a/packages/types/src/documents/app/automation.ts
+++ b/packages/types/src/documents/app/automation.ts
@@ -69,6 +69,7 @@ export enum AutomationActionStepId {
   slack = "slack",
   zapier = "zapier",
   integromat = "integromat",
+  n8n = "n8n",
 }
 
 export interface EmailInvite {
diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts
index 790c297813..81aa90b807 100644
--- a/packages/types/src/documents/app/query.ts
+++ b/packages/types/src/documents/app/query.ts
@@ -64,3 +64,12 @@ export interface ExecuteQueryRequest {
 export interface ExecuteQueryResponse {
   data: Row[]
 }
+
+export enum HttpMethod {
+  GET = "GET",
+  POST = "POST",
+  PATCH = "PATCH",
+  PUT = "PUT",
+  HEAD = "HEAD",
+  DELETE = "DELETE",
+}

From 74957725dd08eeb97a3ba7bef6bf9a97e32e56d3 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 13:44:43 +0000
Subject: [PATCH 17/34] Revert "Fix updating users via cross-service comms
 (public API)"

---
 .../server/src/utilities/workerRequests.ts    | 20 ++++++-------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 5612084216..69ce58c8a9 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -14,23 +14,12 @@ export function request(ctx?: Ctx, request?: any) {
   if (!request.headers) {
     request.headers = {}
   }
-
   if (!ctx) {
     request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
-  } else if (ctx.headers) {
-    // copy all Budibase utilised headers over - copying everything can have
-    // side effects like requests being rejected due to odd content types etc
-    for (let header of Object.values(constants.Header)) {
-      if (ctx.headers[header]) {
-        request.headers[header] = ctx.headers[header]
-      }
+    if (tenancy.isTenantIdSet()) {
+      request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
     }
   }
-
-  // apply tenancy if its available
-  if (tenancy.isTenantIdSet()) {
-    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
-  }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"
     request.body =
@@ -40,6 +29,9 @@ export function request(ctx?: Ctx, request?: any) {
   } else {
     delete request.body
   }
+  if (ctx && ctx.headers) {
+    request.headers = ctx.headers
+  }
 
   // add x-budibase-correlation-id header
   logging.correlation.setHeader(request.headers)
@@ -62,7 +54,7 @@ async function checkResponse(
     }
     const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
     if (ctx) {
-      ctx.throw(response.status || 500, msg)
+      ctx.throw(msg, response.status)
     } else {
       throw msg
     }

From 36e1a20c03476fe7139982b46f86957ec8d2ba5c Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 13:45:08 +0000
Subject: [PATCH 18/34] Revert "Revert "Fix updating users via cross-service
 comms (public API)""

---
 .../server/src/utilities/workerRequests.ts    | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 69ce58c8a9..5612084216 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -14,12 +14,23 @@ export function request(ctx?: Ctx, request?: any) {
   if (!request.headers) {
     request.headers = {}
   }
+
   if (!ctx) {
     request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
-    if (tenancy.isTenantIdSet()) {
-      request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  } else if (ctx.headers) {
+    // copy all Budibase utilised headers over - copying everything can have
+    // side effects like requests being rejected due to odd content types etc
+    for (let header of Object.values(constants.Header)) {
+      if (ctx.headers[header]) {
+        request.headers[header] = ctx.headers[header]
+      }
     }
   }
+
+  // apply tenancy if its available
+  if (tenancy.isTenantIdSet()) {
+    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"
     request.body =
@@ -29,9 +40,6 @@ export function request(ctx?: Ctx, request?: any) {
   } else {
     delete request.body
   }
-  if (ctx && ctx.headers) {
-    request.headers = ctx.headers
-  }
 
   // add x-budibase-correlation-id header
   logging.correlation.setHeader(request.headers)
@@ -54,7 +62,7 @@ async function checkResponse(
     }
     const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
     if (ctx) {
-      ctx.throw(msg, response.status)
+      ctx.throw(response.status || 500, msg)
     } else {
       throw msg
     }

From 766cac0a440564d370d3d41999a16f3cfb0aabbd Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 15 Feb 2024 14:47:12 +0000
Subject: [PATCH 19/34] Delete monolith frontend.js store that was somehow
 added back in

---
 .../src/builderStore/store/frontend.js        | 1520 -----------------
 1 file changed, 1520 deletions(-)
 delete mode 100644 packages/builder/src/builderStore/store/frontend.js

diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js
deleted file mode 100644
index fd492cca0b..0000000000
--- a/packages/builder/src/builderStore/store/frontend.js
+++ /dev/null
@@ -1,1520 +0,0 @@
-import { get, writable } from "svelte/store"
-import { cloneDeep } from "lodash/fp"
-import {
-  selectedScreen,
-  selectedComponent,
-  screenHistoryStore,
-  automationHistoryStore,
-} from "builderStore"
-import {
-  datasources,
-  integrations,
-  queries,
-  database,
-  tables,
-} from "stores/backend"
-import { API } from "api"
-import analytics, { Events } from "analytics"
-import {
-  findComponentParent,
-  findClosestMatchingComponent,
-  findAllMatchingComponents,
-  findComponent,
-  getComponentSettings,
-  makeComponentUnique,
-  findComponentPath,
-} from "../componentUtils"
-import { Helpers } from "@budibase/bbui"
-import { Utils } from "@budibase/frontend-core"
-import {
-  BUDIBASE_INTERNAL_DB_ID,
-  DB_TYPE_INTERNAL,
-  DB_TYPE_EXTERNAL,
-} from "constants/backend"
-import {
-  buildFormSchema,
-  getSchemaForDatasource,
-} from "builderStore/dataBinding"
-import { makePropSafe as safe } from "@budibase/string-templates"
-import { getComponentFieldOptions } from "helpers/formFields"
-import { createBuilderWebsocket } from "builderStore/websocket"
-import { BuilderSocketEvent } from "@budibase/shared-core"
-import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
-
-const INITIAL_FRONTEND_STATE = {
-  initialised: false,
-  apps: [],
-  name: "",
-  url: "",
-  description: "",
-  layouts: [],
-  screens: [],
-  components: [],
-  clientFeatures: {
-    spectrumThemes: false,
-    intelligentLoading: false,
-    deviceAwareness: false,
-    state: false,
-    rowSelection: false,
-    customThemes: false,
-    devicePreview: false,
-    messagePassing: false,
-    continueIfAction: false,
-    showNotificationAction: false,
-    sidePanel: false,
-  },
-  features: {
-    componentValidation: false,
-    disableUserMetadata: false,
-  },
-  errors: [],
-  hasAppPackage: false,
-  libraries: null,
-  appId: "",
-  routes: {},
-  clientLibPath: "",
-  theme: "",
-  customTheme: {},
-  previewDevice: "desktop",
-  highlightedSettingKey: null,
-  propertyFocus: null,
-  builderSidePanel: false,
-  hasLock: true,
-  showPreview: false,
-
-  // URL params
-  selectedScreenId: null,
-  selectedComponentId: null,
-  selectedLayoutId: null,
-
-  // Client state
-  selectedComponentInstance: null,
-
-  // Onboarding
-  onboarding: false,
-  tourNodes: null,
-}
-
-export const getFrontendStore = () => {
-  const store = writable({ ...INITIAL_FRONTEND_STATE })
-  let websocket
-
-  // This is a fake implementation of a "patch" API endpoint to try and prevent
-  // 409s. All screen doc mutations (aside from creation) use this function,
-  // which queues up invocations sequentially and ensures pending mutations are
-  // always applied to the most up-to-date doc revision.
-  // This is slightly better than just a traditional "patch" endpoint and this
-  // supports deeply mutating the current doc rather than just appending data.
-  const sequentialScreenPatch = Utils.sequential(async (patchFn, screenId) => {
-    const state = get(store)
-    const screen = state.screens.find(screen => screen._id === screenId)
-    if (!screen) {
-      return
-    }
-    let clone = cloneDeep(screen)
-    const result = patchFn(clone)
-    // An explicit false result means skip this change
-    if (result === false) {
-      return
-    }
-    return await store.actions.screens.save(clone)
-  })
-
-  store.actions = {
-    reset: () => {
-      store.set({ ...INITIAL_FRONTEND_STATE })
-      websocket?.disconnect()
-      websocket = null
-    },
-    initialise: async pkg => {
-      const { layouts, screens, application, clientLibPath, hasLock } = pkg
-      if (!websocket) {
-        websocket = createBuilderWebsocket(application.appId)
-      }
-      await store.actions.components.refreshDefinitions(application.appId)
-
-      // Reset store state
-      store.update(state => ({
-        ...state,
-        libraries: application.componentLibraries,
-        name: application.name,
-        description: application.description,
-        appId: application.appId,
-        url: application.url,
-        layouts: layouts || [],
-        screens: screens || [],
-        theme: application.theme || "spectrum--light",
-        customTheme: application.customTheme,
-        hasAppPackage: true,
-        appInstance: application.instance,
-        clientLibPath,
-        previousTopNavPath: {},
-        version: application.version,
-        revertableVersion: application.revertableVersion,
-        upgradableVersion: application.upgradableVersion,
-        navigation: application.navigation || {},
-        usedPlugins: application.usedPlugins || [],
-        hasLock,
-        features: {
-          ...INITIAL_FRONTEND_STATE.features,
-          ...application.features,
-        },
-        automations: application.automations || {},
-        icon: application.icon || {},
-        initialised: true,
-      }))
-      screenHistoryStore.reset()
-      automationHistoryStore.reset()
-
-      // Initialise backend stores
-      database.set(application.instance)
-      await datasources.init()
-      await integrations.init()
-      await queries.init()
-      await tables.init()
-    },
-    theme: {
-      save: async theme => {
-        const appId = get(store).appId
-        const app = await API.saveAppMetadata({
-          appId,
-          metadata: { theme },
-        })
-        store.update(state => {
-          state.theme = app.theme
-          return state
-        })
-      },
-    },
-    customTheme: {
-      save: async customTheme => {
-        const appId = get(store).appId
-        const app = await API.saveAppMetadata({
-          appId,
-          metadata: { customTheme },
-        })
-        store.update(state => {
-          state.customTheme = app.customTheme
-          return state
-        })
-      },
-    },
-    navigation: {
-      save: async navigation => {
-        const appId = get(store).appId
-        const app = await API.saveAppMetadata({
-          appId,
-          metadata: { navigation },
-        })
-        store.update(state => {
-          state.navigation = app.navigation
-          return state
-        })
-      },
-    },
-    screens: {
-      select: screenId => {
-        // Check this screen exists
-        const state = get(store)
-        const screen = state.screens.find(screen => screen._id === screenId)
-        if (!screen) {
-          return
-        }
-
-        // Check screen isn't already selected
-        if (state.selectedScreenId === screen._id) {
-          return
-        }
-
-        // Select new screen
-        store.update(state => {
-          state.selectedScreenId = screen._id
-          return state
-        })
-      },
-      validate: screen => {
-        // Recursive function to find any illegal children in component trees
-        const findIllegalChild = (
-          component,
-          illegalChildren = [],
-          legalDirectChildren = []
-        ) => {
-          const type = component._component
-
-          if (illegalChildren.includes(type)) {
-            return type
-          }
-          if (
-            legalDirectChildren.length &&
-            !legalDirectChildren.includes(type)
-          ) {
-            return type
-          }
-          if (!component?._children?.length) {
-            return
-          }
-
-          if (type === "@budibase/standard-components/sidepanel") {
-            illegalChildren = []
-          }
-
-          const definition = store.actions.components.getDefinition(
-            component._component
-          )
-          // Reset whitelist for direct children
-          legalDirectChildren = []
-          if (definition?.legalDirectChildren?.length) {
-            legalDirectChildren = definition.legalDirectChildren.map(x => {
-              return `@budibase/standard-components/${x}`
-            })
-          }
-
-          // Append blacklisted components and remove duplicates
-          if (definition?.illegalChildren?.length) {
-            const blacklist = definition.illegalChildren.map(x => {
-              return `@budibase/standard-components/${x}`
-            })
-            illegalChildren = [...new Set([...illegalChildren, ...blacklist])]
-          }
-
-          // Recurse on all children
-          for (let child of component._children) {
-            const illegalChild = findIllegalChild(
-              child,
-              illegalChildren,
-              legalDirectChildren
-            )
-            if (illegalChild) {
-              return illegalChild
-            }
-          }
-        }
-
-        // Validate the entire tree and throw an error if an illegal child is
-        // found anywhere
-        const illegalChild = findIllegalChild(screen.props)
-        if (illegalChild) {
-          const def = store.actions.components.getDefinition(illegalChild)
-          throw `You can't place a ${def.name} here`
-        }
-      },
-      save: async screen => {
-        const state = get(store)
-
-        // Validate screen structure if the app supports it
-        if (state.features?.componentValidation) {
-          store.actions.screens.validate(screen)
-        }
-
-        // Check screen definition for any component settings which need updated
-        store.actions.screens.enrichEmptySettings(screen)
-
-        // Save screen
-        const creatingNewScreen = screen._id === undefined
-        const savedScreen = await API.saveScreen(screen)
-        const routesResponse = await API.fetchAppRoutes()
-
-        // If plugins changed we need to fetch the latest app metadata
-        let usedPlugins = state.usedPlugins
-        if (savedScreen.pluginAdded) {
-          const { application } = await API.fetchAppPackage(state.appId)
-          usedPlugins = application.usedPlugins || []
-        }
-
-        // Update state
-        store.update(state => {
-          // Update screen object
-          const idx = state.screens.findIndex(x => x._id === savedScreen._id)
-          if (idx !== -1) {
-            state.screens.splice(idx, 1, savedScreen)
-          } else {
-            state.screens.push(savedScreen)
-          }
-
-          // Select the new screen if creating a new one
-          if (creatingNewScreen) {
-            state.selectedScreenId = savedScreen._id
-            state.selectedComponentId = savedScreen.props._id
-          }
-
-          // Update routes
-          state.routes = routesResponse.routes
-
-          // Update used plugins
-          state.usedPlugins = usedPlugins
-          return state
-        })
-        return savedScreen
-      },
-      patch: async (patchFn, screenId) => {
-        // Default to the currently selected screen
-        if (!screenId) {
-          const state = get(store)
-          screenId = state.selectedScreenId
-        }
-        if (!screenId || !patchFn) {
-          return
-        }
-        return await sequentialScreenPatch(patchFn, screenId)
-      },
-      replace: async (screenId, screen) => {
-        if (!screenId) {
-          return
-        }
-        if (!screen) {
-          // Screen deletion
-          store.update(state => ({
-            ...state,
-            screens: state.screens.filter(x => x._id !== screenId),
-          }))
-        } else {
-          const index = get(store).screens.findIndex(x => x._id === screen._id)
-          if (index === -1) {
-            // Screen addition
-            store.update(state => ({
-              ...state,
-              screens: [...state.screens, screen],
-            }))
-          } else {
-            // Screen update
-            store.update(state => {
-              state.screens[index] = screen
-              return state
-            })
-          }
-        }
-      },
-      delete: async screens => {
-        const screensToDelete = Array.isArray(screens) ? screens : [screens]
-
-        // Build array of promises to speed up bulk deletions
-        let promises = []
-        let deleteUrls = []
-        screensToDelete.forEach(screen => {
-          // Delete the screen
-          promises.push(
-            API.deleteScreen({
-              screenId: screen._id,
-              screenRev: screen._rev,
-            })
-          )
-          // Remove links to this screen
-          deleteUrls.push(screen.routing.route)
-        })
-
-        await Promise.all(promises)
-        await store.actions.links.delete(deleteUrls)
-        const deletedIds = screensToDelete.map(screen => screen._id)
-        const routesResponse = await API.fetchAppRoutes()
-        store.update(state => {
-          // Remove deleted screens from state
-          state.screens = state.screens.filter(screen => {
-            return !deletedIds.includes(screen._id)
-          })
-
-          // Deselect the current screen if it was deleted
-          if (deletedIds.includes(state.selectedScreenId)) {
-            state.selectedScreenId = null
-            state.selectedComponentId = null
-          }
-
-          // Update routing
-          state.routes = routesResponse.routes
-
-          return state
-        })
-        return null
-      },
-      updateSetting: async (screen, name, value) => {
-        if (!screen || !name) {
-          return
-        }
-
-        // Apply setting update
-        const patch = screen => {
-          if (!screen) {
-            return false
-          }
-          // Skip update if the value is the same
-          if (Helpers.deepGet(screen, name) === value) {
-            return false
-          }
-          Helpers.deepSet(screen, name, value)
-        }
-        await store.actions.screens.patch(patch, screen._id)
-
-        // Ensure we don't have more than one home screen for this new role.
-        // This could happen after updating multiple different settings.
-        const state = get(store)
-        const updatedScreen = state.screens.find(s => s._id === screen._id)
-        if (!updatedScreen) {
-          return
-        }
-        const otherHomeScreens = state.screens.filter(s => {
-          return (
-            s.routing.roleId === updatedScreen.routing.roleId &&
-            s.routing.homeScreen &&
-            s._id !== screen._id
-          )
-        })
-        if (otherHomeScreens.length && updatedScreen.routing.homeScreen) {
-          const patch = screen => {
-            screen.routing.homeScreen = false
-          }
-          for (let otherHomeScreen of otherHomeScreens) {
-            await store.actions.screens.patch(patch, otherHomeScreen._id)
-          }
-        }
-      },
-      removeCustomLayout: async screen => {
-        // Pull relevant settings from old layout, if required
-        const layout = get(store).layouts.find(x => x._id === screen.layoutId)
-        const patch = screen => {
-          screen.layoutId = null
-          screen.showNavigation = layout?.props.navigation !== "None"
-          screen.width = layout?.props.width || "Large"
-        }
-        await store.actions.screens.patch(patch, screen._id)
-      },
-      enrichEmptySettings: screen => {
-        // Flatten the recursive component tree
-        const components = findAllMatchingComponents(screen.props, x => x)
-
-        // Iterate over all components and run checks
-        components.forEach(component => {
-          store.actions.components.enrichEmptySettings(component, {
-            screen,
-          })
-        })
-      },
-    },
-    preview: {
-      setDevice: device => {
-        store.update(state => {
-          state.previewDevice = device
-          return state
-        })
-      },
-      sendEvent: (name, payload) => {
-        const { previewEventHandler } = get(store)
-        previewEventHandler?.(name, payload)
-      },
-      registerEventHandler: handler => {
-        store.update(state => {
-          state.previewEventHandler = handler
-          return state
-        })
-      },
-    },
-    layouts: {
-      select: layoutId => {
-        // Check this layout exists
-        const state = get(store)
-        const layout = state.layouts.find(layout => layout._id === layoutId)
-        if (!layout) {
-          return
-        }
-
-        // Check layout isn't already selected
-        if (
-          state.selectedLayoutId === layout._id &&
-          state.selectedComponentId === layout.props?._id
-        ) {
-          return
-        }
-
-        // Select new layout
-        store.update(state => {
-          state.selectedLayoutId = layout._id
-          state.selectedComponentId = layout.props?._id
-          return state
-        })
-      },
-      delete: async layout => {
-        if (!layout?._id) {
-          return
-        }
-        await API.deleteLayout({
-          layoutId: layout._id,
-          layoutRev: layout._rev,
-        })
-        store.update(state => {
-          state.layouts = state.layouts.filter(x => x._id !== layout._id)
-          return state
-        })
-      },
-    },
-    components: {
-      refreshDefinitions: async appId => {
-        if (!appId) {
-          appId = get(store).appId
-        }
-
-        // Fetch definitions and filter out custom component definitions so we
-        // can flag them
-        const components = await API.fetchComponentLibDefinitions(appId)
-        const customComponents = Object.keys(components).filter(name =>
-          name.startsWith("plugin/")
-        )
-
-        // Update store
-        store.update(state => ({
-          ...state,
-          components,
-          customComponents,
-          clientFeatures: {
-            ...INITIAL_FRONTEND_STATE.clientFeatures,
-            ...components.features,
-          },
-        }))
-      },
-      getDefinition: componentName => {
-        if (!componentName) {
-          return null
-        }
-        return get(store).components[componentName]
-      },
-      getDefaultDatasource: () => {
-        // Ignore users table
-        const validTables = get(tables).list.filter(x => x._id !== "ta_users")
-
-        // Try to use their own internal table first
-        let table = validTables.find(table => {
-          return (
-            table.sourceId !== BUDIBASE_INTERNAL_DB_ID &&
-            table.sourceType === DB_TYPE_INTERNAL
-          )
-        })
-        if (table) {
-          return table
-        }
-
-        // Then try sample data
-        table = validTables.find(table => {
-          return (
-            table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
-            table.sourceType === DB_TYPE_INTERNAL
-          )
-        })
-        if (table) {
-          return table
-        }
-
-        // Finally try an external table
-        return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL)
-      },
-      migrateSettings: enrichedComponent => {
-        const componentPrefix = "@budibase/standard-components"
-        let migrated = false
-
-        if (enrichedComponent?._component == `${componentPrefix}/formblock`) {
-          // Use default config if the 'buttons' prop has never been initialised
-          if (!("buttons" in enrichedComponent)) {
-            enrichedComponent["buttons"] =
-              Utils.buildFormBlockButtonConfig(enrichedComponent)
-            migrated = true
-          } else if (enrichedComponent["buttons"] == null) {
-            // Ignore legacy config if 'buttons' has been reset by 'resetOn'
-            const { _id, actionType, dataSource } = enrichedComponent
-            enrichedComponent["buttons"] = Utils.buildFormBlockButtonConfig({
-              _id,
-              actionType,
-              dataSource,
-            })
-            migrated = true
-          }
-
-          // Ensure existing Formblocks position their buttons at the top.
-          if (!("buttonPosition" in enrichedComponent)) {
-            enrichedComponent["buttonPosition"] = "top"
-            migrated = true
-          }
-        }
-
-        return migrated
-      },
-      enrichEmptySettings: (component, opts) => {
-        if (!component?._component) {
-          return
-        }
-        const defaultDS = store.actions.components.getDefaultDatasource()
-        const settings = getComponentSettings(component._component)
-        const { parent, screen, useDefaultValues } = opts || {}
-        const treeId = parent?._id || component._id
-        if (!screen) {
-          return
-        }
-        settings.forEach(setting => {
-          const value = component[setting.key]
-
-          // Fill empty settings
-          if (value == null || value === "") {
-            if (setting.type === "multifield" && setting.selectAllFields) {
-              // Select all schema fields where required
-              component[setting.key] = Object.keys(defaultDS?.schema || {})
-            } else if (
-              (setting.type === "dataSource" || setting.type === "table") &&
-              defaultDS
-            ) {
-              // Select default datasource where required
-              component[setting.key] = {
-                label: defaultDS.name,
-                tableId: defaultDS._id,
-                resourceId: defaultDS._id,
-                type: "table",
-              }
-            } else if (setting.type === "dataProvider") {
-              // Pick closest data provider where required
-              const path = findComponentPath(screen.props, treeId)
-              const providers = path.filter(component =>
-                component._component?.endsWith("/dataprovider")
-              )
-              if (providers.length) {
-                const id = providers[providers.length - 1]?._id
-                component[setting.key] = `{{ literal ${safe(id)} }}`
-              }
-            } else if (setting.type.startsWith("field/")) {
-              // Autofill form field names
-              // Get all available field names in this form schema
-              let fieldOptions = getComponentFieldOptions(
-                screen.props,
-                treeId,
-                setting.type,
-                false
-              )
-
-              // Get all currently used fields
-              const form = findClosestMatchingComponent(
-                screen.props,
-                treeId,
-                x => x._component === "@budibase/standard-components/form"
-              )
-              const usedFields = Object.keys(buildFormSchema(form) || {})
-
-              // Filter out already used fields
-              fieldOptions = fieldOptions.filter(x => !usedFields.includes(x))
-
-              // Set field name and also assume we have a label setting
-              if (fieldOptions[0]) {
-                component[setting.key] = fieldOptions[0]
-                component.label = fieldOptions[0]
-              }
-            } else if (useDefaultValues && setting.defaultValue !== undefined) {
-              // Use default value where required
-              component[setting.key] = setting.defaultValue
-            }
-          }
-          // Validate non-empty settings
-          else {
-            if (setting.type === "dataProvider") {
-              // Validate data provider exists, or else clear it
-              const providers = findAllMatchingComponents(
-                screen?.props,
-                component => component._component?.endsWith("/dataprovider")
-              )
-              // Validate non-empty values
-              const valid = providers?.some(dp => value.includes?.(dp._id))
-              if (!valid) {
-                if (providers.length) {
-                  const id = providers[providers.length - 1]?._id
-                  component[setting.key] = `{{ literal ${safe(id)} }}`
-                } else {
-                  delete component[setting.key]
-                }
-              }
-            }
-          }
-        })
-      },
-      createInstance: (componentName, presetProps, parent) => {
-        const definition = store.actions.components.getDefinition(componentName)
-        if (!definition) {
-          return null
-        }
-
-        // Find all existing components of this type so that we can give this
-        // component a unique name
-        const screen = get(selectedScreen).props
-        const otherComponents = findAllMatchingComponents(
-          screen,
-          x => x._component === definition.component && x._id !== screen._id
-        )
-        let name = definition.friendlyName || definition.name
-        name = `${name} ${otherComponents.length + 1}`
-
-        // Generate basic component structure
-        let instance = {
-          _id: Helpers.uuid(),
-          _component: definition.component,
-          _styles: {
-            normal: {},
-            hover: {},
-            active: {},
-          },
-          _instanceName: name,
-          ...presetProps,
-        }
-
-        // Enrich empty settings
-        store.actions.components.enrichEmptySettings(instance, {
-          parent,
-          screen: get(selectedScreen),
-          useDefaultValues: true,
-        })
-
-        // Migrate nested component settings
-        store.actions.components.migrateSettings(instance)
-
-        // Add any extra properties the component needs
-        let extras = {}
-        if (definition.hasChildren) {
-          extras._children = []
-        }
-        if (componentName.endsWith("/formstep")) {
-          const parentForm = findClosestMatchingComponent(
-            get(selectedScreen).props,
-            get(selectedComponent)._id,
-            component => component._component.endsWith("/form")
-          )
-          const formSteps = findAllMatchingComponents(parentForm, component =>
-            component._component.endsWith("/formstep")
-          )
-          extras.step = formSteps.length + 1
-          extras._instanceName = `Step ${formSteps.length + 1}`
-        }
-        return {
-          ...cloneDeep(instance),
-          ...extras,
-        }
-      },
-      create: async (componentName, presetProps, parent, index) => {
-        const state = get(store)
-        const componentInstance = store.actions.components.createInstance(
-          componentName,
-          presetProps,
-          parent
-        )
-        if (!componentInstance) {
-          return
-        }
-
-        // Insert in position if specified
-        if (parent && index != null) {
-          await store.actions.screens.patch(screen => {
-            let parentComponent = findComponent(screen.props, parent)
-            if (!parentComponent._children?.length) {
-              parentComponent._children = [componentInstance]
-            } else {
-              parentComponent._children.splice(index, 0, componentInstance)
-            }
-          })
-        }
-
-        // Otherwise we work out where this component should be inserted
-        else {
-          await store.actions.screens.patch(screen => {
-            // Find the selected component
-            let selectedComponentId = state.selectedComponentId
-            if (selectedComponentId.startsWith(`${screen._id}-`)) {
-              selectedComponentId = screen?.props._id
-            }
-            const currentComponent = findComponent(
-              screen.props,
-              selectedComponentId
-            )
-            if (!currentComponent) {
-              return false
-            }
-
-            // Find parent node to attach this component to
-            let parentComponent
-            if (currentComponent) {
-              // Use selected component as parent if one is selected
-              const definition = store.actions.components.getDefinition(
-                currentComponent._component
-              )
-              if (definition?.hasChildren) {
-                // Use selected component if it allows children
-                parentComponent = currentComponent
-              } else {
-                // Otherwise we need to use the parent of this component
-                parentComponent = findComponentParent(
-                  screen.props,
-                  currentComponent._id
-                )
-              }
-            } else {
-              // Use screen or layout if no component is selected
-              parentComponent = screen.props
-            }
-
-            // Attach new component
-            if (!parentComponent) {
-              return false
-            }
-            if (!parentComponent._children) {
-              parentComponent._children = []
-            }
-            parentComponent._children.push(componentInstance)
-          })
-        }
-
-        // Select new component
-        store.update(state => {
-          state.selectedComponentId = componentInstance._id
-          return state
-        })
-
-        // Log event
-        analytics.captureEvent(Events.COMPONENT_CREATED, {
-          name: componentInstance._component,
-        })
-
-        return componentInstance
-      },
-      patch: async (patchFn, componentId, screenId) => {
-        // Use selected component by default
-        if (!componentId || !screenId) {
-          const state = get(store)
-          componentId = componentId || state.selectedComponentId
-          screenId = screenId || state.selectedScreenId
-        }
-        if (!componentId || !screenId || !patchFn) {
-          return
-        }
-        const patchScreen = screen => {
-          // findComponent looks in the tree not comp.settings[0]
-          let component = findComponent(screen.props, componentId)
-          if (!component) {
-            return false
-          }
-
-          // Mutates the fetched component with updates
-          const patchResult = patchFn(component, screen)
-
-          // Mutates the component with any required settings updates
-          const migrated = store.actions.components.migrateSettings(component)
-
-          // Returning an explicit false signifies that we should skip this
-          // update. If we migrated something, ensure we never skip.
-          return migrated ? null : patchResult
-        }
-        await store.actions.screens.patch(patchScreen, screenId)
-      },
-      delete: async component => {
-        if (!component) {
-          return
-        }
-
-        // Determine the next component to select after deletion
-        const state = get(store)
-        let nextSelectedComponentId
-        if (state.selectedComponentId === component._id) {
-          nextSelectedComponentId = store.actions.components.getNext()
-          if (!nextSelectedComponentId) {
-            nextSelectedComponentId = store.actions.components.getPrevious()
-          }
-        }
-
-        // Patch screen
-        await store.actions.screens.patch(screen => {
-          // Check component exists
-          component = findComponent(screen.props, component._id)
-          if (!component) {
-            return false
-          }
-
-          // Check component has a valid parent
-          const parent = findComponentParent(screen.props, component._id)
-          if (!parent) {
-            return false
-          }
-          parent._children = parent._children.filter(
-            child => child._id !== component._id
-          )
-        })
-
-        // Update selected component if required
-        if (nextSelectedComponentId) {
-          store.update(state => {
-            state.selectedComponentId = nextSelectedComponentId
-            return state
-          })
-        }
-      },
-      copy: (component, cut = false, selectParent = true) => {
-        // Update store with copied component
-        store.update(state => {
-          state.componentToPaste = cloneDeep(component)
-          state.componentToPaste.isCut = cut
-          return state
-        })
-
-        // Select the parent if cutting
-        if (cut && selectParent) {
-          const screen = get(selectedScreen)
-          const parent = findComponentParent(screen?.props, component._id)
-          if (parent) {
-            store.update(state => {
-              state.selectedComponentId = parent._id
-              return state
-            })
-          }
-        }
-      },
-      paste: async (targetComponent, mode, targetScreen) => {
-        const state = get(store)
-        if (!state.componentToPaste) {
-          return
-        }
-        let newComponentId
-
-        // Remove copied component if cutting, regardless if pasting works
-        let componentToPaste = cloneDeep(state.componentToPaste)
-        if (componentToPaste.isCut) {
-          store.update(state => {
-            delete state.componentToPaste
-            return state
-          })
-        }
-
-        // Patch screen
-        const patch = screen => {
-          // Get up to date ref to target
-          targetComponent = findComponent(screen.props, targetComponent._id)
-          if (!targetComponent) {
-            return false
-          }
-          const cut = componentToPaste.isCut
-          const originalId = componentToPaste._id
-          delete componentToPaste.isCut
-
-          // Make new component unique if copying
-          if (!cut) {
-            componentToPaste = makeComponentUnique(componentToPaste)
-          }
-          newComponentId = componentToPaste._id
-
-          // Delete old component if cutting
-          if (cut) {
-            const parent = findComponentParent(screen.props, originalId)
-            if (parent?._children) {
-              parent._children = parent._children.filter(
-                component => component._id !== originalId
-              )
-            }
-          }
-
-          // Check inside is valid
-          if (mode === "inside") {
-            const definition = store.actions.components.getDefinition(
-              targetComponent._component
-            )
-            if (!definition.hasChildren) {
-              mode = "below"
-            }
-          }
-
-          // Paste new component
-          if (mode === "inside") {
-            // Paste inside target component if chosen
-            if (!targetComponent._children) {
-              targetComponent._children = []
-            }
-            targetComponent._children.push(componentToPaste)
-          } else {
-            // Otherwise paste in the correct order in the parent's children
-            const parent = findComponentParent(
-              screen.props,
-              targetComponent._id
-            )
-            if (!parent?._children) {
-              return false
-            }
-            const targetIndex = parent._children.findIndex(component => {
-              return component._id === targetComponent._id
-            })
-            const index = mode === "above" ? targetIndex : targetIndex + 1
-            parent._children.splice(index, 0, componentToPaste)
-          }
-        }
-        const targetScreenId = targetScreen?._id || state.selectedScreenId
-        await store.actions.screens.patch(patch, targetScreenId)
-
-        // Select the new component
-        store.update(state => {
-          state.selectedScreenId = targetScreenId
-          state.selectedComponentId = newComponentId
-          return state
-        })
-      },
-      getPrevious: () => {
-        const state = get(store)
-        const componentId = state.selectedComponentId
-        const screen = get(selectedScreen)
-        const parent = findComponentParent(screen.props, componentId)
-        const index = parent?._children.findIndex(x => x._id === componentId)
-        const componentTreeNodes = get(componentTreeNodesStore)
-
-        // Check for screen and navigation component edge cases
-        const screenComponentId = `${screen._id}-screen`
-        const navComponentId = `${screen._id}-navigation`
-        if (componentId === screenComponentId) {
-          return null
-        }
-        if (componentId === navComponentId) {
-          return screenComponentId
-        }
-        if (parent._id === screen.props._id && index === 0) {
-          return navComponentId
-        }
-
-        // If we have siblings above us, choose the sibling or a descendant
-        if (index > 0) {
-          // If sibling before us accepts children, select a descendant
-          const previousSibling = parent._children[index - 1]
-          if (
-            previousSibling._children?.length &&
-            componentTreeNodes[`nodeOpen-${previousSibling._id}`]
-          ) {
-            let target = previousSibling
-            while (
-              target._children?.length &&
-              componentTreeNodes[`nodeOpen-${target._id}`]
-            ) {
-              target = target._children[target._children.length - 1]
-            }
-            return target._id
-          }
-
-          // Otherwise just select sibling
-          return previousSibling._id
-        }
-
-        // If no siblings above us, select the parent
-        return parent._id
-      },
-      getNext: () => {
-        const state = get(store)
-        const component = get(selectedComponent)
-        const componentId = component?._id
-        const screen = get(selectedScreen)
-        const parent = findComponentParent(screen.props, componentId)
-        const index = parent?._children.findIndex(x => x._id === componentId)
-        const componentTreeNodes = get(componentTreeNodesStore)
-
-        // Check for screen and navigation component edge cases
-        const screenComponentId = `${screen._id}-screen`
-        const navComponentId = `${screen._id}-navigation`
-        if (state.selectedComponentId === screenComponentId) {
-          return navComponentId
-        }
-
-        // If we have children, select first child
-        if (
-          component._children?.length &&
-          (state.selectedComponentId === navComponentId ||
-            componentTreeNodes[`nodeOpen-${component._id}`])
-        ) {
-          return component._children[0]._id
-        } else if (!parent) {
-          return null
-        }
-
-        // Otherwise select the next sibling if we have one
-        if (index < parent._children.length - 1) {
-          const nextSibling = parent._children[index + 1]
-          return nextSibling._id
-        }
-
-        // Last child, select our parents next sibling
-        let target = parent
-        let targetParent = findComponentParent(screen.props, target._id)
-        let targetIndex = targetParent?._children.findIndex(
-          child => child._id === target._id
-        )
-        while (
-          targetParent != null &&
-          targetIndex === targetParent._children?.length - 1
-        ) {
-          target = targetParent
-          targetParent = findComponentParent(screen.props, target._id)
-          targetIndex = targetParent?._children.findIndex(
-            child => child._id === target._id
-          )
-        }
-        if (targetParent) {
-          return targetParent._children[targetIndex + 1]._id
-        } else {
-          return null
-        }
-      },
-      selectPrevious: () => {
-        const previousId = store.actions.components.getPrevious()
-        if (previousId) {
-          store.update(state => {
-            state.selectedComponentId = previousId
-            return state
-          })
-        }
-      },
-      selectNext: () => {
-        const nextId = store.actions.components.getNext()
-        if (nextId) {
-          store.update(state => {
-            state.selectedComponentId = nextId
-            return state
-          })
-        }
-      },
-      moveUp: async component => {
-        await store.actions.screens.patch(screen => {
-          const componentId = component?._id
-          const parent = findComponentParent(screen.props, componentId)
-
-          // Check we aren't right at the top of the tree
-          const index = parent?._children.findIndex(x => x._id === componentId)
-          if (!parent || (index === 0 && parent._id === screen.props._id)) {
-            return
-          }
-
-          // Copy original component and remove it from the parent
-          const originalComponent = cloneDeep(parent._children[index])
-          parent._children = parent._children.filter(
-            component => component._id !== componentId
-          )
-
-          // If we have siblings above us, move up
-          if (index > 0) {
-            // If sibling before us accepts children, move to last child of
-            // sibling
-            const previousSibling = parent._children[index - 1]
-            const definition = store.actions.components.getDefinition(
-              previousSibling._component
-            )
-            if (definition.hasChildren) {
-              previousSibling._children.push(originalComponent)
-            }
-
-            // Otherwise just move component above sibling
-            else {
-              parent._children.splice(index - 1, 0, originalComponent)
-            }
-          }
-
-          // If no siblings above us, go above the parent as long as it isn't
-          // the screen
-          else if (parent._id !== screen.props._id) {
-            const grandParent = findComponentParent(screen.props, parent._id)
-            const parentIndex = grandParent._children.findIndex(
-              child => child._id === parent._id
-            )
-            grandParent._children.splice(parentIndex, 0, originalComponent)
-          }
-        })
-      },
-      moveDown: async component => {
-        await store.actions.screens.patch(screen => {
-          const componentId = component?._id
-          const parent = findComponentParent(screen.props, componentId)
-
-          // Sanity check parent is found
-          if (!parent?._children?.length) {
-            return false
-          }
-
-          // Check we aren't right at the bottom of the tree
-          const index = parent._children.findIndex(x => x._id === componentId)
-          if (
-            index === parent._children.length - 1 &&
-            parent._id === screen.props._id
-          ) {
-            return
-          }
-
-          // Copy the original component and remove from parent
-          const originalComponent = cloneDeep(parent._children[index])
-          parent._children = parent._children.filter(
-            component => component._id !== componentId
-          )
-
-          // Move below the next sibling if we are not the last sibling
-          if (index < parent._children.length) {
-            // If the next sibling has children, become the first child
-            const nextSibling = parent._children[index]
-            const definition = store.actions.components.getDefinition(
-              nextSibling._component
-            )
-            if (definition.hasChildren) {
-              nextSibling._children.splice(0, 0, originalComponent)
-            }
-
-            // Otherwise move below next sibling
-            else {
-              parent._children.splice(index + 1, 0, originalComponent)
-            }
-          }
-
-          // Last child, so move below our parent
-          else {
-            const grandParent = findComponentParent(screen.props, parent._id)
-            const parentIndex = grandParent._children.findIndex(
-              child => child._id === parent._id
-            )
-            grandParent._children.splice(parentIndex + 1, 0, originalComponent)
-          }
-        })
-      },
-      updateStyle: async (name, value) => {
-        await store.actions.components.patch(component => {
-          if (value == null || value === "") {
-            delete component._styles.normal[name]
-          } else {
-            component._styles.normal[name] = value
-          }
-        })
-      },
-      updateStyles: async (styles, id) => {
-        const patchFn = component => {
-          component._styles.normal = {
-            ...component._styles.normal,
-            ...styles,
-          }
-        }
-        await store.actions.components.patch(patchFn, id)
-      },
-      updateCustomStyle: async style => {
-        await store.actions.components.patch(component => {
-          component._styles.custom = style
-        })
-      },
-      updateConditions: async conditions => {
-        await store.actions.components.patch(component => {
-          component._conditions = conditions
-        })
-      },
-      updateSetting: async (name, value) => {
-        await store.actions.components.patch(
-          store.actions.components.updateComponentSetting(name, value)
-        )
-      },
-      updateComponentSetting: (name, value) => {
-        return component => {
-          if (!name || !component) {
-            return false
-          }
-          // Skip update if the value is the same
-          if (component[name] === value) {
-            return false
-          }
-
-          const settings = getComponentSettings(component._component)
-          const updatedSetting = settings.find(setting => setting.key === name)
-
-          // Reset dependent fields
-          settings.forEach(setting => {
-            const needsReset =
-              name === setting.resetOn ||
-              (Array.isArray(setting.resetOn) && setting.resetOn.includes(name))
-            if (needsReset) {
-              component[setting.key] = setting.defaultValue || null
-            }
-          })
-
-          if (
-            updatedSetting?.type === "dataSource" ||
-            updatedSetting?.type === "table"
-          ) {
-            const { schema } = getSchemaForDatasource(null, value)
-            const columnNames = Object.keys(schema || {})
-            const multifieldKeysToSelectAll = settings
-              .filter(setting => {
-                return setting.type === "multifield" && setting.selectAllFields
-              })
-              .map(setting => setting.key)
-
-            multifieldKeysToSelectAll.forEach(key => {
-              component[key] = columnNames
-            })
-          }
-          component[name] = value
-          return true
-        }
-      },
-      requestEjectBlock: componentId => {
-        store.actions.preview.sendEvent("eject-block", componentId)
-      },
-      handleEjectBlock: async (componentId, ejectedDefinition) => {
-        let nextSelectedComponentId
-        await store.actions.screens.patch(screen => {
-          const block = findComponent(screen.props, componentId)
-          const parent = findComponentParent(screen.props, componentId)
-
-          // Sanity check
-          if (!block || !parent?._children?.length) {
-            return false
-          }
-
-          // Log event
-          analytics.captureEvent(Events.BLOCK_EJECTED, {
-            block: block._component,
-          })
-
-          // Attach block children back into ejected definition, using the
-          // _containsSlot flag to know where to insert them
-          const slotContainer = findAllMatchingComponents(
-            ejectedDefinition,
-            x => x._containsSlot
-          )[0]
-          if (slotContainer) {
-            delete slotContainer._containsSlot
-            slotContainer._children = [
-              ...(slotContainer._children || []),
-              ...(block._children || []),
-            ]
-          }
-
-          // Replace block with ejected definition
-          ejectedDefinition = makeComponentUnique(ejectedDefinition)
-          const index = parent._children.findIndex(x => x._id === componentId)
-          parent._children[index] = ejectedDefinition
-          nextSelectedComponentId = ejectedDefinition._id
-        })
-
-        // Select new root component
-        if (nextSelectedComponentId) {
-          store.update(state => {
-            state.selectedComponentId = nextSelectedComponentId
-            return state
-          })
-        }
-      },
-      addParent: async (componentId, parentType) => {
-        if (!componentId || !parentType) {
-          return
-        }
-
-        // Create new parent instance
-        const newParentDefinition = store.actions.components.createInstance(
-          parentType,
-          null,
-          parent
-        )
-        if (!newParentDefinition) {
-          return
-        }
-
-        // Replace component with a version wrapped in a new parent
-        await store.actions.screens.patch(screen => {
-          // Get this component definition and parent definition
-          let definition = findComponent(screen.props, componentId)
-          let oldParentDefinition = findComponentParent(
-            screen.props,
-            componentId
-          )
-          if (!definition || !oldParentDefinition) {
-            return false
-          }
-
-          // Replace component with parent
-          const index = oldParentDefinition._children.findIndex(
-            component => component._id === componentId
-          )
-          if (index === -1) {
-            return false
-          }
-          oldParentDefinition._children[index] = {
-            ...newParentDefinition,
-            _children: [definition],
-          }
-        })
-
-        // Select the new parent
-        store.update(state => {
-          state.selectedComponentId = newParentDefinition._id
-          return state
-        })
-      },
-    },
-    links: {
-      save: async (url, title) => {
-        const navigation = get(store).navigation
-        let links = [...(navigation?.links ?? [])]
-
-        // Skip if we have an identical link
-        if (links.find(link => link.url === url && link.text === title)) {
-          return
-        }
-
-        links.push({
-          text: title,
-          url,
-        })
-        await store.actions.navigation.save({
-          ...navigation,
-          links: [...links],
-        })
-      },
-      delete: async urls => {
-        const navigation = get(store).navigation
-        let links = navigation?.links
-        if (!links?.length) {
-          return
-        }
-
-        // Filter out the URLs to delete
-        urls = Array.isArray(urls) ? urls : [urls]
-        links = links.filter(link => !urls.includes(link.url))
-
-        await store.actions.navigation.save({
-          ...navigation,
-          links,
-        })
-      },
-    },
-    settings: {
-      highlight: key => {
-        store.update(state => ({
-          ...state,
-          highlightedSettingKey: key,
-        }))
-      },
-      propertyFocus: key => {
-        store.update(state => ({
-          ...state,
-          propertyFocus: key,
-        }))
-      },
-    },
-    dnd: {
-      start: component => {
-        store.actions.preview.sendEvent("dragging-new-component", {
-          dragging: true,
-          component,
-        })
-      },
-      stop: () => {
-        store.actions.preview.sendEvent("dragging-new-component", {
-          dragging: false,
-        })
-      },
-    },
-    websocket: {
-      selectResource: id => {
-        websocket.emit(BuilderSocketEvent.SelectResource, {
-          resourceId: id,
-        })
-      },
-    },
-    metadata: {
-      replace: metadata => {
-        store.update(state => ({
-          ...state,
-          ...metadata,
-        }))
-      },
-    },
-  }
-
-  return store
-}

From dde8f778774ab756edb51e1c4e2a208e80b70629 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 15 Feb 2024 14:48:47 +0000
Subject: [PATCH 20/34] Type workerRequests.ts

---
 .../src/logging/correlation/correlation.ts    | 13 +++-
 .../server/src/utilities/workerRequests.ts    | 78 ++++++++++++++-----
 2 files changed, 70 insertions(+), 21 deletions(-)

diff --git a/packages/backend-core/src/logging/correlation/correlation.ts b/packages/backend-core/src/logging/correlation/correlation.ts
index 0498bd43d5..2ee02c16a9 100644
--- a/packages/backend-core/src/logging/correlation/correlation.ts
+++ b/packages/backend-core/src/logging/correlation/correlation.ts
@@ -1,10 +1,19 @@
+import { HeaderInit, Headers } from "node-fetch"
 import { Header } from "../../constants"
 
 const correlator = require("correlation-id")
 
-export const setHeader = (headers: any) => {
+export const setHeader = (headers: HeaderInit) => {
   const correlationId = correlator.getId()
-  if (correlationId) {
+  if (!correlationId) {
+    return
+  }
+
+  if (headers instanceof Headers) {
+    headers.set(Header.CORRELATION_ID, correlationId)
+  } else if (Array.isArray(headers)) {
+    headers.push([Header.CORRELATION_ID, correlationId])
+  } else {
     headers[Header.CORRELATION_ID] = correlationId
   }
 }
diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 5612084216..d3c5cad161 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -1,4 +1,10 @@
-import { Response, default as fetch } from "node-fetch"
+import {
+  Response,
+  default as fetch,
+  type RequestInit,
+  Headers,
+  HeadersInit,
+} from "node-fetch"
 import env from "../environment"
 import { checkSlashesInUrl } from "./index"
 import {
@@ -10,29 +16,61 @@ import {
 } from "@budibase/backend-core"
 import { Ctx, User, EmailInvite } from "@budibase/types"
 
-export function request(ctx?: Ctx, request?: any) {
-  if (!request.headers) {
-    request.headers = {}
+function ensureHeadersIsObject(headers: HeadersInit | undefined): Headers {
+  if (headers instanceof Headers) {
+    return headers
   }
 
+  const headersObj = new Headers()
+  if (headers === undefined) {
+    return headersObj
+  }
+
+  if (Array.isArray(headers)) {
+    for (const [key, value] of headers) {
+      headersObj.append(key, value)
+    }
+  } else {
+    for (const key in headers) {
+      headersObj.append(key, headers[key])
+    }
+  }
+  return headersObj
+}
+
+export function request(request: RequestInit & { ctx?: Ctx }): RequestInit {
+  const ctx = request.ctx
+  request.headers = ensureHeadersIsObject(request.headers)
+
   if (!ctx) {
-    request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
+    if (coreEnv.INTERNAL_API_KEY) {
+      request.headers.set(constants.Header.API_KEY, coreEnv.INTERNAL_API_KEY)
+    }
   } else if (ctx.headers) {
     // copy all Budibase utilised headers over - copying everything can have
     // side effects like requests being rejected due to odd content types etc
     for (let header of Object.values(constants.Header)) {
-      if (ctx.headers[header]) {
-        request.headers[header] = ctx.headers[header]
+      const value = ctx.headers[header]
+      if (value === undefined) {
+        continue
+      }
+
+      if (Array.isArray(value)) {
+        for (let v of value) {
+          request.headers.append(header, v)
+        }
+      } else {
+        request.headers.set(header, value)
       }
     }
   }
 
   // apply tenancy if its available
   if (tenancy.isTenantIdSet()) {
-    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+    request.headers.set(constants.Header.TENANT_ID, tenancy.getTenantId())
   }
   if (request.body && Object.keys(request.body).length > 0) {
-    request.headers["Content-Type"] = "application/json"
+    request.headers.set("Content-Type", "application/json")
     request.body =
       typeof request.body === "object"
         ? JSON.stringify(request.body)
@@ -44,6 +82,7 @@ export function request(ctx?: Ctx, request?: any) {
   // add x-budibase-correlation-id header
   logging.correlation.setHeader(request.headers)
 
+  delete request.ctx
   return request
 }
 
@@ -93,9 +132,9 @@ export async function sendSmtpEmail({
   // tenant ID will be set in header
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
-    request(undefined, {
+    request({
       method: "POST",
-      body: {
+      body: JSON.stringify({
         email: to,
         from,
         contents,
@@ -105,7 +144,7 @@ export async function sendSmtpEmail({
         purpose: "custom",
         automation,
         invite,
-      },
+      }),
     })
   )
   return checkResponse(response, "send email")
@@ -115,7 +154,8 @@ export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {
   const prodAppId = dbCore.getProdAppID(appId)
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${prodAppId}`),
-    request(ctx, {
+    request({
+      ctx,
       method: "DELETE",
     })
   )
@@ -126,7 +166,7 @@ export async function allGlobalUsers(ctx: Ctx) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
     // we don't want to use API key when getting self
-    request(ctx, { method: "GET" })
+    request({ ctx, method: "GET" })
   )
   return checkResponse(response, "get users", { ctx })
 }
@@ -135,7 +175,7 @@ export async function saveGlobalUser(ctx: Ctx) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
     // we don't want to use API key when getting self
-    request(ctx, { method: "POST", body: ctx.request.body })
+    request({ ctx, method: "POST", body: ctx.request.body })
   )
   return checkResponse(response, "save user", { ctx })
 }
@@ -146,7 +186,7 @@ export async function deleteGlobalUser(ctx: Ctx) {
       env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
     ),
     // we don't want to use API key when getting self
-    request(ctx, { method: "DELETE" })
+    request({ ctx, method: "DELETE" })
   )
   return checkResponse(response, "delete user", { ctx })
 }
@@ -157,7 +197,7 @@ export async function readGlobalUser(ctx: Ctx): Promise<User> {
       env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
     ),
     // we don't want to use API key when getting self
-    request(ctx, { method: "GET" })
+    request({ ctx, method: "GET" })
   )
   return checkResponse(response, "get user", { ctx })
 }
@@ -167,7 +207,7 @@ export async function getChecklist(): Promise<{
 }> {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/configs/checklist"),
-    request(undefined, { method: "GET" })
+    request({ method: "GET" })
   )
   return checkResponse(response, "get checklist")
 }
@@ -175,7 +215,7 @@ export async function getChecklist(): Promise<{
 export async function generateApiKey(userId: string) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/self/api_key"),
-    request(undefined, { method: "POST", body: { userId } })
+    request({ method: "POST", body: JSON.stringify({ userId }) })
   )
   return checkResponse(response, "generate API key")
 }

From 1f4a254ec5449e2e04797ce3fa904cf14048ccde Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 15:47:56 +0000
Subject: [PATCH 21/34] Fix for integration test, make sure to carry auth
 headers over correctly.

---
 packages/server/src/utilities/workerRequests.ts | 9 +++++++++
 packages/shared-core/src/constants/api.ts       | 1 +
 2 files changed, 10 insertions(+)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 5612084216..b5c3bfcd4a 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -7,6 +7,7 @@ import {
   tenancy,
   logging,
   env as coreEnv,
+  utils,
 } from "@budibase/backend-core"
 import { Ctx, User, EmailInvite } from "@budibase/types"
 
@@ -25,6 +26,14 @@ export function request(ctx?: Ctx, request?: any) {
         request.headers[header] = ctx.headers[header]
       }
     }
+    // be specific about auth headers
+    const cookie = ctx.headers[constants.Header.COOKIE],
+      apiKey = ctx.headers[constants.Header.API_KEY]
+    if (cookie) {
+      request.headers[constants.Header.COOKIE] = cookie
+    } else if (apiKey) {
+      request.headers[constants.Header.API_KEY] = apiKey
+    }
   }
 
   // apply tenancy if its available
diff --git a/packages/shared-core/src/constants/api.ts b/packages/shared-core/src/constants/api.ts
index d6633649e6..f63849bf3d 100644
--- a/packages/shared-core/src/constants/api.ts
+++ b/packages/shared-core/src/constants/api.ts
@@ -16,4 +16,5 @@ export enum Header {
   CORRELATION_ID = "x-budibase-correlation-id",
   AUTHORIZATION = "authorization",
   MIGRATING_APP = "x-budibase-migrating-app",
+  COOKIE = "cookie",
 }

From d5932605b7a8bcdcdd203907303c7af1f58f1511 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 15:49:12 +0000
Subject: [PATCH 22/34] Fixing an issue where app routing information was
 requested before the app store was fully brought online, meaning that routing
 information wasn't always successfully retrieved.

---
 packages/builder/src/stores/builder/index.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js
index 36af7e7435..bac9fb8826 100644
--- a/packages/builder/src/stores/builder/index.js
+++ b/packages/builder/src/stores/builder/index.js
@@ -92,12 +92,13 @@ const resetBuilderHistory = () => {
 
 export const initialise = async pkg => {
   const { application } = pkg
+  // must be first operation to make sure subsequent requests have correct app ID
+  appStore.syncAppPackage(pkg)
   await Promise.all([
     appStore.syncAppRoutes(),
     componentStore.refreshDefinitions(application?.appId),
   ])
   builderStore.init(application)
-  appStore.syncAppPackage(pkg)
   navigationStore.syncAppNavigation(application?.navigation)
   themeStore.syncAppTheme(application)
   screenStore.syncAppScreens(pkg)

From e4b03308664f5d0054a7ca94fb55b9e89f92e8cf Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 15 Feb 2024 15:49:30 +0000
Subject: [PATCH 23/34] Simplify the typing of workerRequests.ts

---
 .../src/logging/correlation/correlation.ts    | 12 +---
 .../server/src/utilities/workerRequests.ts    | 70 +++++++++----------
 2 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/packages/backend-core/src/logging/correlation/correlation.ts b/packages/backend-core/src/logging/correlation/correlation.ts
index 2ee02c16a9..13cc7aff8f 100644
--- a/packages/backend-core/src/logging/correlation/correlation.ts
+++ b/packages/backend-core/src/logging/correlation/correlation.ts
@@ -1,21 +1,13 @@
-import { HeaderInit, Headers } from "node-fetch"
 import { Header } from "../../constants"
 
 const correlator = require("correlation-id")
 
-export const setHeader = (headers: HeaderInit) => {
+export const setHeader = (headers: Record<string, string>) => {
   const correlationId = correlator.getId()
   if (!correlationId) {
     return
   }
-
-  if (headers instanceof Headers) {
-    headers.set(Header.CORRELATION_ID, correlationId)
-  } else if (Array.isArray(headers)) {
-    headers.push([Header.CORRELATION_ID, correlationId])
-  } else {
-    headers[Header.CORRELATION_ID] = correlationId
-  }
+  headers[Header.CORRELATION_ID] = correlationId
 }
 
 export function getId() {
diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index d3c5cad161..ce43c1aa44 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -38,15 +38,24 @@ function ensureHeadersIsObject(headers: HeadersInit | undefined): Headers {
   return headersObj
 }
 
-export function request(request: RequestInit & { ctx?: Ctx }): RequestInit {
-  const ctx = request.ctx
-  request.headers = ensureHeadersIsObject(request.headers)
+interface Request {
+  ctx?: Ctx
+  method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH"
+  headers?: { [key: string]: string }
+  body?: { [key: string]: any }
+}
 
-  if (!ctx) {
-    if (coreEnv.INTERNAL_API_KEY) {
-      request.headers.set(constants.Header.API_KEY, coreEnv.INTERNAL_API_KEY)
-    }
-  } else if (ctx.headers) {
+export function createRequest(request: Request): RequestInit {
+  const headers: Record<string, string> = {}
+  const requestInit: RequestInit = {
+    method: request.method,
+  }
+
+  const ctx = request.ctx
+
+  if (!ctx && coreEnv.INTERNAL_API_KEY) {
+    headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
+  } else if (ctx && ctx.headers) {
     // copy all Budibase utilised headers over - copying everything can have
     // side effects like requests being rejected due to odd content types etc
     for (let header of Object.values(constants.Header)) {
@@ -56,34 +65,25 @@ export function request(request: RequestInit & { ctx?: Ctx }): RequestInit {
       }
 
       if (Array.isArray(value)) {
-        for (let v of value) {
-          request.headers.append(header, v)
-        }
+        headers[header] = value[0]
       } else {
-        request.headers.set(header, value)
+        headers[header] = value
       }
     }
   }
 
   // apply tenancy if its available
   if (tenancy.isTenantIdSet()) {
-    request.headers.set(constants.Header.TENANT_ID, tenancy.getTenantId())
+    headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
   }
+
   if (request.body && Object.keys(request.body).length > 0) {
-    request.headers.set("Content-Type", "application/json")
-    request.body =
-      typeof request.body === "object"
-        ? JSON.stringify(request.body)
-        : request.body
-  } else {
-    delete request.body
+    headers["Content-Type"] = "application/json"
+    requestInit.body = JSON.stringify(request.body)
   }
 
-  // add x-budibase-correlation-id header
-  logging.correlation.setHeader(request.headers)
-
-  delete request.ctx
-  return request
+  logging.correlation.setHeader(headers)
+  return requestInit
 }
 
 async function checkResponse(
@@ -132,9 +132,9 @@ export async function sendSmtpEmail({
   // tenant ID will be set in header
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
-    request({
+    createRequest({
       method: "POST",
-      body: JSON.stringify({
+      body: {
         email: to,
         from,
         contents,
@@ -144,7 +144,7 @@ export async function sendSmtpEmail({
         purpose: "custom",
         automation,
         invite,
-      }),
+      },
     })
   )
   return checkResponse(response, "send email")
@@ -154,7 +154,7 @@ export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {
   const prodAppId = dbCore.getProdAppID(appId)
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${prodAppId}`),
-    request({
+    createRequest({
       ctx,
       method: "DELETE",
     })
@@ -166,7 +166,7 @@ export async function allGlobalUsers(ctx: Ctx) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
     // we don't want to use API key when getting self
-    request({ ctx, method: "GET" })
+    createRequest({ ctx, method: "GET" })
   )
   return checkResponse(response, "get users", { ctx })
 }
@@ -175,7 +175,7 @@ export async function saveGlobalUser(ctx: Ctx) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
     // we don't want to use API key when getting self
-    request({ ctx, method: "POST", body: ctx.request.body })
+    createRequest({ ctx, method: "POST", body: ctx.request.body })
   )
   return checkResponse(response, "save user", { ctx })
 }
@@ -186,7 +186,7 @@ export async function deleteGlobalUser(ctx: Ctx) {
       env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
     ),
     // we don't want to use API key when getting self
-    request({ ctx, method: "DELETE" })
+    createRequest({ ctx, method: "DELETE" })
   )
   return checkResponse(response, "delete user", { ctx })
 }
@@ -197,7 +197,7 @@ export async function readGlobalUser(ctx: Ctx): Promise<User> {
       env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
     ),
     // we don't want to use API key when getting self
-    request({ ctx, method: "GET" })
+    createRequest({ ctx, method: "GET" })
   )
   return checkResponse(response, "get user", { ctx })
 }
@@ -207,7 +207,7 @@ export async function getChecklist(): Promise<{
 }> {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/configs/checklist"),
-    request({ method: "GET" })
+    createRequest({ method: "GET" })
   )
   return checkResponse(response, "get checklist")
 }
@@ -215,7 +215,7 @@ export async function getChecklist(): Promise<{
 export async function generateApiKey(userId: string) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/self/api_key"),
-    request({ method: "POST", body: JSON.stringify({ userId }) })
+    createRequest({ method: "POST", body: { userId } })
   )
   return checkResponse(response, "generate API key")
 }

From b4669b32f00158e72b637cf13747c5b2a26618f6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 15 Feb 2024 15:52:06 +0000
Subject: [PATCH 24/34] Fix build.

---
 packages/server/src/api/controllers/dev.ts | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/dev.ts b/packages/server/src/api/controllers/dev.ts
index d8c26725f1..497da088c6 100644
--- a/packages/server/src/api/controllers/dev.ts
+++ b/packages/server/src/api/controllers/dev.ts
@@ -1,7 +1,7 @@
 import fetch from "node-fetch"
 import env from "../../environment"
 import { checkSlashesInUrl } from "../../utilities"
-import { request } from "../../utilities/workerRequests"
+import { createRequest } from "../../utilities/workerRequests"
 import { clearLock as redisClearLock } from "../../utilities/redis"
 import { DocumentType } from "../../db/utils"
 import {
@@ -13,14 +13,19 @@ import {
 } from "@budibase/backend-core"
 import { App } from "@budibase/types"
 
-async function redirect(ctx: any, method: string, path: string = "global") {
+async function redirect(
+  ctx: any,
+  method: "GET" | "POST" | "DELETE",
+  path: string = "global"
+) {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
     checkSlashesInUrl(
       `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
     ),
-    request(ctx, {
+    createRequest({
+      ctx,
       method,
       body: ctx.request.body,
     })

From c2c0108e4fd7f6595f7ddda547b46f713fa301c3 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 15 Feb 2024 16:12:47 +0000
Subject: [PATCH 25/34] Fix build (again).

---
 packages/server/src/utilities/workerRequests.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index acf7515784..def3ffe9cf 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -75,9 +75,13 @@ export function createRequest(request: Request): RequestInit {
     const cookie = ctx.headers[constants.Header.COOKIE],
       apiKey = ctx.headers[constants.Header.API_KEY]
     if (cookie) {
-      request.headers[constants.Header.COOKIE] = cookie
+      headers[constants.Header.COOKIE] = cookie
     } else if (apiKey) {
-      request.headers[constants.Header.API_KEY] = apiKey
+      if (Array.isArray(apiKey)) {
+        headers[constants.Header.API_KEY] = apiKey[0]
+      } else {
+        headers[constants.Header.API_KEY] = apiKey
+      }
     }
   }
 

From 32815d8d9bec6c1efe58f4fcdcd226b1e47364ce Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 15 Feb 2024 16:28:59 +0000
Subject: [PATCH 26/34] Quick readability enhancement.

---
 packages/server/src/utilities/workerRequests.ts | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index def3ffe9cf..054514ab59 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -64,12 +64,7 @@ export function createRequest(request: Request): RequestInit {
       if (value === undefined) {
         continue
       }
-
-      if (Array.isArray(value)) {
-        headers[header] = value[0]
-      } else {
-        headers[header] = value
-      }
+      headers[header] = Array.isArray(value) ? value[0] : value
     }
     // be specific about auth headers
     const cookie = ctx.headers[constants.Header.COOKIE],
@@ -77,11 +72,9 @@ export function createRequest(request: Request): RequestInit {
     if (cookie) {
       headers[constants.Header.COOKIE] = cookie
     } else if (apiKey) {
-      if (Array.isArray(apiKey)) {
-        headers[constants.Header.API_KEY] = apiKey[0]
-      } else {
-        headers[constants.Header.API_KEY] = apiKey
-      }
+      headers[constants.Header.API_KEY] = Array.isArray(apiKey)
+        ? apiKey[0]
+        : apiKey
     }
   }
 

From dd4ea4be95c733636f42ed8161f395c2217699f8 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 15 Feb 2024 16:44:19 +0000
Subject: [PATCH 27/34] Nothing to see here, carry on.

---
 .../server/src/utilities/workerRequests.ts    | 23 +------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts
index 054514ab59..91340c8d78 100644
--- a/packages/server/src/utilities/workerRequests.ts
+++ b/packages/server/src/utilities/workerRequests.ts
@@ -17,28 +17,6 @@ import {
 } from "@budibase/backend-core"
 import { Ctx, User, EmailInvite } from "@budibase/types"
 
-function ensureHeadersIsObject(headers: HeadersInit | undefined): Headers {
-  if (headers instanceof Headers) {
-    return headers
-  }
-
-  const headersObj = new Headers()
-  if (headers === undefined) {
-    return headersObj
-  }
-
-  if (Array.isArray(headers)) {
-    for (const [key, value] of headers) {
-      headersObj.append(key, value)
-    }
-  } else {
-    for (const key in headers) {
-      headersObj.append(key, headers[key])
-    }
-  }
-  return headersObj
-}
-
 interface Request {
   ctx?: Ctx
   method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH"
@@ -89,6 +67,7 @@ export function createRequest(request: Request): RequestInit {
   }
 
   logging.correlation.setHeader(headers)
+  requestInit.headers = headers
   return requestInit
 }
 

From 5a29bb7ebafce2ee734c3ba972a0372ef01017e1 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 15 Feb 2024 17:28:54 +0000
Subject: [PATCH 28/34] Bump version to 2.19.5

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 4940336b29..596fb434bc 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.19.4",
+  "version": "2.19.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 9a9fa3c7165bf95ceec9c5226c53d48bc8d7d42f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 16 Feb 2024 10:46:36 +0000
Subject: [PATCH 29/34] Updating CouchDB with the Budi CLI, making sure the
 service is always up to date, as well as removing pull policy.

---
 hosting/docker-compose.yaml        | 1 -
 packages/cli/src/hosting/update.ts | 7 ++++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index 36b88466fe..a72b36aef1 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -98,7 +98,6 @@ services:
   couchdb-service:
     restart: unless-stopped
     image: budibase/couchdb
-    pull_policy: always
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
diff --git a/packages/cli/src/hosting/update.ts b/packages/cli/src/hosting/update.ts
index ca0ecce615..cf70140224 100644
--- a/packages/cli/src/hosting/update.ts
+++ b/packages/cli/src/hosting/update.ts
@@ -13,7 +13,12 @@ import { COMPOSE_PATH } from "./makeFiles"
 import { info, success } from "../utils"
 import { start } from "./start"
 
-const BB_COMPOSE_SERVICES = ["app-service", "worker-service", "proxy-service"]
+const BB_COMPOSE_SERVICES = [
+  "app-service",
+  "worker-service",
+  "proxy-service",
+  "couchdb-service",
+]
 const BB_SINGLE_SERVICE = ["budibase"]
 
 export async function update() {

From 364579acc444952885e3e5a8a09f3f653a77cd28 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 16 Feb 2024 15:41:36 +0000
Subject: [PATCH 30/34] Fixing an issue when invalid version is supplied (not
 semver) shouldn't crash server.

---
 packages/backend-core/src/installation.ts | 45 ++++++++++++++---------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/packages/backend-core/src/installation.ts b/packages/backend-core/src/installation.ts
index ca35b926fb..83166880cc 100644
--- a/packages/backend-core/src/installation.ts
+++ b/packages/backend-core/src/installation.ts
@@ -6,6 +6,7 @@ import * as context from "./context"
 import semver from "semver"
 import { bustCache, withCache, TTL, CacheKey } from "./cache/generic"
 import environment from "./environment"
+import { logAlert } from "./logging"
 
 export const getInstall = async (): Promise<Installation> => {
   return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
@@ -80,27 +81,35 @@ export const checkInstallVersion = async (): Promise<void> => {
   const currentVersion = install.version
   const newVersion = environment.VERSION
 
-  if (currentVersion !== newVersion) {
-    const isUpgrade = semver.gt(newVersion, currentVersion)
-    const isDowngrade = semver.lt(newVersion, currentVersion)
+  try {
+    if (currentVersion !== newVersion) {
+      const isUpgrade = semver.gt(newVersion, currentVersion)
+      const isDowngrade = semver.lt(newVersion, currentVersion)
 
-    const success = await updateVersion(newVersion)
+      const success = await updateVersion(newVersion)
 
-    if (success) {
-      await context.doInIdentityContext(
-        {
-          _id: install.installId,
-          type: IdentityType.INSTALLATION,
-        },
-        async () => {
-          if (isUpgrade) {
-            await events.installation.upgraded(currentVersion, newVersion)
-          } else if (isDowngrade) {
-            await events.installation.downgraded(currentVersion, newVersion)
+      if (success) {
+        await context.doInIdentityContext(
+          {
+            _id: install.installId,
+            type: IdentityType.INSTALLATION,
+          },
+          async () => {
+            if (isUpgrade) {
+              await events.installation.upgraded(currentVersion, newVersion)
+            } else if (isDowngrade) {
+              await events.installation.downgraded(currentVersion, newVersion)
+            }
           }
-        }
-      )
-      await events.identification.identifyInstallationGroup(install.installId)
+        )
+        await events.identification.identifyInstallationGroup(install.installId)
+      }
+    }
+  } catch (err: any) {
+    if (err?.message?.includes("Invalid Version")) {
+      logAlert(`Invalid version "${newVersion}" - is it semver?`)
+    } else {
+      logAlert("Failed to retrieve version", err)
     }
   }
 }

From e0d8362009b5479cdd2b6d67dda79754349ddc92 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 16 Feb 2024 15:57:27 +0000
Subject: [PATCH 31/34] Make a better MySQL healthcheck.

---
 .../src/integrations/tests/utils/mongodb.ts   |  4 +-
 .../src/integrations/tests/utils/mysql.ts     | 39 ++++++++++++-------
 .../src/integrations/tests/utils/postgres.ts  |  4 +-
 3 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/packages/server/src/integrations/tests/utils/mongodb.ts b/packages/server/src/integrations/tests/utils/mongodb.ts
index ebd76e4baf..0baafc6276 100644
--- a/packages/server/src/integrations/tests/utils/mongodb.ts
+++ b/packages/server/src/integrations/tests/utils/mongodb.ts
@@ -11,7 +11,9 @@ export async function start(): Promise<StartedTestContainer> {
       MONGO_INITDB_ROOT_PASSWORD: "password",
     })
     .withWaitStrategy(
-      Wait.forSuccessfulCommand(`mongosh --eval "db.version()"`)
+      Wait.forSuccessfulCommand(
+        `mongosh --eval "db.version()"`
+      ).withStartupTimeout(10000)
     )
     .start()
 }
diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts
index 474819287e..8d5563c6dd 100644
--- a/packages/server/src/integrations/tests/utils/mysql.ts
+++ b/packages/server/src/integrations/tests/utils/mysql.ts
@@ -1,26 +1,37 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import {
+  AbstractWaitStrategy,
+  WaitStrategy,
+} from "testcontainers/build/wait-strategies/wait-strategy"
 
 let container: StartedTestContainer | undefined
 
+class MySQLWaitStrategy extends AbstractWaitStrategy {
+  async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
+    // Because MySQL first starts itself up, runs an init script, then restarts,
+    // it's possible for the mysqladmin ping to succeed early and then tests to
+    // run against a MySQL that's mid-restart and fail. To get around this, we
+    // wait for logs and then do a ping check.
+
+    const logs = Wait.forLogMessage(
+      "/usr/sbin/mysqld: ready for connections",
+      2
+    )
+    await logs.waitUntilReady(container, boundPorts, startTime)
+
+    const command = Wait.forSuccessfulCommand(
+      `mysqladmin ping -h localhost -P 3306 -u root -ppassword`
+    )
+    await command.waitUntilReady(container)
+  }
+}
+
 export async function start(): Promise<StartedTestContainer> {
   return await new GenericContainer("mysql:8.3")
     .withExposedPorts(3306)
     .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        // Because MySQL first starts itself up, runs an init script, then restarts,
-        // it's possible for the mysqladmin ping to succeed early and then tests to
-        // run against a MySQL that's mid-restart and fail. To avoid this, we run
-        // the ping command three times with a small delay between each.
-        `
-          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
-          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
-          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
-          mysqladmin ping -h localhost -P 3306 -u root -ppassword
-        `
-      )
-    )
+    .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
     .start()
 }
 
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 4bf42c7f88..82a62e3916 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -8,7 +8,9 @@ export async function start(): Promise<StartedTestContainer> {
     .withExposedPorts(5432)
     .withEnvironment({ POSTGRES_PASSWORD: "password" })
     .withWaitStrategy(
-      Wait.forSuccessfulCommand("pg_isready -h localhost -p 5432")
+      Wait.forSuccessfulCommand(
+        "pg_isready -h localhost -p 5432"
+      ).withStartupTimeout(10000)
     )
     .start()
 }

From 7211cf4bbf80d563a3b284eaaf6ef25d9594ba89 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 16 Feb 2024 16:11:55 +0000
Subject: [PATCH 32/34] Remove unused import.

---
 packages/server/src/integrations/tests/utils/mysql.ts | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts
index 8d5563c6dd..5e51478998 100644
--- a/packages/server/src/integrations/tests/utils/mysql.ts
+++ b/packages/server/src/integrations/tests/utils/mysql.ts
@@ -1,9 +1,6 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
-import {
-  AbstractWaitStrategy,
-  WaitStrategy,
-} from "testcontainers/build/wait-strategies/wait-strategy"
+import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
 
 let container: StartedTestContainer | undefined
 

From c12e5fd196a38696a3a8a3f0a778e966caef5b97 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Mon, 19 Feb 2024 09:13:03 +0000
Subject: [PATCH 33/34] Fix array type missing from query schema selector
 (#12772)

* Tidy MongoDB aggregation pipeline view

* Remove unused code

* WIP

* Add bindings for bindings drawer

* Is not external table if it's a query

* Add QueryArrayFetch

* Bug fix

* JavaScript is the worst

* refactor

* Add array label to query schema

* Remove console log

* type fix

* Don't include Array in SchemaTypeOptions, but show label

* Fix bindings

* refactor

* Rename isObject to hasSchema

* WIP

* Typing WIP

* Type not Types

* Unused import

* type fix

* Handle json array subtype

* Support queryarray datasource type

* refactor

* yarn lock

* update account portal

---------

Co-authored-by: Sam Rose <hello@samwho.dev>
---
 .../src/events/publishers/query.ts            |   5 +-
 .../DataSourceSelect/DataSourceSelect.svelte  |  11 +-
 .../integration/KeyValueBuilder.svelte        |  14 +-
 .../components/integration/QueryViewer.svelte |   5 +-
 .../src/components/integration/index.svelte   |   2 -
 .../builder/src/constants/backend/index.js    |   1 +
 packages/builder/src/dataBinding.js           |  15 ++-
 .../src/components/app/forms/Form.svelte      |   2 +-
 packages/client/src/utils/schema.js           |   2 +
 .../src/fetch/QueryArrayFetch.js              |  25 ++++
 packages/frontend-core/src/fetch/index.js     |   2 +
 packages/frontend-core/src/utils/json.js      |  32 +++++
 .../server/src/api/controllers/query/index.ts | 126 +++++++++++-------
 packages/shared-core/src/utils.ts             |  10 ++
 packages/types/src/documents/app/query.ts     |  13 ++
 .../src/documents/app/table/constants.ts      |   4 +
 .../types/src/documents/app/table/schema.ts   |   7 +
 yarn.lock                                     |   8 +-
 18 files changed, 219 insertions(+), 65 deletions(-)
 create mode 100644 packages/frontend-core/src/fetch/QueryArrayFetch.js

diff --git a/packages/backend-core/src/events/publishers/query.ts b/packages/backend-core/src/events/publishers/query.ts
index 7d28129cf6..48603257d2 100644
--- a/packages/backend-core/src/events/publishers/query.ts
+++ b/packages/backend-core/src/events/publishers/query.ts
@@ -3,6 +3,7 @@ import {
   Event,
   Datasource,
   Query,
+  QueryPreview,
   QueryCreatedEvent,
   QueryUpdatedEvent,
   QueryDeletedEvent,
@@ -68,9 +69,9 @@ const run = async (count: number, timestamp?: string | number) => {
   await publishEvent(Event.QUERIES_RUN, properties, timestamp)
 }
 
-const previewed = async (datasource: Datasource, query: Query) => {
+const previewed = async (datasource: Datasource, query: QueryPreview) => {
   const properties: QueryPreviewedEvent = {
-    queryId: query._id,
+    queryId: query.queryId,
     datasourceId: datasource._id as string,
     source: datasource.source,
     queryVerb: query.queryVerb,
diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte
index 14cbc973a1..0e01c264fc 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte
@@ -127,10 +127,14 @@
       }
     })
   $: jsonArrays = bindings
-    .filter(x => x.fieldSchema?.type === "jsonarray")
+    .filter(
+      x =>
+        x.fieldSchema?.type === "jsonarray" ||
+        (x.fieldSchema?.type === "json" && x.fieldSchema?.subtype === "array")
+    )
     .map(binding => {
       const { providerId, readableBinding, runtimeBinding, tableId } = binding
-      const { name, type, prefixKeys } = binding.fieldSchema
+      const { name, type, prefixKeys, subtype } = binding.fieldSchema
       return {
         providerId,
         label: readableBinding,
@@ -138,7 +142,8 @@
         fieldType: type,
         tableId,
         prefixKeys,
-        type: "jsonarray",
+        type: type === "jsonarray" ? "jsonarray" : "queryarray",
+        subtype,
         value: `{{ literal ${runtimeBinding} }}`,
       }
     })
diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte
index 8dac07bcec..e71090f613 100644
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@@ -85,6 +85,16 @@
     activity = newActivity
     dispatch("change", fields)
   }
+
+  function isJsonArray(value) {
+    if (!value || typeof value === "string") {
+      return false
+    }
+    if (value.type === "array") {
+      return true
+    }
+    return value.type === "json" && value.subtype === "array"
+  }
 </script>
 
 <!-- Builds Objects with Key Value Pairs. Useful for building things like Request Headers. -->
@@ -112,7 +122,9 @@
         bind:value={field.name}
         on:blur={changed}
       />
-      {#if options}
+      {#if isJsonArray(field.value)}
+        <Select readonly={true} value="Array" options={["Array"]} />
+      {:else if options}
         <Select
           bind:value={field.value}
           {compare}
diff --git a/packages/builder/src/components/integration/QueryViewer.svelte b/packages/builder/src/components/integration/QueryViewer.svelte
index 44a3dd33be..b3f8ddb09b 100644
--- a/packages/builder/src/components/integration/QueryViewer.svelte
+++ b/packages/builder/src/components/integration/QueryViewer.svelte
@@ -40,6 +40,7 @@
   let schemaType
 
   let autoSchema = {}
+  let nestedSchemaFields = {}
   let rows = []
   let keys = {}
 
@@ -83,13 +84,14 @@
         return
       }
 
+      nestedSchemaFields = response.nestedSchemaFields
+
       if (Object.keys(newQuery.schema).length === 0) {
         // Assign this to a variable instead of directly to the newQuery.schema so that a user
         // can change the table they're querying and have the schema update until they first
         // edit it
         autoSchema = response.schema
       }
-
       rows = response.rows
 
       notifications.success("Query executed successfully")
@@ -120,6 +122,7 @@
           Object.keys(newQuery.schema).length === 0
             ? autoSchema
             : newQuery.schema,
+        nestedSchemaFields,
       })
 
       notifications.success("Query saved successfully")
diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte
index 453179f25b..bbbd4c3b50 100644
--- a/packages/builder/src/components/integration/index.svelte
+++ b/packages/builder/src/components/integration/index.svelte
@@ -5,7 +5,6 @@
     Label,
     Input,
     Select,
-    Divider,
     Layout,
     Icon,
     Button,
@@ -124,7 +123,6 @@
         {#each query.fields.steps ?? [] as step, index}
           <div class="block">
             <div class="subblock">
-              <Divider noMargin />
               <div class="blockSection">
                 <div class="block-options">
                   Stage {index + 1}
diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js
index eb47ac97fe..f1e3e1e2c2 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@@ -310,6 +310,7 @@ export const BannedSearchTypes = [
   "formula",
   "json",
   "jsonarray",
+  "queryarray",
 ]
 
 export const DatasourceTypes = {
diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js
index 8c71631b09..0442a67da9 100644
--- a/packages/builder/src/dataBinding.js
+++ b/packages/builder/src/dataBinding.js
@@ -425,7 +425,7 @@ const generateComponentContextBindings = (asset, componentContext) => {
       table = info.table
 
       // Determine what to prefix bindings with
-      if (datasource.type === "jsonarray") {
+      if (datasource.type === "jsonarray" || datasource.type === "queryarray") {
         // For JSON arrays, use the array name as the readable prefix
         const split = datasource.label.split(".")
         readablePrefix = split[split.length - 1]
@@ -904,6 +904,19 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
       schema = JSONUtils.getJSONArrayDatasourceSchema(tableSchema, datasource)
     }
 
+    // "queryarray" datasources are arrays inside JSON responses
+    else if (type === "queryarray") {
+      const queries = get(queriesStores).list
+      table = queries.find(query => query._id === datasource.tableId)
+      let tableSchema = table?.schema
+      let nestedSchemaFields = table?.nestedSchemaFields
+      schema = JSONUtils.generateQueryArraySchemas(
+        tableSchema,
+        nestedSchemaFields
+      )
+      schema = JSONUtils.getJSONArrayDatasourceSchema(schema, datasource)
+    }
+
     // Otherwise we assume we're targeting an internal table or a plus
     // datasource, and we can treat it as a table with a schema
     else {
diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte
index 464ca95829..5522bd4b46 100644
--- a/packages/client/src/components/app/forms/Form.svelte
+++ b/packages/client/src/components/app/forms/Form.svelte
@@ -84,7 +84,7 @@
 
   // Fetches the form schema from this form's dataSource
   const fetchSchema = async dataSource => {
-    if (dataSource?.tableId && dataSource?.type !== "query") {
+    if (dataSource?.tableId && !dataSource?.type?.startsWith("query")) {
       try {
         table = await API.fetchTableDefinition(dataSource.tableId)
       } catch (error) {
diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index f20e724a6e..e2399e8738 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -7,6 +7,7 @@ import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProvide
 import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
 import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
 import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
+import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
 
 /**
  * Fetches the schema of any kind of datasource.
@@ -28,6 +29,7 @@ export const fetchDatasourceSchema = async (
     provider: NestedProviderFetch,
     field: FieldFetch,
     jsonarray: JSONArrayFetch,
+    queryarray: QueryArrayFetch,
   }[datasource?.type]
   if (!handler) {
     return null
diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.js b/packages/frontend-core/src/fetch/QueryArrayFetch.js
new file mode 100644
index 0000000000..0b36b640a6
--- /dev/null
+++ b/packages/frontend-core/src/fetch/QueryArrayFetch.js
@@ -0,0 +1,25 @@
+import FieldFetch from "./FieldFetch.js"
+import {
+  getJSONArrayDatasourceSchema,
+  generateQueryArraySchemas,
+} from "../utils/json"
+
+export default class QueryArrayFetch extends FieldFetch {
+  async getDefinition(datasource) {
+    if (!datasource?.tableId) {
+      return null
+    }
+    // JSON arrays need their table definitions fetched.
+    // We can then extract their schema as a subset of the table schema.
+    try {
+      const table = await this.API.fetchQueryDefinition(datasource.tableId)
+      const schema = generateQueryArraySchemas(
+        table?.schema,
+        table?.nestedSchemaFields
+      )
+      return { schema: getJSONArrayDatasourceSchema(schema, datasource) }
+    } catch (error) {
+      return null
+    }
+  }
+}
diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.js
index a41a859351..903810ac25 100644
--- a/packages/frontend-core/src/fetch/index.js
+++ b/packages/frontend-core/src/fetch/index.js
@@ -9,6 +9,7 @@ import JSONArrayFetch from "./JSONArrayFetch.js"
 import UserFetch from "./UserFetch.js"
 import GroupUserFetch from "./GroupUserFetch.js"
 import CustomFetch from "./CustomFetch.js"
+import QueryArrayFetch from "./QueryArrayFetch.js"
 
 const DataFetchMap = {
   table: TableFetch,
@@ -24,6 +25,7 @@ const DataFetchMap = {
   provider: NestedProviderFetch,
   field: FieldFetch,
   jsonarray: JSONArrayFetch,
+  queryarray: QueryArrayFetch,
 }
 
 // Constructs a new fetch model for a certain datasource
diff --git a/packages/frontend-core/src/utils/json.js b/packages/frontend-core/src/utils/json.js
index 29bf2df34e..8cd37f9ad1 100644
--- a/packages/frontend-core/src/utils/json.js
+++ b/packages/frontend-core/src/utils/json.js
@@ -1,3 +1,5 @@
+import { utils } from "@budibase/shared-core"
+
 /**
  * Gets the schema for a datasource which is targeting a JSON array, including
  * nested JSON arrays. The returned schema is a squashed, table-like schema
@@ -119,3 +121,33 @@ const extractJSONSchemaKeys = (jsonSchema, squashObjects = false) => {
   })
   return keys
 }
+
+export const generateQueryArraySchemas = (schema, nestedSchemaFields) => {
+  for (let key in schema) {
+    if (
+      schema[key]?.type === "json" &&
+      schema[key]?.subtype === "array" &&
+      utils.hasSchema(nestedSchemaFields[key])
+    ) {
+      schema[key] = {
+        schema: {
+          schema: Object.entries(nestedSchemaFields[key] || {}).reduce(
+            (acc, [nestedKey, fieldSchema]) => {
+              acc[nestedKey] = {
+                name: nestedKey,
+                type: fieldSchema.type,
+                subtype: fieldSchema.subtype,
+              }
+              return acc
+            },
+            {}
+          ),
+          type: "json",
+        },
+        type: "json",
+        subtype: "array",
+      }
+    }
+  }
+  return schema
+}
diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts
index 8dabe5b3cc..89330f3216 100644
--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@@ -1,5 +1,4 @@
 import { generateQueryID } from "../../../db/utils"
-import { BaseQueryVerbs } from "../../../constants"
 import { Thread, ThreadType } from "../../../threads"
 import { save as saveDatasource } from "../datasource"
 import { RestImporter } from "./import"
@@ -7,36 +6,27 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { QueryEvent, QueryResponse } from "../../../threads/definitions"
+import { QueryEvent } from "../../../threads/definitions"
 import {
   ConfigType,
   Query,
   UserCtx,
   SessionCookie,
+  JsonFieldSubType,
+  QueryResponse,
+  QueryPreview,
   QuerySchema,
   FieldType,
   type ExecuteQueryRequest,
   type ExecuteQueryResponse,
   type Row,
 } from "@budibase/types"
-import { ValidQueryNameRegex } from "@budibase/shared-core"
+import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT,
 })
 
-// simple function to append "readable" to all read queries
-function enrichQueries(input: any) {
-  const wasArray = Array.isArray(input)
-  const queries = wasArray ? input : [input]
-  for (let query of queries) {
-    if (query.queryVerb === BaseQueryVerbs.READ) {
-      query.readable = true
-    }
-  }
-  return wasArray ? queries : queries[0]
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.queries.fetch()
 }
@@ -84,7 +74,7 @@ export { _import as import }
 
 export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
-  const query = ctx.request.body
+  const query: Query = ctx.request.body
 
   // Validate query name
   if (!query?.name.match(ValidQueryNameRegex)) {
@@ -100,7 +90,6 @@ export async function save(ctx: UserCtx) {
   } else {
     eventFn = () => events.query.updated(datasource, query)
   }
-
   const response = await db.put(query)
   await eventFn()
   query._rev = response.rev
@@ -133,7 +122,7 @@ export async function preview(ctx: UserCtx) {
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     ctx.request.body.datasourceId
   )
-  const query = ctx.request.body
+  const query: QueryPreview = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
   const { fields, parameters, queryVerb, transformer, queryId, schema } = query
@@ -153,6 +142,69 @@ export async function preview(ctx: UserCtx) {
 
   const authConfigCtx: any = getAuthConfig(ctx)
 
+  function getSchemaFields(
+    rows: any[],
+    keys: string[]
+  ): {
+    previewSchema: Record<string, string | QuerySchema>
+    nestedSchemaFields: {
+      [key: string]: Record<string, string | QuerySchema>
+    }
+  } {
+    const previewSchema: Record<string, string | QuerySchema> = {}
+    const nestedSchemaFields: {
+      [key: string]: Record<string, string | QuerySchema>
+    } = {}
+    const makeQuerySchema = (
+      type: FieldType,
+      name: string,
+      subtype?: string
+    ): QuerySchema => ({
+      type,
+      name,
+      subtype,
+    })
+    if (rows?.length > 0) {
+      for (let key of [...new Set(keys)] as string[]) {
+        const field = rows[0][key]
+        let type = typeof field,
+          fieldMetadata = makeQuerySchema(FieldType.STRING, key)
+        if (field)
+          switch (type) {
+            case "boolean":
+              fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
+              break
+            case "object":
+              if (field instanceof Date) {
+                fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
+              } else if (Array.isArray(field)) {
+                if (JsonUtils.hasSchema(field[0])) {
+                  fieldMetadata = makeQuerySchema(
+                    FieldType.JSON,
+                    key,
+                    JsonFieldSubType.ARRAY
+                  )
+                } else {
+                  fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
+                }
+                nestedSchemaFields[key] = getSchemaFields(
+                  field,
+                  Object.keys(field[0])
+                ).previewSchema
+              } else {
+                fieldMetadata = makeQuerySchema(FieldType.JSON, key)
+              }
+              break
+            case "number":
+              fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
+              break
+          }
+        previewSchema[key] = fieldMetadata
+      }
+    }
+    return { previewSchema, nestedSchemaFields }
+  }
+
   try {
     const inputs: QueryEvent = {
       appId: ctx.appId,
@@ -171,38 +223,11 @@ export async function preview(ctx: UserCtx) {
       },
     }
 
-    const { rows, keys, info, extra } = await Runner.run<QueryResponse>(inputs)
-    const previewSchema: Record<string, QuerySchema> = {}
-    const makeQuerySchema = (type: FieldType, name: string): QuerySchema => ({
-      type,
-      name,
-    })
-    if (rows?.length > 0) {
-      for (let key of [...new Set(keys)] as string[]) {
-        const field = rows[0][key]
-        let type = typeof field,
-          fieldMetadata = makeQuerySchema(FieldType.STRING, key)
-        if (field)
-          switch (type) {
-            case "boolean":
-              fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
-              break
-            case "object":
-              if (field instanceof Date) {
-                fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
-              } else if (Array.isArray(field)) {
-                fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
-              } else {
-                fieldMetadata = makeQuerySchema(FieldType.JSON, key)
-              }
-              break
-            case "number":
-              fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
-              break
-          }
-        previewSchema[key] = fieldMetadata
-      }
-    }
+    const { rows, keys, info, extra } = (await Runner.run(
+      inputs
+    )) as QueryResponse
+    const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys)
+
     // if existing schema, update to include any previous schema keys
     if (existingSchema) {
       for (let key of Object.keys(previewSchema)) {
@@ -216,6 +241,7 @@ export async function preview(ctx: UserCtx) {
     await events.query.previewed(datasource, query)
     ctx.body = {
       rows,
+      nestedSchemaFields,
       schema: previewSchema,
       info,
       extra,
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index c775010909..cc5646a00e 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -57,3 +57,13 @@ export function filterValueToLabel() {
     {}
   )
 }
+
+export function hasSchema(test: any) {
+  return (
+    typeof test === "object" &&
+    !Array.isArray(test) &&
+    test !== null &&
+    !(test instanceof Date) &&
+    Object.keys(test).length > 0
+  )
+}
diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts
index 81aa90b807..f4547b9774 100644
--- a/packages/types/src/documents/app/query.ts
+++ b/packages/types/src/documents/app/query.ts
@@ -4,6 +4,7 @@ import type { Row } from "./row"
 export interface QuerySchema {
   name?: string
   type: string
+  subtype?: string
 }
 
 export interface Query extends Document {
@@ -17,11 +18,23 @@ export interface Query extends Document {
   queryVerb: string
 }
 
+export interface QueryPreview extends Omit<Query, "_id"> {
+  queryId: string
+}
+
 export interface QueryParameter {
   name: string
   default: string
 }
 
+export interface QueryResponse {
+  rows: any[]
+  keys: string[]
+  info: any
+  extra: any
+  pagination: any
+}
+
 export interface RestQueryFields {
   path: string
   queryString?: string
diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts
index fc831e7e7c..1d9d14695a 100644
--- a/packages/types/src/documents/app/table/constants.ts
+++ b/packages/types/src/documents/app/table/constants.ts
@@ -16,6 +16,10 @@ export enum AutoFieldSubType {
   AUTO_ID = "autoID",
 }
 
+export enum JsonFieldSubType {
+  ARRAY = "array",
+}
+
 export enum FormulaType {
   STATIC = "static",
   DYNAMIC = "dynamic",
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 47ec303b66..17abf747b2 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -5,6 +5,7 @@ import {
   AutoFieldSubType,
   AutoReason,
   FormulaType,
+  JsonFieldSubType,
   RelationshipType,
 } from "./constants"
 
@@ -81,6 +82,11 @@ export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   }
 }
 
+export interface JsonFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.JSON
+  subtype?: JsonFieldSubType.ARRAY
+}
+
 export interface DateFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.DATETIME
   ignoreTimezones?: boolean
@@ -162,6 +168,7 @@ export type FieldSchema =
   | NumberFieldMetadata
   | LongFormFieldMetadata
   | BBReferenceFieldMetadata
+  | JsonFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema
diff --git a/yarn.lock b/yarn.lock
index 1937482837..c0a11b9bf7 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5572,9 +5572,9 @@
   integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
 
 "@types/node@^18.11.18":
-  version "18.19.10"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.10.tgz#4de314ab66faf6bc8ba691021a091ddcdf13a158"
-  integrity sha512-IZD8kAM02AW1HRDTPOlz3npFava678pr8Ie9Vp8uRhBROXAv8MXT2pCnGZZAKYdromsNQLHQcfWQ6EOatVLtqA==
+  version "18.19.13"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.13.tgz#c3e989ca967b862a1f6c8c4148fe31865eedaf1a"
+  integrity sha512-kgnbRDj8ioDyGxoiaXsiu1Ybm/K14ajCgMOkwiqpHrnF7d7QiYRoRqHIpglMMs3DwXinlK4qJ8TZGlj4hfleJg==
   dependencies:
     undici-types "~5.26.4"
 
@@ -10763,7 +10763,7 @@ fetch-cookie@0.11.0:
   dependencies:
     tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
 
-fflate@^0.4.1:
+fflate@^0.4.1, fflate@^0.4.8:
   version "0.4.8"
   resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.4.8.tgz#f90b82aefbd8ac174213abb338bd7ef848f0f5ae"
   integrity sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==

From b1966114ef77f2a351a982f38ac1e5abaea276af Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 19 Feb 2024 09:28:34 +0000
Subject: [PATCH 34/34] Bump version to 2.19.6

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 596fb434bc..62068e25a8 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.19.5",
+  "version": "2.19.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",