From 80a772f39fb8d7cc23bbd03cee40947cd2e7c3c6 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 5 Apr 2024 13:15:06 +0100
Subject: [PATCH 01/45] Add snippets to app imports

---
 packages/server/src/api/controllers/application.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index ceef421fab..6acdfcd465 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -320,6 +320,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
         "theme",
         "customTheme",
         "icon",
+        "snippets",
       ]
       keys.forEach(key => {
         if (existing[key]) {

From 3201eb5953a8e8b6c5cf4d2b3b6e357ae6062f38 Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Thu, 11 Apr 2024 12:53:32 +0100
Subject: [PATCH 02/45] adds sidepanel open and close actions, and gives the
 user the option to disable click-outside closure of sidepanel

---
 packages/client/manifest.json                 | 22 +++++++++++++-
 .../client/src/components/app/Layout.svelte   |  6 +++-
 .../src/components/app/SidePanel.svelte       | 29 +++++++++++++++++++
 packages/client/src/stores/sidePanel.js       |  9 ++++++
 4 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 08d614391b..2f52085e38 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6487,7 +6487,27 @@
     "illegalChildren": ["section", "sidepanel"],
     "showEmptyState": false,
     "draggable": false,
-    "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action."
+    "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action.",
+    "sendEvents": true,
+    "settings": [
+      {
+        "type": "boolean",
+        "key": "clickOutsideToClose",
+        "label": "Click away to close",
+        "defaultValue": true
+      },
+      {
+        "type": "event",
+        "key": "sidePanelOpen",
+        "label": "Side Panel Open"
+      },
+      {
+        "type": "event",
+        "key": "sidePanelClose",
+        "label": "Side Panel Close",
+        "info": "Side panel actions configured here will run after the 'Open side panel' action runs, and are not capable of preventing or stopping it. Any form validation should therefore be done before that action if invoked, and not here."
+      }
+    ]
   },
   "rowexplorer": {
     "block": true,
diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte
index 992a166143..5b68171539 100644
--- a/packages/client/src/components/app/Layout.svelte
+++ b/packages/client/src/components/app/Layout.svelte
@@ -292,7 +292,11 @@
     id="side-panel-container"
     class:open={$sidePanelStore.open}
     use:clickOutside={{
-      callback: autoCloseSidePanel ? sidePanelStore.actions.close : null,
+      callback:
+        $sidePanelStore.clickOutsideToClose && autoCloseSidePanel
+          ? sidePanelStore.actions.close
+          : null,
+
       allowedType: "mousedown",
     }}
     class:builder={$builderStore.inBuilder}
diff --git a/packages/client/src/components/app/SidePanel.svelte b/packages/client/src/components/app/SidePanel.svelte
index 825b401bb8..48c84828b0 100644
--- a/packages/client/src/components/app/SidePanel.svelte
+++ b/packages/client/src/components/app/SidePanel.svelte
@@ -5,6 +5,13 @@
   const { styleable, sidePanelStore, builderStore, dndIsDragging } =
     getContext("sdk")
 
+  let handlingSidePanelOpen
+  let handlingSidePanelClose
+
+  export let sidePanelOpen
+  export let sidePanelClose
+  export let clickOutsideToClose
+
   // Automatically show and hide the side panel when inside the builder.
   // For some unknown reason, svelte reactivity breaks if we reference the
   // reactive variable "open" inside the following expression, or if we define
@@ -26,6 +33,10 @@
     }
   }
 
+  $: {
+    sidePanelStore.actions.setSidepanelState(clickOutsideToClose)
+  }
+
   // Derive visibility
   $: open = $sidePanelStore.contentId === $component.id
 
@@ -40,6 +51,22 @@
     }
   }
 
+  const handleSidePanelOpen = async () => {
+    handlingSidePanelOpen = true
+    if (sidePanelOpen) {
+      await sidePanelOpen()
+    }
+    handlingSidePanelOpen = false
+  }
+
+  const handleSidePanelClose = async () => {
+    handlingSidePanelClose = true
+    if (sidePanelClose) {
+      await sidePanelClose()
+    }
+    handlingSidePanelOpen = false
+  }
+
   const showInSidePanel = (el, visible) => {
     const update = visible => {
       const target = document.getElementById("side-panel-container")
@@ -47,10 +74,12 @@
       if (visible) {
         if (!target.contains(node)) {
           target.appendChild(node)
+          handleSidePanelOpen()
         }
       } else {
         if (target.contains(node)) {
           target.removeChild(node)
+          handleSidePanelClose()
         }
       }
     }
diff --git a/packages/client/src/stores/sidePanel.js b/packages/client/src/stores/sidePanel.js
index 3b3b9f5f4d..df66eca01c 100644
--- a/packages/client/src/stores/sidePanel.js
+++ b/packages/client/src/stores/sidePanel.js
@@ -3,6 +3,7 @@ import { writable, derived } from "svelte/store"
 export const createSidePanelStore = () => {
   const initialState = {
     contentId: null,
+    clickOutsideToClose: true,
   }
   const store = writable(initialState)
   const derivedStore = derived(store, $store => {
@@ -32,11 +33,19 @@ export const createSidePanelStore = () => {
     }, 50)
   }
 
+  const setSidepanelState = bool => {
+    clearTimeout(timeout)
+    store.update(state => {
+      state.clickOutsideToClose = bool
+      return state
+    })
+  }
   return {
     subscribe: derivedStore.subscribe,
     actions: {
       open,
       close,
+      setSidepanelState,
     },
   }
 }

From a7ec49613c635f95dbf650217efdb515f6e92cfb Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Thu, 11 Apr 2024 15:42:19 +0100
Subject: [PATCH 03/45] fixes typo, removes unused variables

---
 packages/bbui/src/Layout/Page.svelte                | 10 +++++-----
 packages/client/manifest.json                       |  2 +-
 packages/client/src/components/app/SidePanel.svelte |  7 -------
 3 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/packages/bbui/src/Layout/Page.svelte b/packages/bbui/src/Layout/Page.svelte
index 2169a12459..62dd9cc909 100644
--- a/packages/bbui/src/Layout/Page.svelte
+++ b/packages/bbui/src/Layout/Page.svelte
@@ -7,11 +7,11 @@
   export let narrower = false
   export let noPadding = false
 
-  let sidePanelVisble = false
+  let sidePanelVisible = false
 
   setContext("side-panel", {
-    open: () => (sidePanelVisble = true),
-    close: () => (sidePanelVisble = false),
+    open: () => (sidePanelVisible = true),
+    close: () => (sidePanelVisible = false),
   })
 </script>
 
@@ -24,9 +24,9 @@
   </div>
   <div
     id="side-panel"
-    class:visible={sidePanelVisble}
+    class:visible={sidePanelVisible}
     use:clickOutside={() => {
-      sidePanelVisble = false
+      sidePanelVisible = false
     }}
   >
     <slot name="side-panel" />
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 2f52085e38..f7d437a4fd 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6505,7 +6505,7 @@
         "type": "event",
         "key": "sidePanelClose",
         "label": "Side Panel Close",
-        "info": "Side panel actions configured here will run after the 'Open side panel' action runs, and are not capable of preventing or stopping it. Any form validation should therefore be done before that action if invoked, and not here."
+        "info": "Side panel actions configured here will run after the 'Open side panel' action runs, and are not capable of preventing or stopping it. Any form validation should therefore be done before that action is invoked, and not here."
       }
     ]
   },
diff --git a/packages/client/src/components/app/SidePanel.svelte b/packages/client/src/components/app/SidePanel.svelte
index 48c84828b0..98398c4671 100644
--- a/packages/client/src/components/app/SidePanel.svelte
+++ b/packages/client/src/components/app/SidePanel.svelte
@@ -5,9 +5,6 @@
   const { styleable, sidePanelStore, builderStore, dndIsDragging } =
     getContext("sdk")
 
-  let handlingSidePanelOpen
-  let handlingSidePanelClose
-
   export let sidePanelOpen
   export let sidePanelClose
   export let clickOutsideToClose
@@ -52,19 +49,15 @@
   }
 
   const handleSidePanelOpen = async () => {
-    handlingSidePanelOpen = true
     if (sidePanelOpen) {
       await sidePanelOpen()
     }
-    handlingSidePanelOpen = false
   }
 
   const handleSidePanelClose = async () => {
-    handlingSidePanelClose = true
     if (sidePanelClose) {
       await sidePanelClose()
     }
-    handlingSidePanelOpen = false
   }
 
   const showInSidePanel = (el, visible) => {

From 12fdaefe4c57fcc1ff057d67e22dd9ea9e00d80a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 11 Apr 2024 17:12:15 +0100
Subject: [PATCH 04/45] Add tests for sorting to search.spec.ts

---
 .../src/api/routes/tests/search.spec.ts       | 207 ++++++++++++++++--
 .../server/src/sdk/app/rows/search/sqs.ts     |   2 +-
 2 files changed, 184 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index fdf1ed7603..028b970a42 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -8,6 +8,8 @@ import {
   FieldType,
   RowSearchParams,
   SearchFilters,
+  SortOrder,
+  SortType,
   Table,
   TableSchema,
 } from "@budibase/types"
@@ -62,7 +64,25 @@ describe.each([
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    async toFind(expectedRows: any[]) {
+    async toMatch(expectedRows: any[]) {
+      const { rows: foundRows } = await config.api.row.search(table._id!, {
+        ...this.query,
+        tableId: table._id!,
+      })
+
+      // eslint-disable-next-line jest/no-standalone-expect
+      expect(foundRows).toHaveLength(expectedRows.length)
+      // eslint-disable-next-line jest/no-standalone-expect
+      expect(foundRows).toEqual(
+        expectedRows.map((expectedRow: any) =>
+          expect.objectContaining(
+            foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
+          )
+        )
+      )
+    }
+
+    async toContain(expectedRows: any[]) {
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -83,7 +103,17 @@ describe.each([
     }
 
     async toFindNothing() {
-      await this.toFind([])
+      await this.toContain([])
+    }
+
+    async toHaveLength(length: number) {
+      const { rows: foundRows } = await config.api.row.search(table._id!, {
+        ...this.query,
+        tableId: table._id!,
+      })
+
+      // eslint-disable-next-line jest/no-standalone-expect
+      expect(foundRows).toHaveLength(length)
     }
   }
 
@@ -105,28 +135,31 @@ describe.each([
 
     describe("misc", () => {
       it("should return all if no query is passed", () =>
-        expectSearch({} as RowSearchParams).toFind([
+        expectSearch({} as RowSearchParams).toContain([
           { name: "foo" },
           { name: "bar" },
         ]))
 
       it("should return all if empty query is passed", () =>
-        expectQuery({}).toFind([{ name: "foo" }, { name: "bar" }]))
+        expectQuery({}).toContain([{ name: "foo" }, { name: "bar" }]))
 
       it("should return all if onEmptyFilter is RETURN_ALL", () =>
         expectQuery({
           onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-        }).toFind([{ name: "foo" }, { name: "bar" }]))
+        }).toContain([{ name: "foo" }, { name: "bar" }]))
 
       it("should return nothing if onEmptyFilter is RETURN_NONE", () =>
         expectQuery({
           onEmptyFilter: EmptyFilterOption.RETURN_NONE,
         }).toFindNothing())
+
+      it("should respect limit", () =>
+        expectSearch({ limit: 1, paginate: true, query: {} }).toHaveLength(1))
     })
 
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { name: "foo" } }).toFind([{ name: "foo" }]))
+        expectQuery({ equal: { name: "foo" } }).toContain([{ name: "foo" }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { name: "none" } }).toFindNothing())
@@ -134,15 +167,15 @@ describe.each([
 
     describe("notEqual", () => {
       it("successfully finds a row", () =>
-        expectQuery({ notEqual: { name: "foo" } }).toFind([{ name: "bar" }]))
+        expectQuery({ notEqual: { name: "foo" } }).toContain([{ name: "bar" }]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ notEqual: { name: "bar" } }).toFind([{ name: "foo" }]))
+        expectQuery({ notEqual: { name: "bar" } }).toContain([{ name: "foo" }]))
     })
 
     describe("oneOf", () => {
       it("successfully finds a row", () =>
-        expectQuery({ oneOf: { name: ["foo"] } }).toFind([{ name: "foo" }]))
+        expectQuery({ oneOf: { name: ["foo"] } }).toContain([{ name: "foo" }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ oneOf: { name: ["none"] } }).toFindNothing())
@@ -150,11 +183,45 @@ describe.each([
 
     describe("fuzzy", () => {
       it("successfully finds a row", () =>
-        expectQuery({ fuzzy: { name: "oo" } }).toFind([{ name: "foo" }]))
+        expectQuery({ fuzzy: { name: "oo" } }).toContain([{ name: "foo" }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ fuzzy: { name: "none" } }).toFindNothing())
     })
+
+    describe("sort", () => {
+      it("sorts ascending", () =>
+        expectSearch({
+          query: {},
+          sort: "name",
+          sortOrder: SortOrder.ASCENDING,
+        }).toMatch([{ name: "bar" }, { name: "foo" }]))
+
+      it("sorts descending", () =>
+        expectSearch({
+          query: {},
+          sort: "name",
+          sortOrder: SortOrder.DESCENDING,
+        }).toMatch([{ name: "foo" }, { name: "bar" }]))
+
+      describe("sortType STRING", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "name",
+            sortType: SortType.STRING,
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatch([{ name: "bar" }, { name: "foo" }]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "name",
+            sortType: SortType.STRING,
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatch([{ name: "foo" }, { name: "bar" }]))
+      })
+    })
   })
 
   describe("numbers", () => {
@@ -167,7 +234,7 @@ describe.each([
 
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { age: 1 } }).toFind([{ age: 1 }]))
+        expectQuery({ equal: { age: 1 } }).toContain([{ age: 1 }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { age: 2 } }).toFindNothing())
@@ -175,15 +242,15 @@ describe.each([
 
     describe("notEqual", () => {
       it("successfully finds a row", () =>
-        expectQuery({ notEqual: { age: 1 } }).toFind([{ age: 10 }]))
+        expectQuery({ notEqual: { age: 1 } }).toContain([{ age: 10 }]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ notEqual: { age: 10 } }).toFind([{ age: 1 }]))
+        expectQuery({ notEqual: { age: 10 } }).toContain([{ age: 1 }]))
     })
 
     describe("oneOf", () => {
       it("successfully finds a row", () =>
-        expectQuery({ oneOf: { age: [1] } }).toFind([{ age: 1 }]))
+        expectQuery({ oneOf: { age: [1] } }).toContain([{ age: 1 }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ oneOf: { age: [2] } }).toFindNothing())
@@ -193,17 +260,69 @@ describe.each([
       it("successfully finds a row", () =>
         expectQuery({
           range: { age: { low: 1, high: 5 } },
-        }).toFind([{ age: 1 }]))
+        }).toContain([{ age: 1 }]))
 
       it("successfully finds multiple rows", () =>
         expectQuery({
           range: { age: { low: 1, high: 10 } },
-        }).toFind([{ age: 1 }, { age: 10 }]))
+        }).toContain([{ age: 1 }, { age: 10 }]))
 
       it("successfully finds a row with a high bound", () =>
         expectQuery({
           range: { age: { low: 5, high: 10 } },
-        }).toFind([{ age: 10 }]))
+        }).toContain([{ age: 10 }]))
+
+      describe("sort", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "age",
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatch([{ age: 1 }, { age: 10 }]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "age",
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatch([{ age: 10 }, { age: 1 }]))
+      })
+
+      describe("sortType NUMBER", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "age",
+            sortType: SortType.NUMBER,
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatch([{ age: 1 }, { age: 10 }]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "age",
+            sortType: SortType.NUMBER,
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatch([{ age: 10 }, { age: 1 }]))
+      })
+
+      describe("sortType STRING", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "age",
+            sortType: SortType.STRING,
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatch([{ age: 1 }, { age: 10 }]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "age",
+            sortType: SortType.STRING,
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatch([{ age: 10 }, { age: 1 }]))
+      })
     })
   })
 
@@ -223,7 +342,7 @@ describe.each([
 
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { dob: JAN_1ST } }).toFind([{ dob: JAN_1ST }]))
+        expectQuery({ equal: { dob: JAN_1ST } }).toContain([{ dob: JAN_1ST }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing())
@@ -231,15 +350,21 @@ describe.each([
 
     describe("notEqual", () => {
       it("successfully finds a row", () =>
-        expectQuery({ notEqual: { dob: JAN_1ST } }).toFind([{ dob: JAN_10TH }]))
+        expectQuery({ notEqual: { dob: JAN_1ST } }).toContain([
+          { dob: JAN_10TH },
+        ]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ notEqual: { dob: JAN_10TH } }).toFind([{ dob: JAN_1ST }]))
+        expectQuery({ notEqual: { dob: JAN_10TH } }).toContain([
+          { dob: JAN_1ST },
+        ]))
     })
 
     describe("oneOf", () => {
       it("successfully finds a row", () =>
-        expectQuery({ oneOf: { dob: [JAN_1ST] } }).toFind([{ dob: JAN_1ST }]))
+        expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContain([
+          { dob: JAN_1ST },
+        ]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing())
@@ -249,17 +374,51 @@ describe.each([
       it("successfully finds a row", () =>
         expectQuery({
           range: { dob: { low: JAN_1ST, high: JAN_5TH } },
-        }).toFind([{ dob: JAN_1ST }]))
+        }).toContain([{ dob: JAN_1ST }]))
 
       it("successfully finds multiple rows", () =>
         expectQuery({
           range: { dob: { low: JAN_1ST, high: JAN_10TH } },
-        }).toFind([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+        }).toContain([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
 
       it("successfully finds a row with a high bound", () =>
         expectQuery({
           range: { dob: { low: JAN_5TH, high: JAN_10TH } },
-        }).toFind([{ dob: JAN_10TH }]))
+        }).toContain([{ dob: JAN_10TH }]))
+    })
+
+    describe("sort", () => {
+      it("sorts ascending", () =>
+        expectSearch({
+          query: {},
+          sort: "dob",
+          sortOrder: SortOrder.ASCENDING,
+        }).toMatch([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+
+      it("sorts descending", () =>
+        expectSearch({
+          query: {},
+          sort: "dob",
+          sortOrder: SortOrder.DESCENDING,
+        }).toMatch([{ dob: JAN_10TH }, { dob: JAN_1ST }]))
+
+      describe("sortType STRING", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "dob",
+            sortType: SortType.STRING,
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatch([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "dob",
+            sortType: SortType.STRING,
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatch([{ dob: JAN_10TH }, { dob: JAN_1ST }]))
+      })
     })
   })
 })
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 5b0b6e3bc7..7abd7d9e72 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -132,7 +132,7 @@ export async function search(
     type: "row",
   }
 
-  if (params.sort && !params.sortType) {
+  if (params.sort) {
     const sortField = table.schema[params.sort]
     const sortType =
       sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING

From a024a28de1e3430b42b6a9cb2a03f76d38c303a2 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 11 Apr 2024 17:16:32 +0100
Subject: [PATCH 05/45] Fix tests.

---
 .../src/api/routes/tests/search.spec.ts       | 78 +++++++------------
 1 file changed, 30 insertions(+), 48 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 028b970a42..4db47be216 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -271,58 +271,40 @@ describe.each([
         expectQuery({
           range: { age: { low: 5, high: 10 } },
         }).toContain([{ age: 10 }]))
+    })
 
-      describe("sort", () => {
-        it("sorts ascending", () =>
-          expectSearch({
-            query: {},
-            sort: "age",
-            sortOrder: SortOrder.ASCENDING,
-          }).toMatch([{ age: 1 }, { age: 10 }]))
+    describe("sort", () => {
+      it("sorts ascending", () =>
+        expectSearch({
+          query: {},
+          sort: "age",
+          sortOrder: SortOrder.ASCENDING,
+        }).toMatch([{ age: 1 }, { age: 10 }]))
 
-        it("sorts descending", () =>
-          expectSearch({
-            query: {},
-            sort: "age",
-            sortOrder: SortOrder.DESCENDING,
-          }).toMatch([{ age: 10 }, { age: 1 }]))
-      })
+      it("sorts descending", () =>
+        expectSearch({
+          query: {},
+          sort: "age",
+          sortOrder: SortOrder.DESCENDING,
+        }).toMatch([{ age: 10 }, { age: 1 }]))
+    })
 
-      describe("sortType NUMBER", () => {
-        it("sorts ascending", () =>
-          expectSearch({
-            query: {},
-            sort: "age",
-            sortType: SortType.NUMBER,
-            sortOrder: SortOrder.ASCENDING,
-          }).toMatch([{ age: 1 }, { age: 10 }]))
+    describe("sortType NUMBER", () => {
+      it("sorts ascending", () =>
+        expectSearch({
+          query: {},
+          sort: "age",
+          sortType: SortType.NUMBER,
+          sortOrder: SortOrder.ASCENDING,
+        }).toMatch([{ age: 1 }, { age: 10 }]))
 
-        it("sorts descending", () =>
-          expectSearch({
-            query: {},
-            sort: "age",
-            sortType: SortType.NUMBER,
-            sortOrder: SortOrder.DESCENDING,
-          }).toMatch([{ age: 10 }, { age: 1 }]))
-      })
-
-      describe("sortType STRING", () => {
-        it("sorts ascending", () =>
-          expectSearch({
-            query: {},
-            sort: "age",
-            sortType: SortType.STRING,
-            sortOrder: SortOrder.ASCENDING,
-          }).toMatch([{ age: 1 }, { age: 10 }]))
-
-        it("sorts descending", () =>
-          expectSearch({
-            query: {},
-            sort: "age",
-            sortType: SortType.STRING,
-            sortOrder: SortOrder.DESCENDING,
-          }).toMatch([{ age: 10 }, { age: 1 }]))
-      })
+      it("sorts descending", () =>
+        expectSearch({
+          query: {},
+          sort: "age",
+          sortType: SortType.NUMBER,
+          sortOrder: SortOrder.DESCENDING,
+        }).toMatch([{ age: 10 }, { age: 1 }]))
     })
   })
 

From 6e4a66b2e1101566533f6397af9877a5882f20b3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 11 Apr 2024 18:19:47 +0100
Subject: [PATCH 06/45] Initial implementation of generating SQS junction table
 definitions.

---
 hosting/.env                                  |  3 +-
 hosting/docker-compose.dev.yaml               |  3 +-
 packages/backend-core/src/environment.ts      |  2 +-
 packages/server/src/environment.ts            |  2 +
 .../server/src/sdk/app/tables/internal/sqs.ts | 60 ++++++++++++++++---
 packages/types/src/documents/app/sqlite.ts    | 20 ++++---
 6 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/hosting/.env b/hosting/.env
index 8a0756c0e3..173d409d04 100644
--- a/hosting/.env
+++ b/hosting/.env
@@ -17,6 +17,7 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+COUCH_DB_SQS_PORT=4006
 REDIS_PORT=6379
 WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
@@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
 
 # A path that is watched for plugin bundles. Any bundles found are imported automatically/
 PLUGINS_DIR=
-ROLLING_LOG_MAX_SIZE=
\ No newline at end of file
+ROLLING_LOG_MAX_SIZE=
diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml
index 9dba5d427c..77f6bd053b 100644
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@@ -42,12 +42,13 @@ services:
   couchdb-service:
     container_name: budi-couchdb3-dev
     restart: on-failure
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.2.1-sqs
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
+      - "${COUCH_DB_SQS_PORT}:4984"
     volumes:
       - couchdb_data:/data
 
diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 2da2a77d67..8dbc904643 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -107,7 +107,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index f8adcbe0ee..d9d299d5fa 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -28,6 +28,7 @@ const DEFAULTS = {
   PLUGINS_DIR: "/plugins",
   FORKED_PROCESS_NAME: "main",
   JS_RUNNER_MEMORY_LIMIT: 64,
+  COUCH_DB_SQL_URL: "http://localhost:4006",
 }
 
 const QUERY_THREAD_TIMEOUT =
@@ -39,6 +40,7 @@ const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
   AWS_REGION: process.env.AWS_REGION,
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 79d9be2348..5dd16f516c 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -1,8 +1,19 @@
 import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
-import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
+import {
+  FieldType,
+  RelationshipFieldMetadata,
+  SQLiteDefinition,
+  SQLiteTable,
+  SQLiteTables,
+  SQLiteType,
+  Table,
+} from "@budibase/types"
 import { cloneDeep } from "lodash"
 import tablesSdk from "../"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  generateJunctionTableID,
+} from "../../../../db/utils"
 
 const BASIC_SQLITE_DOC: SQLiteDefinition = {
   _id: SQLITE_DESIGN_DOC_ID,
@@ -36,9 +47,38 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BB_REFERENCE]: SQLiteType.TEXT,
 }
 
-function mapTable(table: Table): { [key: string]: SQLiteType } {
+function buildRelationshipDefinitions(
+  table: Table,
+  relationshipColumn: RelationshipFieldMetadata
+): {
+  tableId: string
+  definition: SQLiteTable
+} {
+  const tableId = table._id!,
+    relatedTableId = relationshipColumn.tableId
+  return {
+    tableId: generateJunctionTableID(tableId, relatedTableId),
+    definition: {
+      doc1: SQLiteType.BLOB,
+      doc2: SQLiteType.BLOB,
+      tableId: SQLiteType.TEXT,
+    },
+  }
+}
+
+// this can generate relationship tables as part of the mapping
+function mapTable(table: Table): SQLiteTables {
+  const tables: SQLiteTables = {}
   const fields: Record<string, SQLiteType> = {}
   for (let [key, column] of Object.entries(table.schema)) {
+    // relationships should be handled differently
+    if (column.type === FieldType.LINK) {
+      const { tableId, definition } = buildRelationshipDefinitions(
+        table,
+        column
+      )
+      tables[tableId] = { fields: definition }
+    }
     if (!FieldTypeMap[column.type]) {
       throw new Error(`Unable to map type "${column.type}" to SQLite type`)
     }
@@ -49,10 +89,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } {
   CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
     constantMap[col] = SQLiteType.TEXT
   })
-  return {
+  const thisTable: SQLiteTable = {
     ...constantMap,
     ...fields,
   }
+  tables[table._id!] = { fields: thisTable }
+  return tables
 }
 
 // nothing exists, need to iterate though existing tables
@@ -60,8 +102,9 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
   const definition = cloneDeep(BASIC_SQLITE_DOC)
   for (let table of tables) {
-    definition.sql.tables[table._id!] = {
-      fields: mapTable(table),
+    definition.sql.tables = {
+      ...definition.sql.tables,
+      ...mapTable(table),
     }
   }
   return definition
@@ -75,8 +118,9 @@ export async function addTableToSqlite(table: Table) {
   } catch (err) {
     definition = await buildBaseDefinition()
   }
-  definition.sql.tables[table._id!] = {
-    fields: mapTable(table),
+  definition.sql.tables = {
+    ...definition.sql.tables,
+    ...mapTable(table),
   }
   await db.put(definition)
 }
diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts
index 76c47bbd74..e23a68b336 100644
--- a/packages/types/src/documents/app/sqlite.ts
+++ b/packages/types/src/documents/app/sqlite.ts
@@ -6,17 +6,23 @@ export enum SQLiteType {
   NUMERIC = "NUMERIC",
 }
 
+export type SQLiteTable = Record<
+  string,
+  SQLiteType | { field: string; type: SQLiteType }
+>
+
+export type SQLiteTables = Record<
+  string,
+  {
+    fields: SQLiteTable
+  }
+>
+
 export interface SQLiteDefinition {
   _id: string
   language: string
   sql: {
-    tables: {
-      [tableName: string]: {
-        fields: {
-          [key: string]: SQLiteType | { field: string; type: SQLiteType }
-        }
-      }
-    }
+    tables: SQLiteTables
     options: {
       table_name: string
     }

From d6b252013b80d22bb87677a8494c55663c799de6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 11 Apr 2024 18:25:18 +0100
Subject: [PATCH 07/45] Quick fix to link document structure in sqlite.

---
 packages/server/src/sdk/app/tables/internal/sqs.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 5dd16f516c..99240c28d4 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -59,8 +59,12 @@ function buildRelationshipDefinitions(
   return {
     tableId: generateJunctionTableID(tableId, relatedTableId),
     definition: {
-      doc1: SQLiteType.BLOB,
-      doc2: SQLiteType.BLOB,
+      ["doc1.rowId"]: SQLiteType.TEXT,
+      ["doc1.tableId"]: SQLiteType.TEXT,
+      ["doc1.fieldName"]: SQLiteType.TEXT,
+      ["doc2.rowId"]: SQLiteType.TEXT,
+      ["doc2.tableId"]: SQLiteType.TEXT,
+      ["doc2.fieldName"]: SQLiteType.TEXT,
       tableId: SQLiteType.TEXT,
     },
   }

From e3d427bc821f6b3f78c6b9d941540ef17f8715cb Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Fri, 12 Apr 2024 09:55:14 +0200
Subject: [PATCH 08/45] Fix a problem with missed tenantId for SSO in
 self-hosted

---
 packages/types/src/documents/account/account.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts
index 2f74b9e7b3..239d845722 100644
--- a/packages/types/src/documents/account/account.ts
+++ b/packages/types/src/documents/account/account.ts
@@ -102,6 +102,7 @@ export function isVerifiableSSOProvider(provider: AccountSSOProvider): boolean {
 }
 
 export interface AccountSSO {
+  ssoId?: string
   provider: AccountSSOProvider
   providerType: AccountSSOProviderType
   oauth2?: OAuthTokens

From 859bda0a5128d74e4ddddd239129bfa7717af97b Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Fri, 12 Apr 2024 09:24:12 +0100
Subject: [PATCH 09/45] removes unecessary side-panel-open actions

---
 packages/client/manifest.json                       | 5 -----
 packages/client/src/components/app/SidePanel.svelte | 2 --
 2 files changed, 7 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index bdcbdecb7c..622381847c 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6732,11 +6732,6 @@
         "label": "Click away to close",
         "defaultValue": true
       },
-      {
-        "type": "event",
-        "key": "sidePanelOpen",
-        "label": "Side Panel Open"
-      },
       {
         "type": "event",
         "key": "sidePanelClose",
diff --git a/packages/client/src/components/app/SidePanel.svelte b/packages/client/src/components/app/SidePanel.svelte
index 98398c4671..8e0dcd99e4 100644
--- a/packages/client/src/components/app/SidePanel.svelte
+++ b/packages/client/src/components/app/SidePanel.svelte
@@ -5,7 +5,6 @@
   const { styleable, sidePanelStore, builderStore, dndIsDragging } =
     getContext("sdk")
 
-  export let sidePanelOpen
   export let sidePanelClose
   export let clickOutsideToClose
 
@@ -67,7 +66,6 @@
       if (visible) {
         if (!target.contains(node)) {
           target.appendChild(node)
-          handleSidePanelOpen()
         }
       } else {
         if (target.contains(node)) {

From bdd487f475b44a2f7526f144e3d0094f66783c8c Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Fri, 12 Apr 2024 12:06:41 +0200
Subject: [PATCH 10/45] Update account portal submodule

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index a0ee9cad8c..edd67c7653 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit a0ee9cad8cefb8f9f40228705711be174f018fa9
+Subproject commit edd67c7653a4ca4daa39f907d4045dd22892f819

From 245fec960f815e8e0cd50efc318ec44c83d40d43 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 12 Apr 2024 11:17:05 +0100
Subject: [PATCH 11/45] Allow specifying extra volume mounts for containers in
 our Helm chart.

---
 .../templates/app-service-deployment.yaml     |  8 +++
 .../automation-worker-service-deployment.yaml |  8 +++
 .../templates/minio-service-deployment.yaml   |  6 ++
 .../templates/proxy-service-deployment.yaml   |  7 +++
 .../templates/redis-service-deployment.yaml   |  8 ++-
 .../templates/worker-service-deployment.yaml  |  8 +++
 charts/budibase/values.yaml                   | 62 +++++++++++++++++++
 7 files changed, 106 insertions(+), 1 deletion(-)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 2fd8506e30..b380908dd1 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -235,6 +235,10 @@ spec:
         args:
         {{- toYaml .Values.services.apps.args | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.apps.extraVolumeMounts }}
+        volumeMounts:
+        {{- toYaml .Values.services.apps.extraVolumeMounts | nindent 10 }}
+        {{- end }}
       {{- if .Values.services.apps.extraContainers }}
         {{- toYaml .Values.services.apps.extraContainers | nindent 6 }}
       {{- end }}
@@ -261,4 +265,8 @@ spec:
           - name: ndots
             value: {{ .Values.services.apps.ndots | quote }}
       {{ end }}
+      {{ if .Values.services.apps.extraVolumes }}
+      volumes:
+      {{- toYaml .Values.services.apps.extraVolumes | nindent 6 }}
+      {{- end }}
 status: {}
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index 53d5fcc860..51fa9ee4bb 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -235,6 +235,10 @@ spec:
         args:
         {{- toYaml .Values.services.automationWorkers.args | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.automationWorkers.extraVolumeMounts }}
+        volumeMounts:
+        {{- toYaml .Values.services.automationWorkers.extraVolumeMounts | nindent 10 }}
+        {{ end }}
       {{- if .Values.services.automationWorkers.extraContainers }}
         {{- toYaml .Values.services.automationWorkers.extraContainers | nindent 6 }}
       {{- end }}
@@ -261,5 +265,9 @@ spec:
           - name: ndots
             value: {{ .Values.services.automationWorkers.ndots | quote }}
       {{ end }}
+      {{ if .Values.services.automationWorkers.extraVolumes }}
+      volumes:
+      {{- toYaml .Values.services.automationWorkers.extraVolumes | nindent 8 }}
+      {{ end }}
 status: {}
 {{- end }}
\ No newline at end of file
diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml
index ade1d37cd2..901ead2b46 100644
--- a/charts/budibase/templates/minio-service-deployment.yaml
+++ b/charts/budibase/templates/minio-service-deployment.yaml
@@ -54,6 +54,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: minio-data
+        {{ if .Values.services.objectStore.extraVolumeMounts }}
+          {{- toYaml .Values.services.objectStore.extraVolumeMounts | nindent 8 }}
+        {{- end }}
       {{- if .Values.services.objectStore.extraContainers }}
         {{- toYaml .Values.services.objectStore.extraContainers | nindent 6 }}
       {{- end }}
@@ -78,5 +81,8 @@ spec:
       - name: minio-data
         persistentVolumeClaim:
           claimName: minio-data
+      {{ if .Values.services.objectStore.extraVolumes }}
+        {{- toYaml .Values.services.objectStore.extraVolumes | nindent 6 }}
+      {{- end }}
 status: {}
 {{- end }}
diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml
index 462c6a0749..d5ea696431 100644
--- a/charts/budibase/templates/proxy-service-deployment.yaml
+++ b/charts/budibase/templates/proxy-service-deployment.yaml
@@ -82,6 +82,10 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.proxy.extraVolumeMounts }}
+        volumeMounts:
+        {{- toYaml .Values.services.proxy.extraVolumeMounts | nindent 8 }}
+        {{- end }}
       {{- if .Values.services.proxy.extraContainers }}
         {{- toYaml .Values.services.proxy.extraContainers | nindent 6 }}
       {{- end }}
@@ -110,7 +114,10 @@ spec:
       args:
       {{- toYaml .Values.services.proxy.args | nindent 8 }}
       {{ end }}
+      {{ if .Values.services.proxy.extraVolumes }}
       volumes:
+      {{- toYaml .Values.services.proxy.extraVolumes | nindent 6 }}
+      {{ end }}
       {{ if .Values.services.proxy.ndots }}
       dnsConfig:
         options:
diff --git a/charts/budibase/templates/redis-service-deployment.yaml b/charts/budibase/templates/redis-service-deployment.yaml
index 1a003d3814..9ad12e0167 100644
--- a/charts/budibase/templates/redis-service-deployment.yaml
+++ b/charts/budibase/templates/redis-service-deployment.yaml
@@ -22,7 +22,7 @@ spec:
         - redis-server 
         - --requirepass 
         - {{ .Values.services.redis.password }}
-        image: redis
+        image: {{ .Values.services.redis.image }}
         imagePullPolicy: ""
         name: redis-service
         ports:
@@ -34,6 +34,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: redis-data
+        {{ if .Values.services.redis.extraVolumeMounts }}
+          {{- toYaml .Values.services.redis.extraVolumeMounts | nindent 8 }}
+        {{- end }}
     {{- if .Values.services.redis.extraContainers }}
       {{- toYaml .Values.services.redis.extraContainers | nindent 6 }}
     {{- end }}
@@ -58,6 +61,9 @@ spec:
       - name: redis-data
         persistentVolumeClaim:
           claimName: redis-data
+      {{ if .Values.services.redis.extraVolumes }}
+        {{- toYaml .Values.services.redis.extraVolumes | nindent 6 }}
+      {{- end }}
 
 status: {}
 {{- end }}
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index cc27bf429e..e37b2bc0e4 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -221,6 +221,10 @@ spec:
         args:
         {{- toYaml .Values.services.worker.args | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.worker.extraVolumeMounts }}
+        volumeMounts:
+        {{- toYaml .Values.services.worker.extraVolumeMounts | nindent 10 }}
+        {{- end }}
       {{- if .Values.services.worker.extraContainers }}
         {{- toYaml .Values.services.worker.extraContainers | nindent 6 }}
       {{- end }}
@@ -247,4 +251,8 @@ spec:
           - name: ndots
             value: {{ .Values.services.worker.ndots | quote }}
       {{ end }}
+      {{ if .Values.services.worker.extraVolumes }}
+      volumes:
+      {{- toYaml .Values.services.worker.extraVolumes | nindent 6 }}
+      {{- end }}
 status: {}
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index d368c0ff38..616b847272 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -211,6 +211,16 @@ services:
     #  - name: my-sidecar
     #    image: myimage:latest
 
+    # -- Additional volumeMounts to the main proxy container.
+    extraVolumeMounts: []
+    #  - name: my-volume
+    #   mountPath: /path/to/mount
+
+    # -- Additional volumes to the proxy pod.
+    extraVolumes: []
+    #  - name: my-volume
+    #    emptyDir: {}
+
   apps:
     # @ignore (you shouldn't need to change this)
     port: 4002
@@ -283,6 +293,16 @@ services:
     #  - name: my-sidecar
     #    image: myimage:latest
 
+    # -- Additional volumeMounts to the main apps container.
+    extraVolumeMounts: []
+    #  - name: my-volume
+    #   mountPath: /path/to/mount
+
+    # -- Additional volumes to the apps pod.
+    extraVolumes: []
+    #  - name: my-volume
+    #    emptyDir: {}
+
   automationWorkers:
     # -- Whether or not to enable the automation worker service. If you disable this,
     # automations will be processed by the apps service.
@@ -359,6 +379,16 @@ services:
     #  - name: my-sidecar
     #    image: myimage:latest
 
+    # -- Additional volumeMounts to the main automationWorkers container.
+    extraVolumeMounts: []
+    #  - name: my-volume
+    #   mountPath: /path/to/mount
+
+    # -- Additional volumes to the automationWorkers pod.
+    extraVolumes: []
+    #  - name: my-volume
+    #    emptyDir: {}
+
   worker:
     # @ignore (you shouldn't need to change this)
     port: 4003
@@ -431,6 +461,16 @@ services:
     #  - name: my-sidecar
     #    image: myimage:latest
 
+    # -- Additional volumeMounts to the main worker container.
+    extraVolumeMounts: []
+    #  - name: my-volume
+    #   mountPath: /path/to/mount
+
+    # -- Additional volumes to the worker pod.
+    extraVolumes: []
+    #  - name: my-volume
+    #    emptyDir: {}
+
   couchdb:
     # -- Whether or not to spin up a CouchDB instance in your cluster. True by
     # default, and the configuration for the CouchDB instance is under the
@@ -456,6 +496,8 @@ services:
       resources: {}
 
   redis:
+    # -- The Redis image to use.
+    image: redis
     # -- Whether or not to deploy a Redis pod into your cluster.
     enabled: true
     # -- Port to expose Redis on.
@@ -484,6 +526,16 @@ services:
     #  - name: my-sidecar
     #    image: myimage:latest
 
+    # -- Additional volumeMounts to the main redis container.
+    extraVolumeMounts: []
+    #  - name: my-volume
+    #   mountPath: /path/to/mount
+
+    # -- Additional volumes to the redis pod.
+    extraVolumes: []
+    #  - name: my-volume
+    #    emptyDir: {}
+
   objectStore:
     # -- Set to false if using another object store, such as S3. You will need
     # to set `services.objectStore.url` to point to your bucket if you do this.
@@ -530,6 +582,16 @@ services:
     #  - name: my-sidecar
     #    image: myimage:latest
 
+    # -- Additional volumeMounts to the main objectStore container.
+    extraVolumeMounts: []
+    #  - name: my-volume
+    #   mountPath: /path/to/mount
+
+    # -- Additional volumes to the objectStore pod.
+    extraVolumes: []
+    #  - name: my-volume
+    #    emptyDir: {}
+
 # Override values in couchDB subchart. We're only specifying the values we're changing.
 # If you want to see all of the available values, see:
 #   https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb

From bcec05cd089116ad7ffebab096481e51533cff8f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 12 Apr 2024 11:46:09 +0100
Subject: [PATCH 12/45] Regenerate README.

---
 charts/budibase/README.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/charts/budibase/README.md b/charts/budibase/README.md
index b803da18a4..207992087d 100644
--- a/charts/budibase/README.md
+++ b/charts/budibase/README.md
@@ -152,6 +152,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. |
 | services.apps.extraContainers | list | `[]` | Additional containers to be added to the apps pod. |
 | services.apps.extraEnv | list | `[]` | Extra environment variables to set for apps pods. Takes a list of name=value pairs. |
+| services.apps.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main apps container. |
+| services.apps.extraVolumes | list | `[]` | Additional volumes to the apps pod. |
 | services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. |
 | services.apps.livenessProbe | object | HTTP health checks. | Liveness probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.apps.logLevel | string | `"info"` | The log level for the apps service. |
@@ -166,6 +168,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
 | services.automationWorkers.extraContainers | list | `[]` | Additional containers to be added to the automationWorkers pod. |
 | services.automationWorkers.extraEnv | list | `[]` | Extra environment variables to set for automation worker pods. Takes a list of name=value pairs. |
+| services.automationWorkers.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main automationWorkers container. |
+| services.automationWorkers.extraVolumes | list | `[]` | Additional volumes to the automationWorkers pod. |
 | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. |
 | services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
@@ -185,6 +189,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.objectStore.cloudfront.privateKey64 | string | `""` | Base64 encoded private key for the above public key. |
 | services.objectStore.cloudfront.publicKeyId | string | `""` | ID of public key stored in cloudfront. |
 | services.objectStore.extraContainers | list | `[]` | Additional containers to be added to the objectStore pod. |
+| services.objectStore.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main objectStore container. |
+| services.objectStore.extraVolumes | list | `[]` | Additional volumes to the objectStore pod. |
 | services.objectStore.minio | bool | `true` | Set to false if using another object store, such as S3. You will need to set `services.objectStore.url` to point to your bucket if you do this. |
 | services.objectStore.region | string | `""` | AWS_REGION if using S3 |
 | services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
@@ -197,6 +203,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.proxy.autoscaling.minReplicas | int | `1` |  |
 | services.proxy.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the proxy service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the proxy pods. |
 | services.proxy.extraContainers | list | `[]` |  |
+| services.proxy.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main proxy container. |
+| services.proxy.extraVolumes | list | `[]` | Additional volumes to the proxy pod. |
 | services.proxy.livenessProbe | object | HTTP health checks. | Liveness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.proxy.readinessProbe | object | HTTP health checks. | Readiness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.proxy.replicaCount | int | `1` | The number of proxy replicas to run. |
@@ -204,6 +212,9 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.proxy.startupProbe | object | HTTP health checks. | Startup probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.redis.enabled | bool | `true` | Whether or not to deploy a Redis pod into your cluster. |
 | services.redis.extraContainers | list | `[]` | Additional containers to be added to the redis pod. |
+| services.redis.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main redis container. |
+| services.redis.extraVolumes | list | `[]` | Additional volumes to the redis pod. |
+| services.redis.image | string | `"redis"` | The Redis image to use. |
 | services.redis.password | string | `"budibase"` | The password to use when connecting to Redis. It's recommended that you change this from the default if you're running Redis in-cluster. |
 | services.redis.port | int | `6379` | Port to expose Redis on. |
 | services.redis.resources | object | `{}` | The resources to use for Redis pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
@@ -216,6 +227,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. |
 | services.worker.extraContainers | list | `[]` | Additional containers to be added to the worker pod. |
 | services.worker.extraEnv | list | `[]` | Extra environment variables to set for worker pods. Takes a list of name=value pairs. |
+| services.worker.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main worker container. |
+| services.worker.extraVolumes | list | `[]` | Additional volumes to the worker pod. |
 | services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. |
 | services.worker.livenessProbe | object | HTTP health checks. | Liveness probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.worker.logLevel | string | `"info"` | The log level for the worker service. |

From 1406b057404237f768f9e3784652200e9ac4248a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 12 Apr 2024 13:10:58 +0200
Subject: [PATCH 13/45] Remove endpoints

---
 packages/server/src/api/controllers/static/index.ts |  7 -------
 packages/server/src/api/routes/static.ts            | 11 -----------
 2 files changed, 18 deletions(-)

diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index d767ca9e98..7cc08e1b5c 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -127,13 +127,6 @@ export const uploadFile = async function (
   )
 }
 
-export const deleteObjects = async function (ctx: Ctx) {
-  ctx.body = await objectStore.deleteFiles(
-    ObjectStoreBuckets.APPS,
-    ctx.request.body.keys
-  )
-}
-
 const requiresMigration = async (ctx: Ctx) => {
   const appId = context.getAppId()
   if (!appId) {
diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts
index a5c421b2e6..f331609923 100644
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@@ -32,11 +32,6 @@ router
   .get("/builder/:file*", controller.serveBuilder)
   .get("/api/assets/client", controller.serveClientLibrary)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
-  .post(
-    "/api/attachments/delete",
-    authorized(BUILDER),
-    controller.deleteObjects
-  )
   .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
@@ -44,12 +39,6 @@ router
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     controller.uploadFile
   )
-  .post(
-    "/api/attachments/:tableId/delete",
-    paramResource("tableId"),
-    authorized(PermissionType.TABLE, PermissionLevel.WRITE),
-    controller.deleteObjects
-  )
   .get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview)
   .get("/app/:appUrl/:path*", controller.serveApp)
   .get("/:appId/:path*", controller.serveApp)

From 068c8b8c66b7ed3a9c5cfbde336836f99994065b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 12 Apr 2024 13:11:16 +0200
Subject: [PATCH 14/45] Remove from frontend

---
 .../src/components/common/Dropzone.svelte     |  9 -------
 .../app/forms/AttachmentField.svelte          | 12 ---------
 packages/frontend-core/src/api/attachments.js | 27 -------------------
 .../grid/cells/AttachmentCell.svelte          |  9 -------
 4 files changed, 57 deletions(-)

diff --git a/packages/builder/src/components/common/Dropzone.svelte b/packages/builder/src/components/common/Dropzone.svelte
index daa6ad1807..a864e1d028 100644
--- a/packages/builder/src/components/common/Dropzone.svelte
+++ b/packages/builder/src/components/common/Dropzone.svelte
@@ -27,14 +27,6 @@
       return []
     }
   }
-
-  async function deleteAttachments(fileList) {
-    try {
-      return await API.deleteBuilderAttachments(fileList)
-    } catch (error) {
-      return []
-    }
-  }
 </script>
 
 <Dropzone
@@ -42,6 +34,5 @@
   {label}
   {...$$restProps}
   {processFiles}
-  {deleteAttachments}
   {handleFileTooLarge}
 />
diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte
index 644630810d..3489fd809c 100644
--- a/packages/client/src/components/app/forms/AttachmentField.svelte
+++ b/packages/client/src/components/app/forms/AttachmentField.svelte
@@ -58,17 +58,6 @@
     }
   }
 
-  const deleteAttachments = async fileList => {
-    try {
-      return await API.deleteAttachments({
-        keys: fileList,
-        tableId: formContext?.dataSource?.tableId,
-      })
-    } catch (error) {
-      return []
-    }
-  }
-
   const handleChange = e => {
     const value = fieldApiMapper.set(e.detail)
     const changed = fieldApi.setValue(value)
@@ -98,7 +87,6 @@
       error={fieldState.error}
       on:change={handleChange}
       {processFiles}
-      {deleteAttachments}
       {handleFileTooLarge}
       {handleTooManyFiles}
       {maximum}
diff --git a/packages/frontend-core/src/api/attachments.js b/packages/frontend-core/src/api/attachments.js
index f79b461574..e3b1b74e5b 100644
--- a/packages/frontend-core/src/api/attachments.js
+++ b/packages/frontend-core/src/api/attachments.js
@@ -61,32 +61,5 @@ export const buildAttachmentEndpoints = API => {
       })
       return { publicUrl }
     },
-
-    /**
-     * Deletes attachments from the bucket.
-     * @param keys the attachments to delete
-     * @param tableId the associated table ID
-     */
-    deleteAttachments: async ({ keys, tableId }) => {
-      return await API.post({
-        url: `/api/attachments/${tableId}/delete`,
-        body: {
-          keys,
-        },
-      })
-    },
-
-    /**
-     * Deletes attachments from the builder bucket.
-     * @param keys the attachments to delete
-     */
-    deleteBuilderAttachments: async keys => {
-      return await API.post({
-        url: `/api/attachments/delete`,
-        body: {
-          keys,
-        },
-      })
-    },
   }
 }
diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index 3a1f165b6e..e7dc51e5d5 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -61,14 +61,6 @@
     }
   }
 
-  const deleteAttachments = async fileList => {
-    try {
-      return await API.deleteBuilderAttachments(fileList)
-    } catch (error) {
-      return []
-    }
-  }
-
   onMount(() => {
     api = {
       focus: () => open(),
@@ -101,7 +93,6 @@
       on:change={e => onChange(e.detail)}
       maximum={maximum || schema.constraints?.length?.maximum}
       {processFiles}
-      {deleteAttachments}
       {handleFileTooLarge}
     />
   </div>

From 205858c6d7aa5119d62336bbf86b5c72236dc72b Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Fri, 12 Apr 2024 13:27:41 +0200
Subject: [PATCH 15/45] Account portal submodule to master

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index edd67c7653..bd0e01d639 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit edd67c7653a4ca4daa39f907d4045dd22892f819
+Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344

From 0514641f049285e6139b59a54744d1e79d103105 Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Fri, 12 Apr 2024 13:24:21 +0100
Subject: [PATCH 16/45] removes unecessary on-sidepanel-open actions feature.

---
 packages/client/manifest.json                    |  2 +-
 packages/client/src/components/app/Layout.svelte | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 622381847c..f370b67670 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6729,7 +6729,7 @@
       {
         "type": "boolean",
         "key": "clickOutsideToClose",
-        "label": "Click away to close",
+        "label": "Click outside to close",
         "defaultValue": true
       },
       {
diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte
index acef8001a8..bae2bd0faf 100644
--- a/packages/client/src/components/app/Layout.svelte
+++ b/packages/client/src/components/app/Layout.svelte
@@ -73,7 +73,10 @@
     $context.device.width,
     $context.device.height
   )
-  $: autoCloseSidePanel = !$builderStore.inBuilder && $sidePanelStore.open
+  $: autoCloseSidePanel =
+    !$builderStore.inBuilder &&
+    $sidePanelStore.open &&
+    $sidePanelStore.clickOutsideToClose
   $: screenId = $builderStore.inBuilder
     ? `${$builderStore.screen?._id}-screen`
     : "screen"
@@ -317,11 +320,7 @@
     id="side-panel-container"
     class:open={$sidePanelStore.open}
     use:clickOutside={{
-      callback:
-        $sidePanelStore.clickOutsideToClose && autoCloseSidePanel
-          ? sidePanelStore.actions.close
-          : null,
-
+      callback: autoCloseSidePanel ? sidePanelStore.actions.close : null,
       allowedType: "mousedown",
     }}
     class:builder={$builderStore.inBuilder}

From 5a1de4b45cbed80f743ce1ab2c15c151ff47bfe7 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 12 Apr 2024 13:24:45 +0000
Subject: [PATCH 17/45] Bump version to 2.23.5

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 78a3aa13e9..9839b8b166 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.23.4",
+  "version": "2.23.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From 1632c9d7a84f74e94fa07a0615517e49fd3cc992 Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Fri, 12 Apr 2024 14:33:46 +0100
Subject: [PATCH 18/45] removes unused function

---
 packages/client/src/components/app/SidePanel.svelte | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/client/src/components/app/SidePanel.svelte b/packages/client/src/components/app/SidePanel.svelte
index 8e0dcd99e4..624617ad69 100644
--- a/packages/client/src/components/app/SidePanel.svelte
+++ b/packages/client/src/components/app/SidePanel.svelte
@@ -47,12 +47,6 @@
     }
   }
 
-  const handleSidePanelOpen = async () => {
-    if (sidePanelOpen) {
-      await sidePanelOpen()
-    }
-  }
-
   const handleSidePanelClose = async () => {
     if (sidePanelClose) {
       await sidePanelClose()

From 565ee5f7dac0edee614ba841eeef49adf152dfe3 Mon Sep 17 00:00:00 2001
From: mikesealey <mikesealey@hotmail.com>
Date: Fri, 12 Apr 2024 15:23:24 +0100
Subject: [PATCH 19/45] brings key and label into line with standard practices.
 Removes unecessary info.

---
 packages/client/manifest.json | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index f370b67670..c9e28e202b 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6734,9 +6734,8 @@
       },
       {
         "type": "event",
-        "key": "sidePanelClose",
-        "label": "Side Panel Close",
-        "info": "Side panel actions configured here will run after the 'Open side panel' action runs, and are not capable of preventing or stopping it. Any form validation should therefore be done before that action is invoked, and not here."
+        "key": "onSidePanelClose",
+        "label": "On side panel close"
       }
     ]
   },

From ebb79c16fe7d626d9d201fe5d1090da6e4ab078f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:15:36 +0100
Subject: [PATCH 20/45] Aliasing support for SQS.

---
 .../api/controllers/row/ExternalRequest.ts    |  3 +-
 packages/server/src/db/utils.ts               |  1 +
 .../src/integrations/tests/sqlAlias.spec.ts   |  4 +-
 packages/server/src/sdk/app/rows/index.ts     |  2 +
 .../server/src/sdk/app/rows/search/sqs.ts     | 99 +++++++++++--------
 .../row/alias.ts => sdk/app/rows/sqlAlias.ts} | 28 ++++--
 6 files changed, 86 insertions(+), 51 deletions(-)
 rename packages/server/src/{api/controllers/row/alias.ts => sdk/app/rows/sqlAlias.ts} (87%)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 7fc0333de1..4adbb72c7a 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { db as dbCore } from "@budibase/backend-core"
-import AliasTables from "./alias"
 import sdk from "../../../sdk"
 import env from "../../../environment"
 
@@ -618,7 +617,7 @@ export class ExternalRequest<T extends Operation> {
     if (env.SQL_ALIASING_DISABLE) {
       response = await getDatasourceAndQuery(json)
     } else {
-      const aliasing = new AliasTables(Object.keys(this.tables))
+      const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
       response = await aliasing.queryWithAliasing(json)
     }
 
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index b1c02b1764..ce8d0accbb 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
+export const SQS_DATASOURCE_INTERNAL = "internal"
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index bfca24ff7d..58c3a05245 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -8,8 +8,10 @@ import {
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
-import AliasTables from "../../api/controllers/row/alias"
 import { generator } from "@budibase/backend-core/tests"
+import sdk from "../../sdk"
+
+const AliasTables = sdk.rows.AliasTables
 
 function multiline(sql: string) {
   return sql.replace(/\n/g, "").replace(/ +/g, " ")
diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts
index ea501e93d9..c117941419 100644
--- a/packages/server/src/sdk/app/rows/index.ts
+++ b/packages/server/src/sdk/app/rows/index.ts
@@ -3,6 +3,7 @@ import * as rows from "./rows"
 import * as search from "./search"
 import * as utils from "./utils"
 import * as external from "./external"
+import AliasTables from "./sqlAlias"
 
 export default {
   ...attachments,
@@ -10,4 +11,5 @@ export default {
   ...search,
   utils,
   external,
+  AliasTables,
 }
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 5b0b6e3bc7..20edb988d3 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -20,7 +20,12 @@ import {
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
 import { context } from "@budibase/backend-core"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  SQS_DATASOURCE_INTERNAL,
+} from "../../../../db/utils"
+import AliasTables from "../sqlAlias"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
 
 function buildInternalFieldList(
   table: Table,
@@ -31,19 +36,19 @@ function buildInternalFieldList(
   fieldList = fieldList.concat(
     CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
   )
-  if (opts.relationships) {
-    for (let col of Object.values(table.schema)) {
-      if (col.type === FieldType.LINK) {
-        const linkCol = col as RelationshipFieldMetadata
-        const relatedTable = tables.find(
-          table => table._id === linkCol.tableId
-        )!
-        fieldList = fieldList.concat(
-          buildInternalFieldList(relatedTable, tables, { relationships: false })
-        )
-      } else {
-        fieldList.push(`${table._id}.${col.name}`)
-      }
+  for (let col of Object.values(table.schema)) {
+    const isRelationship = col.type === FieldType.LINK
+    if (!opts.relationships && isRelationship) {
+      continue
+    }
+    if (isRelationship) {
+      const linkCol = col as RelationshipFieldMetadata
+      const relatedTable = tables.find(table => table._id === linkCol.tableId)!
+      fieldList = fieldList.concat(
+        buildInternalFieldList(relatedTable, tables, { relationships: false })
+      )
+    } else {
+      fieldList.push(`${table._id}.${col.name}`)
     }
   }
   return fieldList
@@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) {
 }
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
-  const { tableId, paginate, query, ...params } = options
+  const { paginate, query, ...params } = options
 
   const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
-  const table = allTables.find(table => table._id === tableId)
   if (!table) {
     throw new Error("Unable to find table")
   }
@@ -111,7 +116,7 @@ export async function search(
   const request: QueryJson = {
     endpoint: {
       // not important, we query ourselves
-      datasourceId: "internal",
+      datasourceId: SQS_DATASOURCE_INTERNAL,
       entityId: table._id!,
       operation: Operation.READ,
     },
@@ -154,34 +159,44 @@ export async function search(
     }
   }
   try {
-    const query = builder._query(request, {
-      disableReturning: true,
+    const alias = new AliasTables(allTables.map(table => table.name))
+    const rows = await alias.queryWithAliasing(request, async json => {
+      const query = builder._query(json, {
+        disableReturning: true,
+      })
+
+      if (Array.isArray(query)) {
+        throw new Error("SQS cannot currently handle multiple queries")
+      }
+
+      let sql = query.sql,
+        bindings = query.bindings
+
+      // quick hack for docIds
+      sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+      sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+      const db = context.getAppDB()
+      return await db.sql<Row>(sql, bindings)
     })
 
-    if (Array.isArray(query)) {
-      throw new Error("SQS cannot currently handle multiple queries")
-    }
-
-    let sql = query.sql,
-      bindings = query.bindings
-
-    // quick hack for docIds
-    sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
-    sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
-
-    const db = context.getAppDB()
-    const rows = await db.sql<Row>(sql, bindings)
+    // process from the format of tableId.column to expected format
+    const processed = await sqlOutputProcessing(
+      rows,
+      table!,
+      allTablesMap,
+      relationships,
+      {
+        sqs: true,
+      }
+    )
 
     return {
-      rows: await sqlOutputProcessing(
-        rows,
-        table!,
-        allTablesMap,
-        relationships,
-        {
-          sqs: true,
-        }
-      ),
+      // final row processing for response
+      rows: await outputProcessing<Row[]>(table, processed, {
+        preserveLinks: true,
+        squash: true,
+      }),
     }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts
similarity index 87%
rename from packages/server/src/api/controllers/row/alias.ts
rename to packages/server/src/sdk/app/rows/sqlAlias.ts
index 0ec9d1a09c..0fc338ecbe 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/sdk/app/rows/sqlAlias.ts
@@ -6,11 +6,12 @@ import {
   Row,
   SearchFilters,
 } from "@budibase/types"
-import { getSQLClient } from "../../../sdk/app/rows/utils"
+import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
-import sdk from "../../../sdk"
+import datasources from "../datasources"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { SqlClient } from "../../../integrations/utils"
+import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
 
 const WRITE_OPERATIONS: Operation[] = [
   Operation.CREATE,
@@ -156,12 +157,19 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(
-    json: QueryJson
+    json: QueryJson,
+    queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
   ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
-    const datasource = await sdk.datasources.get(datasourceId)
+    const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
+    let aliasingEnabled: boolean, datasource: Datasource | undefined
+    if (isSqs) {
+      aliasingEnabled = true
+    } else {
+      datasource = await datasources.get(datasourceId)
+      aliasingEnabled = this.isAliasingEnabled(json, datasource)
+    }
 
-    const aliasingEnabled = this.isAliasingEnabled(json, datasource)
     if (aliasingEnabled) {
       json = cloneDeep(json)
       // run through the query json to update anywhere a table may be used
@@ -207,7 +215,15 @@ export default class AliasTables {
       }
       json.tableAliases = invertedTableAliases
     }
-    const response = await makeExternalQuery(datasource, json)
+
+    let response: DatasourcePlusQueryResponse
+    if (datasource && !isSqs) {
+      response = await makeExternalQuery(datasource, json)
+    } else if (queryFn) {
+      response = await queryFn(json)
+    } else {
+      throw new Error("No supplied method to perform aliased query")
+    }
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {

From c40e9656345f5e95ead009976ab00755973b4845 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:16:31 +0100
Subject: [PATCH 21/45] Getting relationships working properly as well as
 renaming internal -> sqs in function opts.

---
 packages/server/src/api/controllers/row/utils/basic.ts   | 9 +++++----
 .../server/src/api/controllers/row/utils/sqlUtils.ts     | 4 ++--
 packages/server/src/api/controllers/row/utils/utils.ts   | 6 ++++--
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index 1fc84de9c7..6255e13c1c 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -62,12 +62,12 @@ export function basicProcessing({
   row,
   table,
   isLinked,
-  internal,
+  sqs,
 }: {
   row: Row
   table: Table
   isLinked: boolean
-  internal?: boolean
+  sqs?: boolean
 }): Row {
   const thisRow: Row = {}
   // filter the row down to what is actually the row (not joined)
@@ -84,12 +84,13 @@ export function basicProcessing({
       thisRow[fieldName] = value
     }
   }
-  if (!internal) {
+  if (!sqs) {
     thisRow._id = generateIdForRow(row, table, isLinked)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
   } else {
-    for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+    const columns = Object.keys(table.schema)
+    for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
       thisRow[internalColumn] = extractFieldValue({
         row,
         tableName: table._id!,
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 6f9837e0ab..372b8394ff 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -51,11 +51,11 @@ export async function updateRelationshipColumns(
       continue
     }
 
-    let linked = await basicProcessing({
+    let linked = basicProcessing({
       row,
       table: linkedTable,
       isLinked: true,
-      internal: opts?.sqs,
+      sqs: opts?.sqs,
     })
     if (!linked._id) {
       continue
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index f387a468cf..bf9ede6fe3 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -132,6 +132,7 @@ export async function sqlOutputProcessing(
     let rowId = row._id
     if (opts?.sqs) {
       rowId = getInternalRowId(row, table)
+      row._id = rowId
     } else if (!rowId) {
       rowId = generateIdForRow(row, table)
       row._id = rowId
@@ -153,7 +154,7 @@ export async function sqlOutputProcessing(
         row,
         table,
         isLinked: false,
-        internal: opts?.sqs,
+        sqs: opts?.sqs,
       }),
       table
     )
@@ -167,7 +168,8 @@ export async function sqlOutputProcessing(
       tables,
       row,
       finalRows,
-      relationships
+      relationships,
+      opts
     )
   }
 

From bfb7750213400e833bfb4ffeae0be9462b66e0bc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:17:06 +0100
Subject: [PATCH 22/45] Getting search input mapping up a level in the search
 SDK - avoids having to call it for every search type.

---
 packages/server/src/sdk/app/rows/search.ts    | 11 +++++++---
 .../src/sdk/app/rows/search/external.ts       | 13 +++++++-----
 .../src/sdk/app/rows/search/internal.ts       | 20 ++++++++++---------
 .../app/rows/search/tests/external.spec.ts    |  6 +++---
 .../app/rows/search/tests/internal.spec.ts    |  4 ++--
 5 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index f681bfeb90..5d8f7ef80b 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -13,6 +13,8 @@ import * as sqs from "./search/sqs"
 import env from "../../../environment"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
+import sdk from "../../index"
+import { searchInputMapping } from "./search/utils"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -72,12 +74,15 @@ export async function search(
     }
   }
 
+  const table = await sdk.tables.getTable(options.tableId)
+  options = searchInputMapping(table, options)
+
   if (isExternalTable) {
-    return external.search(options)
+    return external.search(options, table)
   } else if (env.SQS_SEARCH_ENABLE) {
-    return sqs.search(options)
+    return sqs.search(options, table)
   } else {
-    return internal.search(options)
+    return internal.search(options, table)
   }
 }
 
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index e0a3bad94e..077f971903 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -8,6 +8,7 @@ import {
   SearchFilters,
   RowSearchParams,
   SearchResponse,
+  Table,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
 import { handleRequest } from "../../../../api/controllers/row/external"
@@ -18,13 +19,13 @@ import {
 import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "./types"
 import { HTTPError, db } from "@budibase/backend-core"
-import { searchInputMapping } from "./utils"
 import pick from "lodash/pick"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import sdk from "../../../"
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
   const { tableId } = options
   const { paginate, query, ...params } = options
@@ -68,8 +69,6 @@ export async function search(
   }
 
   try {
-    const table = await sdk.tables.getTable(tableId)
-    options = searchInputMapping(table, options)
     let rows = await handleRequest(Operation.READ, tableId, {
       filters: query,
       sort,
@@ -150,11 +149,15 @@ export async function exportRows(
   }
 
   const datasource = await sdk.datasources.get(datasourceId!)
+  const table = await sdk.tables.getTable(tableId)
   if (!datasource || !datasource.entities) {
     throw new HTTPError("Datasource has not been configured for plus API.", 400)
   }
 
-  let result = await search({ tableId, query: requestQuery, sort, sortOrder })
+  let result = await search(
+    { tableId, query: requestQuery, sort, sortOrder },
+    table
+  )
   let rows: Row[] = []
   let headers
 
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 610807a10e..ffd13ed731 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -33,7 +33,8 @@ import pick from "lodash/pick"
 import { breakRowIdField } from "../../../../integrations/utils"
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
   const { tableId } = options
 
@@ -51,8 +52,6 @@ export async function search(
     query: {},
   }
 
-  let table = await sdk.tables.getTable(tableId)
-  options = searchInputMapping(table, options)
   if (params.sort && !params.sortType) {
     const schema = table.schema
     const sortField = schema[params.sort]
@@ -122,12 +121,15 @@ export async function exportRows(
 
     result = await outputProcessing<Row[]>(table, response)
   } else if (query) {
-    let searchResponse = await search({
-      tableId,
-      query,
-      sort,
-      sortOrder,
-    })
+    let searchResponse = await search(
+      {
+        tableId,
+        query,
+        sort,
+        sortOrder,
+      },
+      table
+    )
     result = searchResponse.rows
   }
 
diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
index f2bdec4692..53bc049a9b 100644
--- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts
@@ -112,7 +112,7 @@ describe("external search", () => {
         tableId,
         query: {},
       }
-      const result = await search(searchParams)
+      const result = await search(searchParams, config.table!)
 
       expect(result.rows).toHaveLength(10)
       expect(result.rows).toEqual(
@@ -130,7 +130,7 @@ describe("external search", () => {
         query: {},
         fields: ["name", "age"],
       }
-      const result = await search(searchParams)
+      const result = await search(searchParams, config.table!)
 
       expect(result.rows).toHaveLength(10)
       expect(result.rows).toEqual(
@@ -157,7 +157,7 @@ describe("external search", () => {
           },
         },
       }
-      const result = await search(searchParams)
+      const result = await search(searchParams, config.table!)
 
       expect(result.rows).toHaveLength(3)
       expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])
diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
index 5be0f4a258..1c5f396737 100644
--- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
@@ -81,7 +81,7 @@ describe("internal", () => {
           tableId,
           query: {},
         }
-        const result = await search(searchParams)
+        const result = await search(searchParams, config.table!)
 
         expect(result.rows).toHaveLength(10)
         expect(result.rows).toEqual(
@@ -99,7 +99,7 @@ describe("internal", () => {
           query: {},
           fields: ["name", "age"],
         }
-        const result = await search(searchParams)
+        const result = await search(searchParams, config.table!)
 
         expect(result.rows).toHaveLength(10)
         expect(result.rows).toEqual(

From 7d7de33cabbcfcefb7ea668a92813b96c1e28b3b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:29:48 +0100
Subject: [PATCH 23/45] Removing CouchDB SQS image for now.

---
 hosting/docker-compose.dev.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml
index 77f6bd053b..9dba5d427c 100644
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@@ -42,13 +42,12 @@ services:
   couchdb-service:
     container_name: budi-couchdb3-dev
     restart: on-failure
-    image: budibase/couchdb:v3.2.1-sqs
+    image: budibase/couchdb
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
-      - "${COUCH_DB_SQS_PORT}:4984"
     volumes:
       - couchdb_data:/data
 

From aeda5931c07c84ed826e5e37bd54b3382d955653 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 12 Apr 2024 16:34:33 +0100
Subject: [PATCH 24/45] Fixing lint.

---
 packages/server/src/sdk/app/rows/search/internal.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index ffd13ed731..906ca016d1 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -1,6 +1,6 @@
 import { context, db, HTTPError } from "@budibase/backend-core"
 import env from "../../../../environment"
-import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
+import { fullSearch, paginatedSearch } from "./utils"
 import { getRowParams, InternalTables } from "../../../../db/utils"
 import {
   Database,

From fbff5c0a316c0acf732fe97b61ee8f6c8edbdf6e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 12 Apr 2024 16:44:34 +0100
Subject: [PATCH 25/45] Rename toContains to toContainsExactly to better
 reflect what it does.

---
 .../src/api/routes/tests/search.spec.ts       | 114 ++++++++++++------
 1 file changed, 77 insertions(+), 37 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 4db47be216..a473cb77b4 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -64,7 +64,11 @@ describe.each([
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    async toMatch(expectedRows: any[]) {
+    // Asserts that the query returns rows matching exactly the set of rows
+    // passed in. The order of the rows matters. Rows returned in an order
+    // different to the one passed in will cause the assertion to fail.  Extra
+    // rows returned by the query will also cause the assertion to fail.
+    async toMatchExactly(expectedRows: any[]) {
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -82,7 +86,10 @@ describe.each([
       )
     }
 
-    async toContain(expectedRows: any[]) {
+    // Asserts that the query returns rows matching exactly the set of rows
+    // passed in. The order of the rows is not important, but extra rows will
+    // cause the assertion to fail.
+    async toContainExactly(expectedRows: any[]) {
       const { rows: foundRows } = await config.api.row.search(table._id!, {
         ...this.query,
         tableId: table._id!,
@@ -102,8 +109,29 @@ describe.each([
       )
     }
 
+    // Asserts that the query returns rows matching the set of rows passed in.
+    // The order of the rows is not important. Extra rows will not cause the
+    // assertion to fail.
+    async toContain(expectedRows: any[]) {
+      const { rows: foundRows } = await config.api.row.search(table._id!, {
+        ...this.query,
+        tableId: table._id!,
+      })
+
+      // eslint-disable-next-line jest/no-standalone-expect
+      expect(foundRows).toEqual(
+        expect.arrayContaining(
+          expectedRows.map((expectedRow: any) =>
+            expect.objectContaining(
+              foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
+            )
+          )
+        )
+      )
+    }
+
     async toFindNothing() {
-      await this.toContain([])
+      await this.toContainExactly([])
     }
 
     async toHaveLength(length: number) {
@@ -135,18 +163,18 @@ describe.each([
 
     describe("misc", () => {
       it("should return all if no query is passed", () =>
-        expectSearch({} as RowSearchParams).toContain([
+        expectSearch({} as RowSearchParams).toContainExactly([
           { name: "foo" },
           { name: "bar" },
         ]))
 
       it("should return all if empty query is passed", () =>
-        expectQuery({}).toContain([{ name: "foo" }, { name: "bar" }]))
+        expectQuery({}).toContainExactly([{ name: "foo" }, { name: "bar" }]))
 
       it("should return all if onEmptyFilter is RETURN_ALL", () =>
         expectQuery({
           onEmptyFilter: EmptyFilterOption.RETURN_ALL,
-        }).toContain([{ name: "foo" }, { name: "bar" }]))
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
 
       it("should return nothing if onEmptyFilter is RETURN_NONE", () =>
         expectQuery({
@@ -159,7 +187,9 @@ describe.each([
 
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { name: "foo" } }).toContain([{ name: "foo" }]))
+        expectQuery({ equal: { name: "foo" } }).toContainExactly([
+          { name: "foo" },
+        ]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { name: "none" } }).toFindNothing())
@@ -167,15 +197,21 @@ describe.each([
 
     describe("notEqual", () => {
       it("successfully finds a row", () =>
-        expectQuery({ notEqual: { name: "foo" } }).toContain([{ name: "bar" }]))
+        expectQuery({ notEqual: { name: "foo" } }).toContainExactly([
+          { name: "bar" },
+        ]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ notEqual: { name: "bar" } }).toContain([{ name: "foo" }]))
+        expectQuery({ notEqual: { name: "bar" } }).toContainExactly([
+          { name: "foo" },
+        ]))
     })
 
     describe("oneOf", () => {
       it("successfully finds a row", () =>
-        expectQuery({ oneOf: { name: ["foo"] } }).toContain([{ name: "foo" }]))
+        expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([
+          { name: "foo" },
+        ]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ oneOf: { name: ["none"] } }).toFindNothing())
@@ -183,7 +219,9 @@ describe.each([
 
     describe("fuzzy", () => {
       it("successfully finds a row", () =>
-        expectQuery({ fuzzy: { name: "oo" } }).toContain([{ name: "foo" }]))
+        expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([
+          { name: "foo" },
+        ]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ fuzzy: { name: "none" } }).toFindNothing())
@@ -195,14 +233,14 @@ describe.each([
           query: {},
           sort: "name",
           sortOrder: SortOrder.ASCENDING,
-        }).toMatch([{ name: "bar" }, { name: "foo" }]))
+        }).toMatchExactly([{ name: "bar" }, { name: "foo" }]))
 
       it("sorts descending", () =>
         expectSearch({
           query: {},
           sort: "name",
           sortOrder: SortOrder.DESCENDING,
-        }).toMatch([{ name: "foo" }, { name: "bar" }]))
+        }).toMatchExactly([{ name: "foo" }, { name: "bar" }]))
 
       describe("sortType STRING", () => {
         it("sorts ascending", () =>
@@ -211,7 +249,7 @@ describe.each([
             sort: "name",
             sortType: SortType.STRING,
             sortOrder: SortOrder.ASCENDING,
-          }).toMatch([{ name: "bar" }, { name: "foo" }]))
+          }).toMatchExactly([{ name: "bar" }, { name: "foo" }]))
 
         it("sorts descending", () =>
           expectSearch({
@@ -219,7 +257,7 @@ describe.each([
             sort: "name",
             sortType: SortType.STRING,
             sortOrder: SortOrder.DESCENDING,
-          }).toMatch([{ name: "foo" }, { name: "bar" }]))
+          }).toMatchExactly([{ name: "foo" }, { name: "bar" }]))
       })
     })
   })
@@ -234,7 +272,7 @@ describe.each([
 
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { age: 1 } }).toContain([{ age: 1 }]))
+        expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { age: 2 } }).toFindNothing())
@@ -242,15 +280,15 @@ describe.each([
 
     describe("notEqual", () => {
       it("successfully finds a row", () =>
-        expectQuery({ notEqual: { age: 1 } }).toContain([{ age: 10 }]))
+        expectQuery({ notEqual: { age: 1 } }).toContainExactly([{ age: 10 }]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ notEqual: { age: 10 } }).toContain([{ age: 1 }]))
+        expectQuery({ notEqual: { age: 10 } }).toContainExactly([{ age: 1 }]))
     })
 
     describe("oneOf", () => {
       it("successfully finds a row", () =>
-        expectQuery({ oneOf: { age: [1] } }).toContain([{ age: 1 }]))
+        expectQuery({ oneOf: { age: [1] } }).toContainExactly([{ age: 1 }]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ oneOf: { age: [2] } }).toFindNothing())
@@ -260,17 +298,17 @@ describe.each([
       it("successfully finds a row", () =>
         expectQuery({
           range: { age: { low: 1, high: 5 } },
-        }).toContain([{ age: 1 }]))
+        }).toContainExactly([{ age: 1 }]))
 
       it("successfully finds multiple rows", () =>
         expectQuery({
           range: { age: { low: 1, high: 10 } },
-        }).toContain([{ age: 1 }, { age: 10 }]))
+        }).toContainExactly([{ age: 1 }, { age: 10 }]))
 
       it("successfully finds a row with a high bound", () =>
         expectQuery({
           range: { age: { low: 5, high: 10 } },
-        }).toContain([{ age: 10 }]))
+        }).toContainExactly([{ age: 10 }]))
     })
 
     describe("sort", () => {
@@ -279,14 +317,14 @@ describe.each([
           query: {},
           sort: "age",
           sortOrder: SortOrder.ASCENDING,
-        }).toMatch([{ age: 1 }, { age: 10 }]))
+        }).toMatchExactly([{ age: 1 }, { age: 10 }]))
 
       it("sorts descending", () =>
         expectSearch({
           query: {},
           sort: "age",
           sortOrder: SortOrder.DESCENDING,
-        }).toMatch([{ age: 10 }, { age: 1 }]))
+        }).toMatchExactly([{ age: 10 }, { age: 1 }]))
     })
 
     describe("sortType NUMBER", () => {
@@ -296,7 +334,7 @@ describe.each([
           sort: "age",
           sortType: SortType.NUMBER,
           sortOrder: SortOrder.ASCENDING,
-        }).toMatch([{ age: 1 }, { age: 10 }]))
+        }).toMatchExactly([{ age: 1 }, { age: 10 }]))
 
       it("sorts descending", () =>
         expectSearch({
@@ -304,7 +342,7 @@ describe.each([
           sort: "age",
           sortType: SortType.NUMBER,
           sortOrder: SortOrder.DESCENDING,
-        }).toMatch([{ age: 10 }, { age: 1 }]))
+        }).toMatchExactly([{ age: 10 }, { age: 1 }]))
     })
   })
 
@@ -324,7 +362,9 @@ describe.each([
 
     describe("equal", () => {
       it("successfully finds a row", () =>
-        expectQuery({ equal: { dob: JAN_1ST } }).toContain([{ dob: JAN_1ST }]))
+        expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([
+          { dob: JAN_1ST },
+        ]))
 
       it("fails to find nonexistent row", () =>
         expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing())
@@ -332,19 +372,19 @@ describe.each([
 
     describe("notEqual", () => {
       it("successfully finds a row", () =>
-        expectQuery({ notEqual: { dob: JAN_1ST } }).toContain([
+        expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([
           { dob: JAN_10TH },
         ]))
 
       it("fails to find nonexistent row", () =>
-        expectQuery({ notEqual: { dob: JAN_10TH } }).toContain([
+        expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([
           { dob: JAN_1ST },
         ]))
     })
 
     describe("oneOf", () => {
       it("successfully finds a row", () =>
-        expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContain([
+        expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([
           { dob: JAN_1ST },
         ]))
 
@@ -356,17 +396,17 @@ describe.each([
       it("successfully finds a row", () =>
         expectQuery({
           range: { dob: { low: JAN_1ST, high: JAN_5TH } },
-        }).toContain([{ dob: JAN_1ST }]))
+        }).toContainExactly([{ dob: JAN_1ST }]))
 
       it("successfully finds multiple rows", () =>
         expectQuery({
           range: { dob: { low: JAN_1ST, high: JAN_10TH } },
-        }).toContain([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+        }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
 
       it("successfully finds a row with a high bound", () =>
         expectQuery({
           range: { dob: { low: JAN_5TH, high: JAN_10TH } },
-        }).toContain([{ dob: JAN_10TH }]))
+        }).toContainExactly([{ dob: JAN_10TH }]))
     })
 
     describe("sort", () => {
@@ -375,14 +415,14 @@ describe.each([
           query: {},
           sort: "dob",
           sortOrder: SortOrder.ASCENDING,
-        }).toMatch([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+        }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
 
       it("sorts descending", () =>
         expectSearch({
           query: {},
           sort: "dob",
           sortOrder: SortOrder.DESCENDING,
-        }).toMatch([{ dob: JAN_10TH }, { dob: JAN_1ST }]))
+        }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]))
 
       describe("sortType STRING", () => {
         it("sorts ascending", () =>
@@ -391,7 +431,7 @@ describe.each([
             sort: "dob",
             sortType: SortType.STRING,
             sortOrder: SortOrder.ASCENDING,
-          }).toMatch([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+          }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
 
         it("sorts descending", () =>
           expectSearch({
@@ -399,7 +439,7 @@ describe.each([
             sort: "dob",
             sortType: SortType.STRING,
             sortOrder: SortOrder.DESCENDING,
-          }).toMatch([{ dob: JAN_10TH }, { dob: JAN_1ST }]))
+          }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]))
       })
     })
   })

From 6d8dc7c2f6129c082bac9f97afe7b93f65e3cd1e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 12 Apr 2024 17:30:56 +0100
Subject: [PATCH 26/45] Add some more range tests.

---
 .../src/api/routes/tests/search.spec.ts       | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index a473cb77b4..f6945cbe46 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -227,6 +227,28 @@ describe.each([
         expectQuery({ fuzzy: { name: "none" } }).toFindNothing())
     })
 
+    describe("range", () => {
+      it("successfully finds multiple rows", () =>
+        expectQuery({
+          range: { name: { low: "a", high: "z" } },
+        }).toContainExactly([{ name: "bar" }, { name: "foo" }]))
+
+      it("successfully finds a row with a high bound", () =>
+        expectQuery({
+          range: { name: { low: "a", high: "c" } },
+        }).toContainExactly([{ name: "bar" }]))
+
+      it("successfully finds a row with a low bound", () =>
+        expectQuery({
+          range: { name: { low: "f", high: "z" } },
+        }).toContainExactly([{ name: "foo" }]))
+
+      it("successfully finds no rows", () =>
+        expectQuery({
+          range: { name: { low: "g", high: "h" } },
+        }).toFindNothing())
+    })
+
     describe("sort", () => {
       it("sorts ascending", () =>
         expectSearch({
@@ -309,6 +331,11 @@ describe.each([
         expectQuery({
           range: { age: { low: 5, high: 10 } },
         }).toContainExactly([{ age: 10 }]))
+
+      it("successfully finds no rows", () =>
+        expectQuery({
+          range: { age: { low: 5, high: 9 } },
+        }).toFindNothing())
     })
 
     describe("sort", () => {
@@ -350,6 +377,7 @@ describe.each([
     const JAN_1ST = "2020-01-01T00:00:00.000Z"
     const JAN_2ND = "2020-01-02T00:00:00.000Z"
     const JAN_5TH = "2020-01-05T00:00:00.000Z"
+    const JAN_9TH = "2020-01-09T00:00:00.000Z"
     const JAN_10TH = "2020-01-10T00:00:00.000Z"
 
     beforeAll(async () => {
@@ -407,6 +435,11 @@ describe.each([
         expectQuery({
           range: { dob: { low: JAN_5TH, high: JAN_10TH } },
         }).toContainExactly([{ dob: JAN_10TH }]))
+
+      it("successfully finds no rows", () =>
+        expectQuery({
+          range: { dob: { low: JAN_5TH, high: JAN_9TH } },
+        }).toFindNothing())
     })
 
     describe("sort", () => {

From 4e4dfefedef34af178d27cb78bc64cbfa9192a89 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 13:41:32 +0100
Subject: [PATCH 27/45] Revert "adds sidepanel open and close actions, and
 gives the user the option to disable click-outside closure of sidepanel"

---
 packages/bbui/src/Layout/Page.svelte             | 10 +++++-----
 packages/client/manifest.json                    | 16 +---------------
 packages/client/src/components/app/Layout.svelte |  5 +----
 .../client/src/components/app/SidePanel.svelte   | 14 --------------
 packages/client/src/stores/sidePanel.js          |  9 ---------
 5 files changed, 7 insertions(+), 47 deletions(-)

diff --git a/packages/bbui/src/Layout/Page.svelte b/packages/bbui/src/Layout/Page.svelte
index 62dd9cc909..2169a12459 100644
--- a/packages/bbui/src/Layout/Page.svelte
+++ b/packages/bbui/src/Layout/Page.svelte
@@ -7,11 +7,11 @@
   export let narrower = false
   export let noPadding = false
 
-  let sidePanelVisible = false
+  let sidePanelVisble = false
 
   setContext("side-panel", {
-    open: () => (sidePanelVisible = true),
-    close: () => (sidePanelVisible = false),
+    open: () => (sidePanelVisble = true),
+    close: () => (sidePanelVisble = false),
   })
 </script>
 
@@ -24,9 +24,9 @@
   </div>
   <div
     id="side-panel"
-    class:visible={sidePanelVisible}
+    class:visible={sidePanelVisble}
     use:clickOutside={() => {
-      sidePanelVisible = false
+      sidePanelVisble = false
     }}
   >
     <slot name="side-panel" />
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index c9e28e202b..40abc7a9a0 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -6723,21 +6723,7 @@
     "illegalChildren": ["section", "sidepanel"],
     "showEmptyState": false,
     "draggable": false,
-    "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action.",
-    "sendEvents": true,
-    "settings": [
-      {
-        "type": "boolean",
-        "key": "clickOutsideToClose",
-        "label": "Click outside to close",
-        "defaultValue": true
-      },
-      {
-        "type": "event",
-        "key": "onSidePanelClose",
-        "label": "On side panel close"
-      }
-    ]
+    "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action."
   },
   "rowexplorer": {
     "block": true,
diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte
index bae2bd0faf..8508e943ff 100644
--- a/packages/client/src/components/app/Layout.svelte
+++ b/packages/client/src/components/app/Layout.svelte
@@ -73,10 +73,7 @@
     $context.device.width,
     $context.device.height
   )
-  $: autoCloseSidePanel =
-    !$builderStore.inBuilder &&
-    $sidePanelStore.open &&
-    $sidePanelStore.clickOutsideToClose
+  $: autoCloseSidePanel = !$builderStore.inBuilder && $sidePanelStore.open
   $: screenId = $builderStore.inBuilder
     ? `${$builderStore.screen?._id}-screen`
     : "screen"
diff --git a/packages/client/src/components/app/SidePanel.svelte b/packages/client/src/components/app/SidePanel.svelte
index 624617ad69..825b401bb8 100644
--- a/packages/client/src/components/app/SidePanel.svelte
+++ b/packages/client/src/components/app/SidePanel.svelte
@@ -5,9 +5,6 @@
   const { styleable, sidePanelStore, builderStore, dndIsDragging } =
     getContext("sdk")
 
-  export let sidePanelClose
-  export let clickOutsideToClose
-
   // Automatically show and hide the side panel when inside the builder.
   // For some unknown reason, svelte reactivity breaks if we reference the
   // reactive variable "open" inside the following expression, or if we define
@@ -29,10 +26,6 @@
     }
   }
 
-  $: {
-    sidePanelStore.actions.setSidepanelState(clickOutsideToClose)
-  }
-
   // Derive visibility
   $: open = $sidePanelStore.contentId === $component.id
 
@@ -47,12 +40,6 @@
     }
   }
 
-  const handleSidePanelClose = async () => {
-    if (sidePanelClose) {
-      await sidePanelClose()
-    }
-  }
-
   const showInSidePanel = (el, visible) => {
     const update = visible => {
       const target = document.getElementById("side-panel-container")
@@ -64,7 +51,6 @@
       } else {
         if (target.contains(node)) {
           target.removeChild(node)
-          handleSidePanelClose()
         }
       }
     }
diff --git a/packages/client/src/stores/sidePanel.js b/packages/client/src/stores/sidePanel.js
index df66eca01c..3b3b9f5f4d 100644
--- a/packages/client/src/stores/sidePanel.js
+++ b/packages/client/src/stores/sidePanel.js
@@ -3,7 +3,6 @@ import { writable, derived } from "svelte/store"
 export const createSidePanelStore = () => {
   const initialState = {
     contentId: null,
-    clickOutsideToClose: true,
   }
   const store = writable(initialState)
   const derivedStore = derived(store, $store => {
@@ -33,19 +32,11 @@ export const createSidePanelStore = () => {
     }, 50)
   }
 
-  const setSidepanelState = bool => {
-    clearTimeout(timeout)
-    store.update(state => {
-      state.clickOutsideToClose = bool
-      return state
-    })
-  }
   return {
     subscribe: derivedStore.subscribe,
     actions: {
       open,
       close,
-      setSidepanelState,
     },
   }
 }

From 68c5e657ddd0fb5b45948e418531d72cd16b178e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 13:46:31 +0100
Subject: [PATCH 28/45] Updating @types/archiver to be more specific.

---
 packages/server/package.json | 2 +-
 yarn.lock                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index ad03033e67..76402785d7 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -125,7 +125,7 @@
     "@babel/preset-env": "7.16.11",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
-    "@types/archiver": "^6.0.2",
+    "@types/archiver": "6.0.2",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/jest": "29.5.5",
diff --git a/yarn.lock b/yarn.lock
index a36b54d3be..ce39c89075 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5174,7 +5174,7 @@
   dependencies:
     "@types/node" "*"
 
-"@types/archiver@^6.0.2":
+"@types/archiver@6.0.2":
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2"
   integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==

From 8b9d07fed6896ffabdc16d6dc98798d99dd5e358 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Mon, 15 Apr 2024 14:37:12 +0100
Subject: [PATCH 29/45] Simplify camunda account-portal local dev setup
 (#13482)

---
 package.json              |  1 +
 scripts/deploy-camunda.sh | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100755 scripts/deploy-camunda.sh

diff --git a/package.json b/package.json
index 2816247939..e520b7c2cf 100644
--- a/package.json
+++ b/package.json
@@ -56,6 +56,7 @@
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
     "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
     "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
+    "dev:camunda": "./scripts/deploy-camunda.sh",
     "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
diff --git a/scripts/deploy-camunda.sh b/scripts/deploy-camunda.sh
new file mode 100755
index 0000000000..d01ed64b5a
--- /dev/null
+++ b/scripts/deploy-camunda.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+yarn global add zbctl
+export ZEEBE_ADDRESS='localhost:26500'
+
+cd ../budibase-bpm
+
+is_camunda_ready() {
+    if (zbctl --insecure status 2>/dev/null) | grep -q 'Healthy'; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+docker-compose up -d
+echo "waiting for Camunda to be ready..."
+
+while is_camunda_ready -eq 0; do sleep 1; done
+
+cd src/main/resources/models
+
+echo "deploy processes..."
+zbctl deploy resource offboarding.bpmn --insecure
+zbctl deploy resource onboarding.bpmn --insecure
+
+cd ../../../../../budibase/packages/account-portal/packages/server 
+
+yarn worker:run & cd ../../../.. && yarn dev:accountportal
+
+
+

From 203e32ecc6c39f836f74711f2a3b89f70a4ef167 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 15:09:17 +0100
Subject: [PATCH 30/45] Commenting the field type enumeration to better explain
 what all of the types do and how they are represented within Budibase.

---
 packages/types/src/documents/app/row.ts | 54 +++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index 222c346591..ccdf001965 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -1,22 +1,76 @@
 import { Document } from "../document"
 
 export enum FieldType {
+  // a primitive type, stores a string, called Text within Budibase. This is one of the default
+  // types of Budibase, if an external type is not fully understood, we will treat it as text.
   STRING = "string",
+  // similar to string type, called Long Form Text within Budibase. This is mainly a frontend
+  // orientated type which disables a larger text input area. This can also be used
+  // in conjunction with the 'useRichText' option to support a markdown editor/viewer.
   LONGFORM = "longform",
+  // similar to string type, called Options within Budibase. This works very similarly to
+  // the string type within the backend, but is validated to a list of options. This will
+  // be displayed a select input within the builder/client.
   OPTIONS = "options",
+  // a primitive type, stores a number, as a floating point, called Number within Budibase.
+  // this type will always represent numbers as reals/floating point - there is no integer only
+  // type within Budibase.
   NUMBER = "number",
+  // a primitive type, stores a boolean, called Boolean within Budibase. This is often represented
+  // as a toggle or checkbox within forms/grids.
   BOOLEAN = "boolean",
+  // a JSON type, this type is always an array of strings, called Multi-select within Budibase.
+  // This type can be compared to the options type, as it functions similarly, but allows picking
+  // multiple options rather than a single option.
   ARRAY = "array",
+  // a string type, this is always a string when input/returned from the API, called Date/Time within
+  // Budibase. We utilise ISO date strings for representing dates, this type has a range of sub-types
+  // to restrict it to date only, time only and ignore timezone capabilities.
   DATETIME = "datetime",
+  // a JSON type, an array of metadata about files held in object storage, called Attachment List within
+  // Budibase. To utilise this type there is an API for uploading files to Budibase, which returns metadata
+  // that can be stored against columns of this type. Currently this is not supported on external databases.
   ATTACHMENTS = "attachment",
+  // a JSON type, similar to the attachments type, called Attachment within Budibase. This type functions
+  // much the same as the attachment list, but only holds a single attachment metadata as an object.
+  // This simpifies the binding experience of using this column type.
   ATTACHMENT_SINGLE = "attachment_single",
+  // a complex type, called Relationships within Budibase. This is the most complex type of Budibase,
+  // nothing should be stored against rows under link columns; this type simply represents the
+  // relationship between tables as part of the table schema. When rows are input to the Budibase API
+  // relationships to be made are represented as a list of row IDs to link. When rows are returned
+  // from the Budibase API it will contain a list of row IDs and display column values of the related rows.
   LINK = "link",
+  // a complex type, called Formulas within Budibase. This type has two variants, static and dynamic, with
+  // static only being supported against internal tables. Dynamic formulas calculate a provided HBS/JS binding
+  // based on the row context and enrich it when rows are being returned from the API. Static bindings calculate
+  // this when rows are being stored, so that the formula output can be searched upon within the DB.
   FORMULA = "formula",
+  // a complex type, called Auto Column within Budibase. This type has a few variants, with options such as a
+  // date for created at/updated at, an auto ID column with auto-increments as rows are saved and a user
+  // relationship type which stores the created by/updated by user details. This sub-types all depend on the
+  // date, number of link types respectively.
   AUTO = "auto",
+  // a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
+  // type, which will be represented a string in the row. This type depends on a schema being provided to make the
+  // JSON searchable/bindable, the JSON cannot be fully dynamic.
   JSON = "json",
+  // an internal type, this is an old deprecated type which is no longer used - still represented to note it
+  // could appear in very old tables.
   INTERNAL = "internal",
+  // a string type, called Barcode/QR within Budibase. This type is used to denote to forms to that this column
+  // should be filled in using a camera to read a barcode, there is a form component which will be used when this
+  // type is found. The column will contain the contents of any barcode scanned.
   BARCODEQR = "barcodeqr",
+  // a string type, this allows representing very large integers, but they are held/managed within Budibase as
+  // strings. When stored in external databases Budibase will attempt to use a real big integer type and depend
+  // on the database parsing the string to this type as part of saving.
   BIGINT = "bigint",
+  // a JSON type, called User within Budibase. This type is used to represent a link to an internal Budibase
+  // resource, like a user or group, today only users are supported. This type will be represented as an
+  // array of internal resource IDs (e.g. user IDs) within the row - this ID list will be enriched with
+  // the full resources when rows are returned from the API. The full resources can be input to the API, or
+  // an array of resource IDs, the API will squash these down and validate them before saving the row.
   BB_REFERENCE = "bb_reference",
 }
 

From d61d5f51cc5c791041a23e0e41469e5603f59575 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 15 Apr 2024 15:31:46 +0100
Subject: [PATCH 31/45] Add tests for array column types, fixing some bugs
 along the way.

---
 .../src/api/routes/tests/search.spec.ts       | 74 +++++++++++++++++++
 packages/server/src/constants/index.ts        |  2 +
 packages/server/src/integrations/base/sql.ts  |  5 ++
 packages/server/src/sdk/app/rows/search.ts    |  6 +-
 4 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index f6945cbe46..5b71ec9044 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -476,4 +476,78 @@ describe.each([
       })
     })
   })
+
+  describe("array of strings", () => {
+    beforeAll(async () => {
+      await createTable({
+        numbers: {
+          name: "numbers",
+          type: FieldType.ARRAY,
+          constraints: { inclusion: ["one", "two", "three"] },
+        },
+      })
+      await createRows([{ numbers: ["one", "two"] }, { numbers: ["three"] }])
+    })
+
+    describe("contains", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([
+          { numbers: ["one", "two"] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ contains: { numbers: ["none"] } }).toFindNothing())
+
+      it("fails to find row containing all", () =>
+        expectQuery({
+          contains: { numbers: ["one", "two", "three"] },
+        }).toFindNothing())
+
+      it("finds all with empty list", () =>
+        expectQuery({ contains: { numbers: [] } }).toContainExactly([
+          { numbers: ["one", "two"] },
+          { numbers: ["three"] },
+        ]))
+    })
+
+    describe("notContains", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notContains: { numbers: ["one"] } }).toContainExactly([
+          { numbers: ["three"] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({
+          notContains: { numbers: ["one", "two", "three"] },
+        }).toContainExactly([
+          { numbers: ["one", "two"] },
+          { numbers: ["three"] },
+        ]))
+
+      it("finds all with empty list", () =>
+        expectQuery({ notContains: { numbers: [] } }).toContainExactly([
+          { numbers: ["one", "two"] },
+          { numbers: ["three"] },
+        ]))
+    })
+
+    describe("containsAny", () => {
+      it("successfully finds rows", () =>
+        expectQuery({
+          containsAny: { numbers: ["one", "two", "three"] },
+        }).toContainExactly([
+          { numbers: ["one", "two"] },
+          { numbers: ["three"] },
+        ]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ containsAny: { numbers: ["none"] } }).toFindNothing())
+
+      it("finds all with empty list", () =>
+        expectQuery({ containsAny: { numbers: [] } }).toContainExactly([
+          { numbers: ["one", "two"] },
+          { numbers: ["three"] },
+        ]))
+    })
+  })
 })
diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts
index 42a1b53224..37c275c8a3 100644
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@@ -20,6 +20,7 @@ export enum FilterTypes {
   NOT_EMPTY = "notEmpty",
   CONTAINS = "contains",
   NOT_CONTAINS = "notContains",
+  CONTAINS_ANY = "containsAny",
   ONE_OF = "oneOf",
 }
 
@@ -30,6 +31,7 @@ export const NoEmptyFilterStrings = [
   FilterTypes.NOT_EQUAL,
   FilterTypes.CONTAINS,
   FilterTypes.NOT_CONTAINS,
+  FilterTypes.CONTAINS_ANY,
 ]
 
 export const CanSwitchTypes = [
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index f5828f9419..259abec106 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -233,6 +233,11 @@ class InternalBuilder {
               (statement ? andOr : "") +
               `LOWER(${likeKey(this.client, key)}) LIKE ?`
           }
+
+          if (statement === "") {
+            return
+          }
+
           // @ts-ignore
           query = query[rawFnc](`${not}(${statement})`, value)
         })
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index f681bfeb90..5a016c821f 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -29,6 +29,10 @@ function pickApi(tableId: any) {
   return internal
 }
 
+function isEmptyArray(value: any) {
+  return Array.isArray(value) && value.length === 0
+}
+
 // don't do a pure falsy check, as 0 is included
 // https://github.com/Budibase/budibase/issues/10118
 export function removeEmptyFilters(filters: SearchFilters) {
@@ -47,7 +51,7 @@ export function removeEmptyFilters(filters: SearchFilters) {
         for (let [key, value] of Object.entries(
           filters[filterType] as object
         )) {
-          if (value == null || value === "") {
+          if (value == null || value === "" || isEmptyArray(value)) {
             // @ts-ignore
             delete filters[filterField][key]
           }

From 81425b3d287340054d4796e7e6763ce72e7e7d23 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 15:50:25 +0100
Subject: [PATCH 32/45] Addressing PR comment.s

---
 packages/types/src/documents/app/row.ts | 142 +++++++++++++++---------
 1 file changed, 88 insertions(+), 54 deletions(-)

diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index ccdf001965..4f2f9f99ef 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -1,76 +1,110 @@
 import { Document } from "../document"
 
 export enum FieldType {
-  // a primitive type, stores a string, called Text within Budibase. This is one of the default
-  // types of Budibase, if an external type is not fully understood, we will treat it as text.
+  /**
+   * a primitive type, stores a string, called Text within Budibase. This is one of the default
+   * types of Budibase, if an external type is not fully understood, we will treat it as text.
+   */
   STRING = "string",
-  // similar to string type, called Long Form Text within Budibase. This is mainly a frontend
-  // orientated type which disables a larger text input area. This can also be used
-  // in conjunction with the 'useRichText' option to support a markdown editor/viewer.
+  /**
+   * similar to string type, called Long Form Text within Budibase. This is mainly a frontend
+   * orientated type which enables a larger text input area. This can also be used
+   * in conjunction with the 'useRichText' option to support a markdown editor/viewer.
+   */
   LONGFORM = "longform",
-  // similar to string type, called Options within Budibase. This works very similarly to
-  // the string type within the backend, but is validated to a list of options. This will
-  // be displayed a select input within the builder/client.
+  /**
+   * similar to string type, called Options within Budibase. This works very similarly to
+   * the string type within the backend, but is validated to a list of options. This will
+   * display a <select> input within the builder/client.
+   */
   OPTIONS = "options",
-  // a primitive type, stores a number, as a floating point, called Number within Budibase.
-  // this type will always represent numbers as reals/floating point - there is no integer only
-  // type within Budibase.
+  /**
+   * a primitive type, stores a number, as a floating point, called Number within Budibase.
+   * this type will always represent numbers as reals/floating point - there is no integer only
+   * type within Budibase.
+   */
   NUMBER = "number",
-  // a primitive type, stores a boolean, called Boolean within Budibase. This is often represented
-  // as a toggle or checkbox within forms/grids.
+  /**
+   * a primitive type, stores a boolean, called Boolean within Budibase. This is often represented
+   * as a toggle or checkbox within forms/grids.
+   */
   BOOLEAN = "boolean",
-  // a JSON type, this type is always an array of strings, called Multi-select within Budibase.
-  // This type can be compared to the options type, as it functions similarly, but allows picking
-  // multiple options rather than a single option.
+  /**
+   * a JSON type, this type is always an array of strings, called Multi-select within Budibase.
+   * This type can be compared to the options type, as it functions similarly, but allows picking
+   * multiple options rather than a single option.
+   */
   ARRAY = "array",
-  // a string type, this is always a string when input/returned from the API, called Date/Time within
-  // Budibase. We utilise ISO date strings for representing dates, this type has a range of sub-types
-  // to restrict it to date only, time only and ignore timezone capabilities.
+  /**
+   * a string type, this is always a string when input/returned from the API, called Date/Time within
+   * Budibase. We utilise ISO date strings for representing dates, this type has a range of subtypes
+   * to restrict it to date only, time only and ignore timezone capabilities.
+   */
   DATETIME = "datetime",
-  // a JSON type, an array of metadata about files held in object storage, called Attachment List within
-  // Budibase. To utilise this type there is an API for uploading files to Budibase, which returns metadata
-  // that can be stored against columns of this type. Currently this is not supported on external databases.
+  /**
+   * a JSON type, an array of metadata about files held in object storage, called Attachment List within
+   * Budibase. To utilise this type there is an API for uploading files to Budibase, which returns metadata
+   * that can be stored against columns of this type. Currently this is not supported on external databases.
+   */
   ATTACHMENTS = "attachment",
-  // a JSON type, similar to the attachments type, called Attachment within Budibase. This type functions
-  // much the same as the attachment list, but only holds a single attachment metadata as an object.
-  // This simpifies the binding experience of using this column type.
+  /**
+   * a JSON type, similar to the attachments type, called Attachment within Budibase. This type functions
+   * much the same as the attachment list, but only holds a single attachment metadata as an object.
+   * This simplifies the binding experience of using this column type.
+   */
   ATTACHMENT_SINGLE = "attachment_single",
-  // a complex type, called Relationships within Budibase. This is the most complex type of Budibase,
-  // nothing should be stored against rows under link columns; this type simply represents the
-  // relationship between tables as part of the table schema. When rows are input to the Budibase API
-  // relationships to be made are represented as a list of row IDs to link. When rows are returned
-  // from the Budibase API it will contain a list of row IDs and display column values of the related rows.
+  /**
+   * a complex type, called Relationships within Budibase. This is the most complex type of Budibase,
+   * nothing should be stored against rows under link columns; this type simply represents the
+   * relationship between tables as part of the table schema. When rows are input to the Budibase API
+   * relationships to be made are represented as a list of row IDs to link. When rows are returned
+   * from the Budibase API it will contain a list of row IDs and display column values of the related rows.
+   */
   LINK = "link",
-  // a complex type, called Formulas within Budibase. This type has two variants, static and dynamic, with
-  // static only being supported against internal tables. Dynamic formulas calculate a provided HBS/JS binding
-  // based on the row context and enrich it when rows are being returned from the API. Static bindings calculate
-  // this when rows are being stored, so that the formula output can be searched upon within the DB.
+  /**
+   * a complex type, called Formulas within Budibase. This type has two variants, static and dynamic, with
+   * static only being supported against internal tables. Dynamic formulas calculate a provided HBS/JS binding
+   * based on the row context and enrich it when rows are being returned from the API. Static bindings calculate
+   * this when rows are being stored, so that the formula output can be searched upon within the DB.
+   */
   FORMULA = "formula",
-  // a complex type, called Auto Column within Budibase. This type has a few variants, with options such as a
-  // date for created at/updated at, an auto ID column with auto-increments as rows are saved and a user
-  // relationship type which stores the created by/updated by user details. This sub-types all depend on the
-  // date, number of link types respectively.
+  /**
+   * a complex type, called Auto Column within Budibase. This type has a few variants, with options such as a
+   * date for created at/updated at, an auto ID column with auto-increments as rows are saved and a user
+   * relationship type which stores the created by/updated by user details. This sub-types all depend on the
+   * date, number of link types respectively.
+   */
   AUTO = "auto",
-  // a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
-  // type, which will be represented a string in the row. This type depends on a schema being provided to make the
-  // JSON searchable/bindable, the JSON cannot be fully dynamic.
+  /**
+   * a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
+   * type, which will be represented as a JSON object in the row. This type depends on a schema being
+   * provided to make the JSON searchable/bindable, the JSON cannot be fully dynamic.
+   */
   JSON = "json",
-  // an internal type, this is an old deprecated type which is no longer used - still represented to note it
-  // could appear in very old tables.
+  /**
+   * @deprecated an internal type, this is an old deprecated type which is no longer used - still represented to note it
+   * could appear in very old tables.
+   */
   INTERNAL = "internal",
-  // a string type, called Barcode/QR within Budibase. This type is used to denote to forms to that this column
-  // should be filled in using a camera to read a barcode, there is a form component which will be used when this
-  // type is found. The column will contain the contents of any barcode scanned.
+  /**
+   * a string type, called Barcode/QR within Budibase. This type is used to denote to forms to that this column
+   * should be filled in using a camera to read a barcode, there is a form component which will be used when this
+   * type is found. The column will contain the contents of any barcode scanned.
+   */
   BARCODEQR = "barcodeqr",
-  // a string type, this allows representing very large integers, but they are held/managed within Budibase as
-  // strings. When stored in external databases Budibase will attempt to use a real big integer type and depend
-  // on the database parsing the string to this type as part of saving.
+  /**
+   * a string type, this allows representing very large integers, but they are held/managed within Budibase as
+   * strings. When stored in external databases Budibase will attempt to use a real big integer type and depend
+   * on the database parsing the string to this type as part of saving.
+   */
   BIGINT = "bigint",
-  // a JSON type, called User within Budibase. This type is used to represent a link to an internal Budibase
-  // resource, like a user or group, today only users are supported. This type will be represented as an
-  // array of internal resource IDs (e.g. user IDs) within the row - this ID list will be enriched with
-  // the full resources when rows are returned from the API. The full resources can be input to the API, or
-  // an array of resource IDs, the API will squash these down and validate them before saving the row.
+  /**
+   * a JSON type, called User within Budibase. This type is used to represent a link to an internal Budibase
+   * resource, like a user or group, today only users are supported. This type will be represented as an
+   * array of internal resource IDs (e.g. user IDs) within the row - this ID list will be enriched with
+   * the full resources when rows are returned from the API. The full resources can be input to the API, or
+   * an array of resource IDs, the API will squash these down and validate them before saving the row.
+   */
   BB_REFERENCE = "bb_reference",
 }
 

From fea6cd623cbe823955fd52061adcf9a64666ea76 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 17:07:08 +0100
Subject: [PATCH 33/45] Final change.

---
 packages/types/src/documents/app/row.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index 4f2f9f99ef..865ab4ba64 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -71,8 +71,9 @@ export enum FieldType {
   /**
    * a complex type, called Auto Column within Budibase. This type has a few variants, with options such as a
    * date for created at/updated at, an auto ID column with auto-increments as rows are saved and a user
-   * relationship type which stores the created by/updated by user details. This sub-types all depend on the
-   * date, number of link types respectively.
+   * relationship type which stores the created by/updated by user details. These subtypes all depend on the
+   * date, number of link types respectively. There is one case where these will be executed in the browser,
+   * that is part of the initial formula definition, the formula will be live evaluated in the browser.
    */
   AUTO = "auto",
   /**

From b13b7df678768a460c6c4b4dc464bbb4a25dcda1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 18:23:39 +0100
Subject: [PATCH 34/45] Correctly handling aliasing for sorting/json field
 types with SQS.

---
 packages/server/src/integrations/base/sql.ts | 31 ++++++++++++++++----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 259abec106..b622ca39be 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -22,6 +22,8 @@ import {
   SortDirection,
   SqlQueryBinding,
   Table,
+  TableSourceType,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import environment from "../../environment"
 
@@ -135,6 +137,21 @@ function generateSelectStatement(
   })
 }
 
+function getTableName(table?: Table): string {
+  if (!table) {
+    throw new Error("No table name available.")
+  }
+  // SQS uses the table ID rather than the table name
+  if (
+    table.sourceType === TableSourceType.INTERNAL ||
+    table.sourceId === INTERNAL_TABLE_SOURCE_ID
+  ) {
+    return table._id!
+  } else {
+    return table.name
+  }
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -149,7 +166,7 @@ class InternalBuilder {
     tableName: string,
     opts: { aliases?: Record<string, string>; relationship?: boolean }
   ): Knex.QueryBuilder {
-    function getTableName(name: string) {
+    function getTableAlias(name: string) {
       const alias = opts.aliases?.[name]
       return alias || name
     }
@@ -161,11 +178,11 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableName(tableName)}.${updatedKey}`, value)
+          fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
-          fn(`${getTableName(filterTableName)}.${property}`, value)
+          fn(`${getTableAlias(filterTableName)}.${property}`, value)
         }
       }
     }
@@ -346,9 +363,10 @@ class InternalBuilder {
   addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
     let { sort, paginate } = json
     const table = json.meta?.table
+    const tableName = getTableName(table)
     const aliases = json.tableAliases
     const aliased =
-      table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
+      table?.name && aliases?.[tableName] ? aliases[tableName] : table?.name
     if (sort && Object.keys(sort || {}).length > 0) {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
@@ -729,12 +747,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     results: Record<string, any>[],
     aliases?: Record<string, string>
   ): Record<string, any>[] {
+    const tableName = getTableName(table)
     for (const [name, field] of Object.entries(table.schema)) {
       if (!this._isJsonColumn(field)) {
         continue
       }
-      const tableName = aliases?.[table.name] || table.name
-      const fullName = `${tableName}.${name}`
+      const aliasedTableName = aliases?.[tableName] || tableName
+      const fullName = `${aliasedTableName}.${name}`
       for (let row of results) {
         if (typeof row[fullName] === "string") {
           row[fullName] = JSON.parse(row[fullName])

From 69ae603fa4d3aae6e61427149057edbbd885e64a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 15 Apr 2024 18:24:11 +0100
Subject: [PATCH 35/45] Updating errors from sql table actions and making sure
 SQS tables cannot be reach the sql table actions/will error if they do.

---
 .../server/src/integrations/base/sqlTable.ts  | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts
index 4ff336421f..3c55d75b8b 100644
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@@ -1,19 +1,20 @@
 import { Knex, knex } from "knex"
 import {
-  RelationshipType,
   FieldSubtype,
+  FieldType,
   NumberFieldMetadata,
   Operation,
   QueryJson,
+  RelationshipType,
   RenameColumn,
-  Table,
-  FieldType,
   SqlQuery,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
 import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
+import { utils } from "@budibase/shared-core"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-import { utils } from "@budibase/shared-core"
 
 function isIgnoredType(type: FieldType) {
   const ignored = [FieldType.LINK, FieldType.FORMULA]
@@ -105,13 +106,13 @@ function generateSchema(
           column.relationshipType !== RelationshipType.MANY_TO_MANY
         ) {
           if (!column.foreignKey || !column.tableId) {
-            throw "Invalid relationship schema"
+            throw new Error("Invalid relationship schema")
           }
           const { tableName } = breakExternalTableId(column.tableId)
           // @ts-ignore
           const relatedTable = tables[tableName]
           if (!relatedTable) {
-            throw "Referenced table doesn't exist"
+            throw new Error("Referenced table doesn't exist")
           }
           const relatedPrimary = relatedTable.primary[0]
           const externalType = relatedTable.schema[relatedPrimary].externalType
@@ -209,15 +210,19 @@ class SqlTableQueryBuilder {
 
     let query: Knex.SchemaBuilder
     if (!json.table || !json.meta || !json.meta.tables) {
-      throw "Cannot execute without table being specified"
+      throw new Error("Cannot execute without table being specified")
     }
+    if (json.table.sourceType === TableSourceType.INTERNAL) {
+      throw new Error("Cannot perform table actions for SQS.")
+    }
+
     switch (this._operation(json)) {
       case Operation.CREATE_TABLE:
         query = buildCreateTable(client, json.table, json.meta.tables)
         break
       case Operation.UPDATE_TABLE:
         if (!json.meta || !json.meta.table) {
-          throw "Must specify old table for update"
+          throw new Error("Must specify old table for update")
         }
         // renameColumn does not work for MySQL, so return a raw query
         if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
@@ -264,7 +269,7 @@ class SqlTableQueryBuilder {
         query = buildDeleteTable(client, json.table)
         break
       default:
-        throw "Table operation is of unknown type"
+        throw new Error("Table operation is of unknown type")
     }
     return getNativeSql(query)
   }

From 128596cd4174320ae9381593ffcc6830a4105ba9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 16 Apr 2024 10:41:11 +0100
Subject: [PATCH 36/45] Ensure normal spectrum colors are used for nested links
 in top navs

---
 packages/client/src/components/app/NavItem.svelte | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/client/src/components/app/NavItem.svelte b/packages/client/src/components/app/NavItem.svelte
index 360ea4f8bb..fcdda57ace 100644
--- a/packages/client/src/components/app/NavItem.svelte
+++ b/packages/client/src/components/app/NavItem.svelte
@@ -157,6 +157,11 @@
     width: 100%;
   }
 
+  /* Use normal theme colors for links when using a top nav */
+  .dropdown:not(.left) .sublinks a {
+    color: var(--spectrum-alias-text-color);
+  }
+
   /* Left dropdowns */
   .dropdown.left .sublinks-wrapper {
     display: none;

From e2ca21053e52c88994e2ec75845548673e0707e1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 16 Apr 2024 11:38:00 +0100
Subject: [PATCH 37/45] Fixing build issue caught in CI.

---
 packages/server/src/integrations/base/sql.ts | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index b622ca39be..59684422e7 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -137,18 +137,15 @@ function generateSelectStatement(
   })
 }
 
-function getTableName(table?: Table): string {
-  if (!table) {
-    throw new Error("No table name available.")
-  }
+function getTableName(table?: Table): string | undefined {
   // SQS uses the table ID rather than the table name
   if (
-    table.sourceType === TableSourceType.INTERNAL ||
-    table.sourceId === INTERNAL_TABLE_SOURCE_ID
+    table?.sourceType === TableSourceType.INTERNAL ||
+    table?.sourceId === INTERNAL_TABLE_SOURCE_ID
   ) {
-    return table._id!
+    return table?._id
   } else {
-    return table.name
+    return table?.name
   }
 }
 
@@ -366,7 +363,7 @@ class InternalBuilder {
     const tableName = getTableName(table)
     const aliases = json.tableAliases
     const aliased =
-      table?.name && aliases?.[tableName] ? aliases[tableName] : table?.name
+      tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
     if (sort && Object.keys(sort || {}).length > 0) {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
@@ -752,7 +749,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       if (!this._isJsonColumn(field)) {
         continue
       }
-      const aliasedTableName = aliases?.[tableName] || tableName
+      const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
       const fullName = `${aliasedTableName}.${name}`
       for (let row of results) {
         if (typeof row[fullName] === "string") {

From 7d2861718a3511963a02bbb9737eafb504bd4f7f Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 16 Apr 2024 16:41:39 +0100
Subject: [PATCH 38/45] Making meta required in query JSON.

---
 .../api/controllers/row/ExternalRequest.ts    | 16 ++++++++++
 .../routes/tests/queries/generic-sql.spec.ts  |  3 ++
 .../server/src/integrations/tests/sql.spec.ts | 30 ++++++++++++++-----
 .../src/integrations/tests/sqlAlias.spec.ts   | 15 ++++++++++
 packages/types/src/sdk/search.ts              |  2 +-
 5 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 4adbb72c7a..da620a92e6 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -119,6 +119,9 @@ async function removeManyToManyRelationships(
       endpoint: getEndpoint(tableId, Operation.DELETE),
       body: { [colName]: null },
       filters,
+      meta: {
+        table,
+      }
     })
   } else {
     return []
@@ -133,6 +136,9 @@ async function removeOneToManyRelationships(rowId: string, table: Table) {
     return getDatasourceAndQuery({
       endpoint: getEndpoint(tableId, Operation.UPDATE),
       filters,
+      meta: {
+        table,
+      }
     })
   } else {
     return []
@@ -248,6 +254,9 @@ export class ExternalRequest<T extends Operation> {
     const response = await getDatasourceAndQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
       filters: buildFilters(rowId, {}, table),
+      meta: {
+        table,
+      }
     })
     if (Array.isArray(response) && response.length > 0) {
       return response[0]
@@ -395,6 +404,9 @@ export class ExternalRequest<T extends Operation> {
             [fieldName]: row[lookupField],
           },
         },
+        meta: {
+          table,
+        }
       })
       // this is the response from knex if no rows found
       const rows: Row[] =
@@ -425,6 +437,7 @@ export class ExternalRequest<T extends Operation> {
     // if we're creating (in a through table) need to wipe the existing ones first
     const promises = []
     const related = await this.lookupRelations(mainTableId, row)
+    const table = this.getTable(mainTableId)
     for (let relationship of relationships) {
       const { key, tableId, isUpdate, id, ...rest } = relationship
       const body: { [key: string]: any } = processObjectSync(rest, row, {})
@@ -470,6 +483,9 @@ export class ExternalRequest<T extends Operation> {
             // if we're doing many relationships then we're writing, only one response
             body,
             filters: buildFilters(id, {}, linkTable),
+            meta: {
+              table,
+            }
           })
         )
       } else {
diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
index 7790f909e7..1f83adc663 100644
--- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
@@ -755,6 +755,9 @@ describe.each(
             name: "two",
           },
         },
+        meta: {
+          table: config.table,
+        },
       })
       expect(res).toHaveLength(1)
       expect(res[0]).toEqual({
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index dc2a06446b..d056159d7d 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -9,6 +9,14 @@ import {
 } from "@budibase/types"
 
 const TABLE_NAME = "test"
+const TABLE: Table = {
+  type: "table",
+  sourceType: TableSourceType.EXTERNAL,
+  sourceId: "SOURCE_ID",
+  schema: {},
+  name: TABLE_NAME,
+  primary: ["id"],
+}
 
 function endpoint(table: any, operation: any) {
   return {
@@ -25,6 +33,10 @@ function generateReadJson({
   sort,
   paginate,
 }: any = {}): QueryJson {
+  const tableObj = { ...TABLE }
+  if (table) {
+    tableObj.name = table
+  }
   return {
     endpoint: endpoint(table || TABLE_NAME, "READ"),
     resource: {
@@ -34,14 +46,7 @@ function generateReadJson({
     sort: sort || {},
     paginate: paginate || {},
     meta: {
-      table: {
-        type: "table",
-        sourceType: TableSourceType.EXTERNAL,
-        sourceId: "SOURCE_ID",
-        schema: {},
-        name: table || TABLE_NAME,
-        primary: ["id"],
-      } as any,
+      table: tableObj,
     },
   }
 }
@@ -49,6 +54,9 @@ function generateReadJson({
 function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
   return {
     endpoint: endpoint(table, "CREATE"),
+    meta: {
+      table: TABLE,
+    },
     body,
   }
 }
@@ -70,6 +78,9 @@ function generateUpdateJson({
 function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson {
   return {
     endpoint: endpoint(table, "DELETE"),
+    meta: {
+      table: TABLE,
+    },
     filters,
   }
 }
@@ -102,6 +113,9 @@ function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
       },
     ],
     extra: { idFilter: {} },
+    meta: {
+      table: TABLE,
+    },
   }
 }
 
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 58c3a05245..f4edab8dad 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -4,6 +4,8 @@ import {
   QueryJson,
   SourceName,
   SqlQuery,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
 import { join } from "path"
 import Sql from "../base/sql"
@@ -11,6 +13,16 @@ import { SqlClient } from "../utils"
 import { generator } from "@budibase/backend-core/tests"
 import sdk from "../../sdk"
 
+// this doesn't exist strictly
+const TABLE: Table = {
+  type: "table",
+  sourceType: TableSourceType.EXTERNAL,
+  sourceId: "SOURCE_ID",
+  schema: {},
+  name: "tableName",
+  primary: ["id"],
+}
+
 const AliasTables = sdk.rows.AliasTables
 
 function multiline(sql: string) {
@@ -222,6 +234,9 @@ describe("Captures of real examples", () => {
         resource: {
           fields,
         },
+        meta: {
+          table: TABLE,
+        },
       }
     }
 
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 9325f09eed..6cac76e01d 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -90,7 +90,7 @@ export interface QueryJson {
   paginate?: PaginationJson
   body?: Row | Row[]
   table?: Table
-  meta?: {
+  meta: {
     table?: Table
     tables?: Record<string, Table>
     renamed?: RenameColumn

From ccb56c8788391d046243d059ce48898275b073ca Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 16 Apr 2024 17:05:09 +0100
Subject: [PATCH 39/45] Linting.

---
 .../server/src/api/controllers/row/ExternalRequest.ts  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index da620a92e6..835e5bb20d 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -121,7 +121,7 @@ async function removeManyToManyRelationships(
       filters,
       meta: {
         table,
-      }
+      },
     })
   } else {
     return []
@@ -138,7 +138,7 @@ async function removeOneToManyRelationships(rowId: string, table: Table) {
       filters,
       meta: {
         table,
-      }
+      },
     })
   } else {
     return []
@@ -256,7 +256,7 @@ export class ExternalRequest<T extends Operation> {
       filters: buildFilters(rowId, {}, table),
       meta: {
         table,
-      }
+      },
     })
     if (Array.isArray(response) && response.length > 0) {
       return response[0]
@@ -406,7 +406,7 @@ export class ExternalRequest<T extends Operation> {
         },
         meta: {
           table,
-        }
+        },
       })
       // this is the response from knex if no rows found
       const rows: Row[] =
@@ -485,7 +485,7 @@ export class ExternalRequest<T extends Operation> {
             filters: buildFilters(id, {}, linkTable),
             meta: {
               table,
-            }
+            },
           })
         )
       } else {

From 8b0fc5ed5d1b640a15adcf7fd1932ca71909cc1c Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 16 Apr 2024 17:22:17 +0100
Subject: [PATCH 40/45] Making table required.

---
 .../server/src/api/controllers/row/ExternalRequest.ts     | 2 +-
 .../server/src/api/controllers/table/ExternalRequest.ts   | 1 +
 .../src/api/routes/tests/queries/generic-sql.spec.ts      | 2 +-
 packages/server/src/integrations/tests/sql.spec.ts        | 8 ++++++++
 packages/types/src/sdk/search.ts                          | 2 +-
 5 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 835e5bb20d..3dd3f9b8e7 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -437,7 +437,7 @@ export class ExternalRequest<T extends Operation> {
     // if we're creating (in a through table) need to wipe the existing ones first
     const promises = []
     const related = await this.lookupRelations(mainTableId, row)
-    const table = this.getTable(mainTableId)
+    const table = this.getTable(mainTableId)!
     for (let relationship of relationships) {
       const { key, tableId, isUpdate, id, ...rest } = relationship
       const body: { [key: string]: any } = processObjectSync(rest, row, {})
diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts
index 65cead3a1d..1e57ea3294 100644
--- a/packages/server/src/api/controllers/table/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/table/ExternalRequest.ts
@@ -22,6 +22,7 @@ export async function makeTableRequest(
       operation,
     },
     meta: {
+      table,
       tables,
     },
     table,
diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
index 1f83adc663..718f18203c 100644
--- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
@@ -756,7 +756,7 @@ describe.each(
           },
         },
         meta: {
-          table: config.table,
+          table: config.table!,
         },
       })
       expect(res).toHaveLength(1)
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index d056159d7d..4ee544cc5e 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -66,7 +66,15 @@ function generateUpdateJson({
   body = {},
   filters = {},
   meta = {},
+}: {
+  table: string
+  body?: any
+  filters?: any
+  meta?: any
 }): QueryJson {
+  if (!meta.table) {
+    meta.table = table
+  }
   return {
     endpoint: endpoint(table, "UPDATE"),
     filters,
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 6cac76e01d..0b93fb9215 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -91,7 +91,7 @@ export interface QueryJson {
   body?: Row | Row[]
   table?: Table
   meta: {
-    table?: Table
+    table: Table
     tables?: Record<string, Table>
     renamed?: RenameColumn
   }

From 2efbd6726c5878fc2e99793427ca6079e4ae8265 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Tue, 16 Apr 2024 17:28:13 +0100
Subject: [PATCH 41/45] Removing meta from test API, it cannot be supplied, it
 is an internal property.

---
 .../server/src/api/routes/tests/queries/generic-sql.spec.ts  | 3 ---
 packages/server/src/tests/utilities/api/datasource.ts        | 5 ++++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
index 718f18203c..7790f909e7 100644
--- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
@@ -755,9 +755,6 @@ describe.each(
             name: "two",
           },
         },
-        meta: {
-          table: config.table!,
-        },
       })
       expect(res).toHaveLength(1)
       expect(res[0]).toEqual({
diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index 0296f58f7d..0362a25940 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -60,7 +60,10 @@ export class DatasourceAPI extends TestAPI {
     })
   }
 
-  query = async (query: QueryJson, expectations?: Expectations) => {
+  query = async (
+    query: Omit<QueryJson, "meta">,
+    expectations?: Expectations
+  ) => {
     return await this._post<any>(`/api/datasources/query`, {
       body: query,
       expectations,

From a3a13d21d9847f5bd180a84c7aea57d9c2364349 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 16 Apr 2024 17:35:15 +0100
Subject: [PATCH 42/45] bumping pro sm

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index ef186d0024..78bd9cd090 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a
+Subproject commit 78bd9cd0909cba873253ebaf4affe0af4cf27d1c

From aff495e0bc1c038079d6ddb082db9eaec1ab8e0e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 16 Apr 2024 17:40:35 +0100
Subject: [PATCH 43/45] bump pro to master ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 78bd9cd090..c68183402b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 78bd9cd0909cba873253ebaf4affe0af4cf27d1c
+Subproject commit c68183402b8fb17248572006531d5293ffc8a9ac

From 141718d53200f8afd7bc7881b3e527d2d1ed9373 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 16 Apr 2024 18:10:18 +0100
Subject: [PATCH 44/45] bump account portal module

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index bd0e01d639..328c84234d 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344
+Subproject commit 328c84234d11d97d840f0eb2c72665b04ba9e4f8

From b529e98b057eb7704fd0f01ca83690ca1be18f9f Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 16 Apr 2024 17:20:33 +0000
Subject: [PATCH 45/45] Bump version to 2.23.6

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 9839b8b166..a2be7be7b4 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.23.5",
+  "version": "2.23.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",