From 27f6fa7de46ba2f5c482cdcc20b4a866181606e1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 16 Sep 2024 16:36:17 +0100
Subject: [PATCH 01/19] Add a test for row exports on Google Sheets.

---
 .../src/integrations/tests/googlesheets.spec.ts   | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index 62d56bb2c2..cef4decfa7 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -10,6 +10,7 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import { GoogleSheetsMock } from "./utils/googlesheets"
+import rows from "src/sdk/app/rows"
 
 describe("Google Sheets Integration", () => {
   const config = new TestConfiguration()
@@ -244,6 +245,20 @@ describe("Google Sheets Integration", () => {
         expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`))
       )
     })
+
+    it("can export rows", async () => {
+      const resp = await config.api.row.exportRows(table._id!, {})
+      const parsed = JSON.parse(resp)
+      expect(parsed.length).toEqual(2)
+      expect(parsed[0]).toMatchObject({
+        name: "Test Contact 1",
+        description: "original description 1",
+      })
+      expect(parsed[1]).toMatchObject({
+        name: "Test Contact 2",
+        description: "original description 2",
+      })
+    })
   })
 
   describe("update", () => {

From fd3c8c4a119fbf28140c25c23cc804e669a599c7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 17 Sep 2024 16:29:09 +0100
Subject: [PATCH 02/19] Make view readonly columns available for free

---
 .../tests/core/utilities/mocks/licenses.ts        |  4 ----
 .../backend/DataTable/ViewV2DataTable.svelte      |  1 -
 packages/builder/src/stores/portal/licensing.js   |  5 -----
 .../grid/controls/ColumnsSettingButton.svelte     | 14 ++------------
 .../src/components/grid/layout/Grid.svelte        |  4 +---
 .../server/src/api/routes/tests/viewV2.spec.ts    | 15 ---------------
 packages/server/src/sdk/app/views/index.ts        |  7 -------
 7 files changed, 3 insertions(+), 47 deletions(-)

diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
index 2d8e81d125..bc9a3b635c 100644
--- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts
@@ -102,10 +102,6 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
-export const useViewReadonlyColumns = () => {
-  return useFeature(Feature.VIEW_READONLY_COLUMNS)
-}
-
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {
diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index b56c5f6568..ed35c5d6a3 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -30,7 +30,6 @@
     showAvatars={false}
     on:updatedatasource={handleGridViewUpdate}
     isCloud={$admin.cloud}
-    allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
     canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
   >
     <svelte:fragment slot="filter">
diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js
index d48377207c..f0ce7046f2 100644
--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@@ -140,10 +140,6 @@ export const createLicensingStore = () => {
         Constants.Features.VIEW_PERMISSIONS
       )
 
-      const isViewReadonlyColumnsEnabled = license.features.includes(
-        Constants.Features.VIEW_READONLY_COLUMNS
-      )
-
       const budibaseAIEnabled = license.features.includes(
         Constants.Features.BUDIBASE_AI
       )
@@ -173,7 +169,6 @@ export const createLicensingStore = () => {
           triggerAutomationRunEnabled,
           isViewPermissionsEnabled,
           perAppBuildersEnabled,
-          isViewReadonlyColumnsEnabled,
         }
       })
     },
diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
index b4940c8903..dafc25a00b 100644
--- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
@@ -4,16 +4,13 @@
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
   import { FieldPermissions } from "../../../constants"
 
-  export let allowViewReadonlyColumns = false
-
   const { columns, datasource } = getContext("grid")
 
   let open = false
   let anchor
 
   $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
-  $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns"
-
+  $: text = anyRestricted ? `Columns: ${anyRestricted} restricted` : "Columns"
   $: permissions =
     $datasource.type === "viewV2"
       ? [
@@ -22,9 +19,6 @@
           FieldPermissions.HIDDEN,
         ]
       : [FieldPermissions.WRITABLE, FieldPermissions.HIDDEN]
-  $: disabledPermissions = allowViewReadonlyColumns
-    ? []
-    : [FieldPermissions.READONLY]
 </script>
 
 <div bind:this={anchor}>
@@ -41,9 +35,5 @@
 </div>
 
 <Popover bind:open {anchor} align="left">
-  <ColumnsSettingContent
-    columns={$columns}
-    {permissions}
-    {disabledPermissions}
-  />
+  <ColumnsSettingContent columns={$columns} {permissions} />
 </Popover>
diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte
index f24ff0ae10..f2aeffb9f4 100644
--- a/packages/frontend-core/src/components/grid/layout/Grid.svelte
+++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte
@@ -58,7 +58,6 @@
   export let buttons = null
   export let darkMode
   export let isCloud = null
-  export let allowViewReadonlyColumns = false
   export let rowConditions = null
 
   // Unique identifier for DOM nodes inside this instance
@@ -115,7 +114,6 @@
     buttons,
     darkMode,
     isCloud,
-    allowViewReadonlyColumns,
     rowConditions,
   })
 
@@ -157,7 +155,7 @@
       <div class="controls-left">
         <slot name="filter" />
         <SortButton />
-        <ColumnsSettingButton {allowViewReadonlyColumns} />
+        <ColumnsSettingButton />
         <SizeButton />
         <slot name="controls" />
       </div>
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index f86291e9cd..824b69a6bb 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -309,10 +309,6 @@ describe.each([
       })
 
       describe("readonly fields", () => {
-        beforeEach(() => {
-          mocks.licenses.useViewReadonlyColumns()
-        })
-
         it("readonly fields are persisted", async () => {
           const table = await config.api.table.save(
             saveTableRequest({
@@ -513,7 +509,6 @@ describe.each([
       })
 
       it("display fields can be readonly", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const table = await config.api.table.save(
           saveTableRequest({
             schema: {
@@ -588,7 +583,6 @@ describe.each([
       })
 
       it("can update all fields", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const tableId = table._id!
 
         const updatedData: Required<UpdateViewRequest> = {
@@ -803,8 +797,6 @@ describe.each([
       })
 
       it("cannot update views with readonly on on free license", async () => {
-        mocks.licenses.useViewReadonlyColumns()
-
         view = await config.api.viewV2.update({
           ...view,
           schema: {
@@ -826,8 +818,6 @@ describe.each([
       })
 
       it("can remove readonly config after license downgrade", async () => {
-        mocks.licenses.useViewReadonlyColumns()
-
         view = await config.api.viewV2.update({
           ...view,
           schema: {
@@ -1046,7 +1036,6 @@ describe.each([
       })
 
       it("should be able to fetch readonly config after downgrades", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const res = await config.api.viewV2.create({
           name: generator.name(),
           tableId: table._id!,
@@ -1112,8 +1101,6 @@ describe.each([
         })
 
         it("rejects if field is readonly in any view", async () => {
-          mocks.licenses.useViewReadonlyColumns()
-
           await config.api.viewV2.create({
             name: "view a",
             tableId: table._id!,
@@ -1538,7 +1525,6 @@ describe.each([
       })
 
       it("can't persist readonly columns", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
@@ -1607,7 +1593,6 @@ describe.each([
       })
 
       it("can't update readonly columns", async () => {
-        mocks.licenses.useViewReadonlyColumns()
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           name: generator.guid(),
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index d7e05abf2f..3ba91e152f 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -59,13 +59,6 @@ async function guardViewSchema(
     }
 
     if (viewSchema[field].readonly) {
-      if (
-        !(await features.isViewReadonlyColumnsEnabled()) &&
-        !(tableSchemaField as ViewFieldMetadata).readonly
-      ) {
-        throw new HTTPError(`Readonly fields are not enabled`, 400)
-      }
-
       if (!viewSchema[field].visible) {
         throw new HTTPError(
           `Field "${field}" must be visible if you want to make it readonly`,

From 4bd4cb57f80ffad3c05f93ad850501b6b68953e9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 17 Sep 2024 16:34:47 +0100
Subject: [PATCH 03/19] Undo accidental UI change

---
 .../src/components/grid/controls/ColumnsSettingButton.svelte    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
index dafc25a00b..2a3ca139fc 100644
--- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
@@ -10,7 +10,7 @@
   let anchor
 
   $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
-  $: text = anyRestricted ? `Columns: ${anyRestricted} restricted` : "Columns"
+  $: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns"
   $: permissions =
     $datasource.type === "viewV2"
       ? [

From 4325805ce361d3e04792785753eed6bde24061ac Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 11:54:24 +0100
Subject: [PATCH 04/19] Lint

---
 .../src/components/backend/DataTable/ViewV2DataTable.svelte     | 2 +-
 packages/server/src/sdk/app/views/index.ts                      | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index ed35c5d6a3..90e5e216f3 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2 } from "stores/builder"
-  import { admin, licensing } from "stores/portal"
+  import { admin } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index 3ba91e152f..269158e61e 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -5,13 +5,11 @@ import {
   Table,
   TableSchema,
   View,
-  ViewFieldMetadata,
   ViewV2,
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
 import { HTTPError } from "@budibase/backend-core"
-import { features } from "@budibase/pro"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,

From c8c4248b3cab659802c5f77251633573bc340061 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 11:55:40 +0100
Subject: [PATCH 05/19] Update tests

---
 .../src/api/routes/tests/viewV2.spec.ts       | 47 +------------------
 1 file changed, 2 insertions(+), 45 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 824b69a6bb..201c7db3d7 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -796,7 +796,7 @@ describe.each([
         )
       })
 
-      it("cannot update views with readonly on on free license", async () => {
+      it("can views with readonly on on free license", async () => {
         view = await config.api.viewV2.update({
           ...view,
           schema: {
@@ -810,53 +810,10 @@ describe.each([
 
         mocks.licenses.useCloudFree()
         await config.api.viewV2.update(view, {
-          status: 400,
-          body: {
-            message: "Readonly fields are not enabled",
-          },
+          status: 200,
         })
       })
 
-      it("can remove readonly config after license downgrade", async () => {
-        view = await config.api.viewV2.update({
-          ...view,
-          schema: {
-            id: { visible: true },
-            Price: {
-              visible: true,
-              readonly: true,
-            },
-            Category: {
-              visible: true,
-              readonly: true,
-            },
-          },
-        })
-        mocks.licenses.useCloudFree()
-        const res = await config.api.viewV2.update({
-          ...view,
-          schema: {
-            id: { visible: true },
-            Price: {
-              visible: true,
-              readonly: false,
-            },
-          },
-        })
-        expect(res).toEqual(
-          expect.objectContaining({
-            ...view,
-            schema: {
-              id: { visible: true },
-              Price: {
-                visible: true,
-                readonly: false,
-              },
-            },
-          })
-        )
-      })
-
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({

From fd9a2f87d12dcc6b5cafa8f640bfaa508bbb7b2d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 12:35:05 +0100
Subject: [PATCH 06/19] Update more tests

---
 packages/server/src/api/routes/tests/viewV2.spec.ts | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 201c7db3d7..65448a5bf3 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -432,7 +432,7 @@ describe.each([
           })
         })
 
-        it("readonly fields cannot be used on free license", async () => {
+        it("readonly fields can be used on free license", async () => {
           mocks.licenses.useCloudFree()
           const table = await config.api.table.save(
             saveTableRequest({
@@ -462,11 +462,7 @@ describe.each([
           }
 
           await config.api.viewV2.create(newView, {
-            status: 400,
-            body: {
-              message: "Readonly fields are not enabled",
-              status: 400,
-            },
+            status: 200,
           })
         })
       })

From a0350a75efe5fd8132975a56d84d18554fd16a9f Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 13:17:30 +0100
Subject: [PATCH 07/19] Update tests again

---
 .../src/api/routes/tests/viewV2.spec.ts       | 20 +------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 65448a5bf3..c4a39ae8a9 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -462,7 +462,7 @@ describe.each([
           }
 
           await config.api.viewV2.create(newView, {
-            status: 200,
+            status: 201,
           })
         })
       })
@@ -792,24 +792,6 @@ describe.each([
         )
       })
 
-      it("can views with readonly on on free license", async () => {
-        view = await config.api.viewV2.update({
-          ...view,
-          schema: {
-            id: { visible: true },
-            Price: {
-              visible: true,
-              readonly: true,
-            },
-          },
-        })
-
-        mocks.licenses.useCloudFree()
-        await config.api.viewV2.update(view, {
-          status: 200,
-        })
-      })
-
       isInternal &&
         it("updating schema will only validate modified field", async () => {
           let view = await config.api.viewV2.create({

From da122d2ac157bc26ba69bfdec32a184469033539 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 18 Sep 2024 14:55:09 +0100
Subject: [PATCH 08/19] Add table ID to row deletion requests that only contain
 string IDs

---
 packages/server/src/api/controllers/row/index.ts |  2 +-
 packages/server/src/api/routes/tests/row.spec.ts | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index cd85f57982..2e5785157d 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
   const { tableId } = utils.getSourceId(ctx)
 
   const processedRows = request.rows.map(row => {
-    let processedRow: Row = typeof row == "string" ? { _id: row } : row
+    let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row
     return !processedRow._rev
       ? addRev(fixRow(processedRow, ctx.params), tableId)
       : fixRow(processedRow, ctx.params)
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9790703806..dc03a21d6d 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1138,6 +1138,18 @@ describe.each([
       await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
     })
 
+    it("should be able to delete a row with ID only", async () => {
+      const createdRow = await config.api.row.save(table._id!, {})
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.bulkDelete(table._id!, {
+        rows: [createdRow._id!],
+      })
+      expect(res[0]._id).toEqual(createdRow._id)
+      expect(res[0].tableId).toEqual(table._id!)
+      await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
+    })
+
     it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
       const createdRow = await config.api.row.save(table._id!, {})
       const createdRow2 = await config.api.row.save(table._id!, {})

From f2f13bb84eda9645275598580443ee54c60a3194 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 19 Sep 2024 09:25:25 +0100
Subject: [PATCH 09/19] Fix conditions with empty values falsely evaluating to
 true

---
 packages/client/src/utils/conditions.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js
index a23d879e45..b68afa4ab3 100644
--- a/packages/client/src/utils/conditions.js
+++ b/packages/client/src/utils/conditions.js
@@ -1,4 +1,5 @@
 import { QueryUtils } from "@budibase/frontend-core"
+import { EmptyFilterOption } from "@budibase/types"
 
 export const getActiveConditions = conditions => {
   if (!conditions?.length) {
@@ -33,7 +34,8 @@ export const getActiveConditions = conditions => {
       value: condition.referenceValue,
     }
 
-    const query = QueryUtils.buildQuery([luceneCondition])
+    let query = QueryUtils.buildQuery([luceneCondition])
+    query.onEmptyFilter = EmptyFilterOption.RETURN_NONE
     const result = QueryUtils.runQuery([luceneCondition], query)
     return result.length > 0
   })

From fd353b2ecf6c292a4ba6bcd5e18ba5c40ddfc194 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 10:09:21 +0100
Subject: [PATCH 10/19] wip

---
 .../server/src/integrations/googlesheets.ts   | 63 ++++++++------
 .../integrations/tests/googlesheets.spec.ts   | 19 ++++-
 .../integrations/tests/utils/googlesheets.ts  | 82 ++++++++++++-------
 packages/shared-core/src/utils.ts             | 53 ++++++------
 4 files changed, 133 insertions(+), 84 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index dd9bef84ab..65606ad75c 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -25,7 +25,11 @@ import {
   checkExternalTables,
   finaliseExternalTables,
 } from "./utils"
-import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
+import {
+  GoogleSpreadsheet,
+  GoogleSpreadsheetRow,
+  GoogleSpreadsheetWorksheet,
+} from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"
@@ -330,15 +334,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
       return { tables: {}, errors: {} }
     }
     await this.connect()
+
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
-    await utils.parallelForeach(
-      sheets,
-      async sheet => {
-        // must fetch rows to determine schema
+
+    let promises: Promise<void>[] = []
+    for (const sheet of sheets) {
+      const f = async (sheet: GoogleSpreadsheetWorksheet) => {
         try {
-          await sheet.getRows()
+          await sheet.getRows({ limit: 1 })
         } catch (err) {
           // We expect this to always be an Error so if it's not, rethrow it to
           // make sure we don't fail quietly.
@@ -346,26 +351,36 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
             throw err
           }
 
-          if (err.message.startsWith("No values in the header row")) {
-            errors[sheet.title] = err.message
-          } else {
-            // If we get an error we don't expect, rethrow to avoid failing
-            // quietly.
-            throw err
+          if (
+            err.message.startsWith("No values in the header row") ||
+            err.message.startsWith("Header values are not yet loaded")
+          ) {
+            errors[
+              sheet.title
+            ] = `Failed to find a header row in sheet "${sheet.title}", is the first row blank?`
+            return
           }
-          return
-        }
 
-        const id = buildExternalTableId(datasourceId, sheet.title)
-        tables[sheet.title] = this.getTableSchema(
-          sheet.title,
-          sheet.headerValues,
-          datasourceId,
-          id
-        )
-      },
-      10
-    )
+          // If we get an error we don't expect, rethrow to avoid failing
+          // quietly.
+          throw err
+        }
+      }
+      promises.push(f(sheet))
+    }
+
+    await Promise.all(promises)
+
+    for (const sheet of sheets) {
+      const id = buildExternalTableId(datasourceId, sheet.title)
+      tables[sheet.title] = this.getTableSchema(
+        sheet.title,
+        sheet.headerValues,
+        datasourceId,
+        id
+      )
+    }
+
     let externalTables = finaliseExternalTables(tables, entities)
     errors = { ...errors, ...checkExternalTables(externalTables) }
     return { tables: externalTables, errors }
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index cef4decfa7..fe2da479c3 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -10,7 +10,6 @@ import {
   TableSourceType,
 } from "@budibase/types"
 import { GoogleSheetsMock } from "./utils/googlesheets"
-import rows from "src/sdk/app/rows"
 
 describe("Google Sheets Integration", () => {
   const config = new TestConfiguration()
@@ -506,4 +505,22 @@ describe("Google Sheets Integration", () => {
       expect(emptyRows.length).toEqual(0)
     })
   })
+
+  describe("import spreadsheet", () => {
+    it.only("should fail to import a completely blank sheet", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource._id!,
+          tablesFilter: ["Sheet1"],
+        },
+        {
+          status: 400,
+          body: {
+            message: "",
+          },
+        }
+      )
+    })
+  })
 })
diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts
index 4b17c25b01..4747f5f9bf 100644
--- a/packages/server/src/integrations/tests/utils/googlesheets.ts
+++ b/packages/server/src/integrations/tests/utils/googlesheets.ts
@@ -22,6 +22,7 @@ import type {
   CellPadding,
   Color,
   GridRange,
+  DataSourceSheetProperties,
 } from "google-spreadsheet/src/lib/types/sheets-types"
 
 const BLACK: Color = { red: 0, green: 0, blue: 0 }
@@ -91,7 +92,7 @@ interface UpdateValuesResponse {
 
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest
 interface AddSheetRequest {
-  properties: WorksheetProperties
+  properties: Partial<WorksheetProperties>
 }
 
 // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse
@@ -236,6 +237,38 @@ export class GoogleSheetsMock {
     this.mockAPI()
   }
 
+  public cell(cell: string): Value | undefined {
+    const cellData = this.cellData(cell)
+    if (!cellData) {
+      return undefined
+    }
+    return this.cellValue(cellData)
+  }
+
+  public set(cell: string, value: Value): void {
+    const cellData = this.cellData(cell)
+    if (!cellData) {
+      throw new Error(`Cell ${cell} not found`)
+    }
+    cellData.userEnteredValue = this.createValue(value)
+  }
+
+  public sheet(name: string | number): Sheet | undefined {
+    if (typeof name === "number") {
+      return this.getSheetById(name)
+    }
+    return this.getSheetByName(name)
+  }
+
+  public createSheet(opts: Partial<WorksheetProperties>): Sheet {
+    const properties = this.defaultWorksheetProperties(opts)
+    if (this.getSheetByName(properties.title)) {
+      throw new Error(`Sheet ${properties.title} already exists`)
+    }
+    const resp = this.handleAddSheet({ properties })
+    return this.getSheetById(resp.properties.sheetId)!
+  }
+
   private route(
     method: "get" | "put" | "post",
     path: string | RegExp,
@@ -462,35 +495,39 @@ export class GoogleSheetsMock {
     return response
   }
 
-  private handleAddSheet(request: AddSheetRequest): AddSheetResponse {
-    const properties: Omit<WorksheetProperties, "dataSourceSheetProperties"> = {
+  private defaultWorksheetProperties(
+    opts: Partial<WorksheetProperties>
+  ): WorksheetProperties {
+    return {
       index: this.spreadsheet.sheets.length,
       hidden: false,
       rightToLeft: false,
       tabColor: BLACK,
       tabColorStyle: { rgbColor: BLACK },
       sheetType: "GRID",
-      title: request.properties.title,
+      title: "Sheet",
       sheetId: this.spreadsheet.sheets.length,
       gridProperties: {
         rowCount: 100,
         columnCount: 26,
-        frozenRowCount: 0,
-        frozenColumnCount: 0,
-        hideGridlines: false,
-        rowGroupControlAfter: false,
-        columnGroupControlAfter: false,
       },
+      dataSourceSheetProperties: {} as DataSourceSheetProperties,
+      ...opts,
     }
+  }
 
+  private handleAddSheet(request: AddSheetRequest): AddSheetResponse {
+    const properties = this.defaultWorksheetProperties(request.properties)
     this.spreadsheet.sheets.push({
-      properties: properties as WorksheetProperties,
-      data: [this.createEmptyGrid(100, 26)],
+      properties,
+      data: [
+        this.createEmptyGrid(
+          properties.gridProperties.rowCount,
+          properties.gridProperties.columnCount
+        ),
+      ],
     })
-
-    // dataSourceSheetProperties is only returned by the API if the sheet type is
-    // DATA_SOURCE, which we aren't using, so sadly we need to cast here.
-    return { properties: properties as WorksheetProperties }
+    return { properties }
   }
 
   private handleDeleteRange(request: DeleteRangeRequest) {
@@ -767,21 +804,6 @@ export class GoogleSheetsMock {
     return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex)
   }
 
-  public cell(cell: string): Value | undefined {
-    const cellData = this.cellData(cell)
-    if (!cellData) {
-      return undefined
-    }
-    return this.cellValue(cellData)
-  }
-
-  public sheet(name: string | number): Sheet | undefined {
-    if (typeof name === "number") {
-      return this.getSheetById(name)
-    }
-    return this.getSheetByName(name)
-  }
-
   private getCellNumericIndexes(
     sheet: Sheet | number,
     row: number,
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index b69a059745..31ce8778cd 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -7,43 +7,38 @@ export function unreachable(
   throw new Error(message)
 }
 
+interface PromiseWithId<T> {
+  promise: Promise<T>
+  id: number
+}
+
 export async function parallelForeach<T>(
   items: T[],
   task: (item: T) => Promise<void>,
   maxConcurrency: number
 ): Promise<void> {
-  const promises: Promise<void>[] = []
-  let index = 0
+  try {
+    let next = 0
+    let inProgress: PromiseWithId<number>[] = []
+    while (next < items.length) {
+      if (inProgress.length === maxConcurrency) {
+        const finished = await Promise.race(inProgress.map(t => t.promise))
+        inProgress = inProgress.filter(task => task.id !== finished)
+      }
 
-  const processItem = async (item: T) => {
-    try {
-      await task(item)
-    } finally {
-      processNext()
+      const promise = async (next: number) => {
+        await task(items[next])
+        return next
+      }
+
+      inProgress.push({ promise: promise(next), id: next })
+      next++
     }
+    await Promise.all(inProgress.map(t => t.promise))
+  } catch (e) {
+    console.error(e)
+    throw e
   }
-
-  const processNext = () => {
-    if (index >= items.length) {
-      // No more items to process
-      return
-    }
-
-    const item = items[index]
-    index++
-
-    const promise = processItem(item)
-    promises.push(promise)
-
-    if (promises.length >= maxConcurrency) {
-      Promise.race(promises).then(processNext)
-    } else {
-      processNext()
-    }
-  }
-  processNext()
-
-  await Promise.all(promises)
 }
 
 export function filterValueToLabel() {

From be5a4f5d9763122f38d5c083cd4ccd70944f5f0b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 10:11:24 +0100
Subject: [PATCH 11/19] Require a flag to be set to query PostHog for feature
 flags.

---
 packages/backend-core/src/environment.ts    | 1 +
 packages/backend-core/src/features/index.ts | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 6bef6efeb3..0ffe9966b1 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -143,6 +143,7 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
   POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
+  POSTHOG_FEATURE_FLAGS: process.env.POSTHOG_FEATURE_FLAGS,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
   CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index 5b6ea4ca92..fda95c6553 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -6,7 +6,12 @@ import tracer from "dd-trace"
 
 let posthog: PostHog | undefined
 export function init(opts?: PostHogOptions) {
-  if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) {
+  if (
+    env.POSTHOG_TOKEN &&
+    env.POSTHOG_API_HOST &&
+    !env.SELF_HOSTED &&
+    env.POSTHOG_FEATURE_FLAGS
+  ) {
     console.log("initializing posthog client...")
     posthog = new PostHog(env.POSTHOG_TOKEN, {
       host: env.POSTHOG_API_HOST,

From 6d7cffa43e269ffd1672e4e7de169cf45b2d69f4 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 10:15:18 +0100
Subject: [PATCH 12/19] Make flag name more accurate.

---
 packages/backend-core/src/environment.ts    | 2 +-
 packages/backend-core/src/features/index.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 0ffe9966b1..2ab8c550cc 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -143,7 +143,7 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
   POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
-  POSTHOG_FEATURE_FLAGS: process.env.POSTHOG_FEATURE_FLAGS,
+  POSTHOG_FEATURE_FLAGS_ENABLED: process.env.POSTHOG_FEATURE_FLAGS_ENABLED,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
   CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index fda95c6553..0765d09036 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -10,7 +10,7 @@ export function init(opts?: PostHogOptions) {
     env.POSTHOG_TOKEN &&
     env.POSTHOG_API_HOST &&
     !env.SELF_HOSTED &&
-    env.POSTHOG_FEATURE_FLAGS
+    env.POSTHOG_FEATURE_FLAGS_ENABLED
   ) {
     console.log("initializing posthog client...")
     posthog = new PostHog(env.POSTHOG_TOKEN, {

From 74effbb55b51da624bee420c5ef054ee93fbd31a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 11:41:10 +0100
Subject: [PATCH 13/19] Fix tests.

---
 packages/backend-core/src/features/tests/features.spec.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts
index d092585cc6..01c9bfa3c6 100644
--- a/packages/backend-core/src/features/tests/features.spec.ts
+++ b/packages/backend-core/src/features/tests/features.spec.ts
@@ -148,6 +148,7 @@ describe("feature flags", () => {
       const env: Partial<typeof environment> = {
         TENANT_FEATURE_FLAGS: environmentFlags,
         SELF_HOSTED: false,
+        POSTHOG_FEATURE_FLAGS_ENABLED: "true",
       }
 
       if (posthogFlags) {

From 5d945b90ff8a650239eac13b5d9a2b7f54c6c973 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 19 Sep 2024 12:58:57 +0000
Subject: [PATCH 14/19] Bump version to 2.32.6

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index c710d888c7..d695869907 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.5",
+  "version": "2.32.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From aecd4f9e4d205411e89084574c03c670a4426827 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:48:50 +0100
Subject: [PATCH 15/19] Fetch schema tests.

---
 packages/server/package.json                  |  2 +-
 .../server/src/integrations/googlesheets.ts   | 17 ++--
 .../integrations/tests/googlesheets.spec.ts   | 83 +++++++++++++++++-
 packages/shared-core/src/utils.ts             | 40 ++++-----
 yarn.lock                                     | 85 ++++++++++++++++++-
 5 files changed, 187 insertions(+), 40 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 6dfd528963..0b36d49c2e 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -80,7 +80,7 @@
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
-    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
     "jimp": "0.22.12",
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 65606ad75c..ea06ca8d69 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -339,9 +339,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
     const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
 
-    let promises: Promise<void>[] = []
-    for (const sheet of sheets) {
-      const f = async (sheet: GoogleSpreadsheetWorksheet) => {
+    await utils.parallelForEach(
+      sheets,
+      async sheet => {
         try {
           await sheet.getRows({ limit: 1 })
         } catch (err) {
@@ -353,7 +353,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
 
           if (
             err.message.startsWith("No values in the header row") ||
-            err.message.startsWith("Header values are not yet loaded")
+            err.message.startsWith("Header values are not yet loaded") ||
+            err.message.startsWith("All your header cells are blank")
           ) {
             errors[
               sheet.title
@@ -365,11 +366,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
           // quietly.
           throw err
         }
-      }
-      promises.push(f(sheet))
-    }
-
-    await Promise.all(promises)
+      },
+      { maxConcurrency: 2 }
+    )
 
     for (const sheet of sheets) {
       const id = buildExternalTableId(datasourceId, sheet.title)
diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts
index fe2da479c3..dcf4a61b50 100644
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@@ -506,8 +506,8 @@ describe("Google Sheets Integration", () => {
     })
   })
 
-  describe("import spreadsheet", () => {
-    it.only("should fail to import a completely blank sheet", async () => {
+  describe("fetch schema", () => {
+    it("should fail to import a completely blank sheet", async () => {
       mock.createSheet({ title: "Sheet1" })
       await config.api.datasource.fetchSchema(
         {
@@ -515,12 +515,87 @@ describe("Google Sheets Integration", () => {
           tablesFilter: ["Sheet1"],
         },
         {
-          status: 400,
+          status: 200,
           body: {
-            message: "",
+            errors: {
+              Sheet1:
+                'Failed to find a header row in sheet "Sheet1", is the first row blank?',
+            },
           },
         }
       )
     })
+
+    it("should fail to import multiple sheets with blank headers", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      mock.createSheet({ title: "Sheet2" })
+
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource!._id!,
+          tablesFilter: ["Sheet1", "Sheet2"],
+        },
+        {
+          status: 200,
+          body: {
+            errors: {
+              Sheet1:
+                'Failed to find a header row in sheet "Sheet1", is the first row blank?',
+              Sheet2:
+                'Failed to find a header row in sheet "Sheet2", is the first row blank?',
+            },
+          },
+        }
+      )
+    })
+
+    it("should only fail the sheet with missing headers", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      mock.createSheet({ title: "Sheet2" })
+      mock.createSheet({ title: "Sheet3" })
+
+      mock.set("Sheet1!A1", "name")
+      mock.set("Sheet1!B1", "dob")
+      mock.set("Sheet2!A1", "name")
+      mock.set("Sheet2!B1", "dob")
+
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource!._id!,
+          tablesFilter: ["Sheet1", "Sheet2", "Sheet3"],
+        },
+        {
+          status: 200,
+          body: {
+            errors: {
+              Sheet3:
+                'Failed to find a header row in sheet "Sheet3", is the first row blank?',
+            },
+          },
+        }
+      )
+    })
+
+    it("should only succeed if sheet with missing headers is not being imported", async () => {
+      mock.createSheet({ title: "Sheet1" })
+      mock.createSheet({ title: "Sheet2" })
+      mock.createSheet({ title: "Sheet3" })
+
+      mock.set("Sheet1!A1", "name")
+      mock.set("Sheet1!B1", "dob")
+      mock.set("Sheet2!A1", "name")
+      mock.set("Sheet2!B1", "dob")
+
+      await config.api.datasource.fetchSchema(
+        {
+          datasourceId: datasource!._id!,
+          tablesFilter: ["Sheet1", "Sheet2"],
+        },
+        {
+          status: 200,
+          body: { errors: {} },
+        }
+      )
+    })
   })
 })
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 31ce8778cd..9ead522192 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -12,33 +12,29 @@ interface PromiseWithId<T> {
   id: number
 }
 
-export async function parallelForeach<T>(
+export async function parallelForEach<T>(
   items: T[],
   task: (item: T) => Promise<void>,
-  maxConcurrency: number
+  opts?: { maxConcurrency?: number }
 ): Promise<void> {
-  try {
-    let next = 0
-    let inProgress: PromiseWithId<number>[] = []
-    while (next < items.length) {
-      if (inProgress.length === maxConcurrency) {
-        const finished = await Promise.race(inProgress.map(t => t.promise))
-        inProgress = inProgress.filter(task => task.id !== finished)
-      }
-
-      const promise = async (next: number) => {
-        await task(items[next])
-        return next
-      }
-
-      inProgress.push({ promise: promise(next), id: next })
-      next++
+  const { maxConcurrency = 10 } = opts || {}
+  let next = 0
+  let inProgress: PromiseWithId<number>[] = []
+  while (next < items.length) {
+    if (inProgress.length === maxConcurrency) {
+      const finished = await Promise.race(inProgress.map(t => t.promise))
+      inProgress = inProgress.filter(task => task.id !== finished)
     }
-    await Promise.all(inProgress.map(t => t.promise))
-  } catch (e) {
-    console.error(e)
-    throw e
+
+    const promise = async (next: number) => {
+      await task(items[next])
+      return next
+    }
+
+    inProgress.push({ promise: promise(next), id: next })
+    next++
   }
+  await Promise.all(inProgress.map(t => t.promise))
 }
 
 export function filterValueToLabel() {
diff --git a/yarn.lock b/yarn.lock
index 110cbd7a15..34c9cb15d9 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2053,6 +2053,44 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.32.6":
+  version "0.0.0"
+  dependencies:
+    "@budibase/nano" "10.1.5"
+    "@budibase/pouchdb-replication-stream" "1.2.11"
+    "@budibase/shared-core" "0.0.0"
+    "@budibase/types" "0.0.0"
+    aws-cloudfront-sign "3.0.2"
+    aws-sdk "2.1030.0"
+    bcrypt "5.1.0"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dd-trace "5.2.0"
+    dotenv "16.0.1"
+    ioredis "5.3.2"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    knex "2.4.2"
+    koa-passport "^6.0.0"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    pino "8.11.0"
+    pino-http "8.3.3"
+    posthog-node "4.0.1"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    rotating-file-stream "3.1.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "^7.5.4"
+    tar-fs "2.1.1"
+    uuid "^8.3.2"
+
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2095,6 +2133,45 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@npm:@budibase/pro@latest":
+  version "2.32.6"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f"
+  integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ==
+  dependencies:
+    "@budibase/backend-core" "2.32.6"
+    "@budibase/shared-core" "2.32.6"
+    "@budibase/string-templates" "2.32.6"
+    "@budibase/types" "2.32.6"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    dd-trace "5.2.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    lru-cache "^7.14.1"
+    memorystream "^0.3.1"
+    node-fetch "2.6.7"
+    scim-patch "^0.8.1"
+    scim2-parse-filter "^0.2.8"
+
+"@budibase/shared-core@2.32.6":
+  version "0.0.0"
+  dependencies:
+    "@budibase/types" "0.0.0"
+    cron-validate "1.4.5"
+
+"@budibase/string-templates@2.32.6":
+  version "0.0.0"
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.13.2"
+    dayjs "^1.10.8"
+    handlebars "^4.7.8"
+    lodash.clonedeep "^4.5.0"
+
+"@budibase/types@2.32.6":
+  version "0.0.0"
+  dependencies:
+    scim-patch "^0.8.1"
+
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -12277,10 +12354,10 @@ google-p12-pem@^4.0.0:
   dependencies:
     node-forge "^1.3.1"
 
-"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
-  integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
+"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5":
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f"
+  integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ==
   dependencies:
     axios "^1.4.0"
     lodash "^4.17.21"

From ce105d8f4e590d68008427ce3702e10ae3398a62 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:51:00 +0100
Subject: [PATCH 16/19] Revert unnecessary change.

---
 .../server/src/integrations/googlesheets.ts   |  4 +-
 packages/shared-core/src/utils.ts             | 53 +++++++++++--------
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index ea06ca8d69..d5667c276b 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -339,7 +339,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
     const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
 
-    await utils.parallelForEach(
+    await utils.parallelForeach(
       sheets,
       async sheet => {
         try {
@@ -367,7 +367,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
           throw err
         }
       },
-      { maxConcurrency: 2 }
+      10
     )
 
     for (const sheet of sheets) {
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 9ead522192..b69a059745 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -7,34 +7,43 @@ export function unreachable(
   throw new Error(message)
 }
 
-interface PromiseWithId<T> {
-  promise: Promise<T>
-  id: number
-}
-
-export async function parallelForEach<T>(
+export async function parallelForeach<T>(
   items: T[],
   task: (item: T) => Promise<void>,
-  opts?: { maxConcurrency?: number }
+  maxConcurrency: number
 ): Promise<void> {
-  const { maxConcurrency = 10 } = opts || {}
-  let next = 0
-  let inProgress: PromiseWithId<number>[] = []
-  while (next < items.length) {
-    if (inProgress.length === maxConcurrency) {
-      const finished = await Promise.race(inProgress.map(t => t.promise))
-      inProgress = inProgress.filter(task => task.id !== finished)
-    }
+  const promises: Promise<void>[] = []
+  let index = 0
 
-    const promise = async (next: number) => {
-      await task(items[next])
-      return next
+  const processItem = async (item: T) => {
+    try {
+      await task(item)
+    } finally {
+      processNext()
     }
-
-    inProgress.push({ promise: promise(next), id: next })
-    next++
   }
-  await Promise.all(inProgress.map(t => t.promise))
+
+  const processNext = () => {
+    if (index >= items.length) {
+      // No more items to process
+      return
+    }
+
+    const item = items[index]
+    index++
+
+    const promise = processItem(item)
+    promises.push(promise)
+
+    if (promises.length >= maxConcurrency) {
+      Promise.race(promises).then(processNext)
+    } else {
+      processNext()
+    }
+  }
+  processNext()
+
+  await Promise.all(promises)
 }
 
 export function filterValueToLabel() {

From d0a0e74f393903143896d4c7ffda596bfb717705 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:51:26 +0100
Subject: [PATCH 17/19] Remove unused type.

---
 packages/server/src/integrations/googlesheets.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index d5667c276b..8710932ef6 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -25,11 +25,7 @@ import {
   checkExternalTables,
   finaliseExternalTables,
 } from "./utils"
-import {
-  GoogleSpreadsheet,
-  GoogleSpreadsheetRow,
-  GoogleSpreadsheetWorksheet,
-} from "google-spreadsheet"
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"

From bd618f2b006055ec70dcfce8e3233f335085bd8f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 19 Sep 2024 16:53:00 +0100
Subject: [PATCH 18/19] Remove unneeded error message check.

---
 packages/server/src/integrations/googlesheets.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 8710932ef6..6012ff7789 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -349,7 +349,6 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
 
           if (
             err.message.startsWith("No values in the header row") ||
-            err.message.startsWith("Header values are not yet loaded") ||
             err.message.startsWith("All your header cells are blank")
           ) {
             errors[

From 627c1e713eaaa05a3454bc51f3b29d782cb998e7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 20 Sep 2024 14:52:28 +0100
Subject: [PATCH 19/19] Updating CouchDB image versions across the board to
 v3.3.3-sqs-v2.1.1

---
 .github/workflows/budibase_ci.yml | 4 ++--
 charts/budibase/values.yaml       | 2 +-
 globalSetup.ts                    | 2 +-
 hosting/single/Dockerfile         | 2 +-
 scripts/build-single-image-sqs.sh | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 4c5cc94d2b..4b9ebf1e5d 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -108,7 +108,7 @@ jobs:
       - name: Pull testcontainers images
         run: |
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb:v3.3.3 &
+          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
           docker pull redis &
 
           wait $(jobs -p)
@@ -179,7 +179,7 @@ jobs:
           docker pull minio/minio &
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb:v3.3.3 &
+          docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
 
           wait $(jobs -p)
 
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index dde912410c..2c1525bd90 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -641,7 +641,7 @@ couchdb:
     # @ignore
     repository: budibase/couchdb
     # @ignore
-    tag: v3.3.3
+    tag: v3.3.3-sqs-v2.1.1
     # @ignore
     pullPolicy: Always
 
diff --git a/globalSetup.ts b/globalSetup.ts
index aa1cb00fe1..5d8b0381c0 100644
--- a/globalSetup.ts
+++ b/globalSetup.ts
@@ -46,7 +46,7 @@ export default async function setup() {
   await killContainers(containers)
 
   try {
-    const couchdb = new GenericContainer("budibase/couchdb:v3.3.3")
+    const couchdb = new GenericContainer("budibase/couchdb:v3.3.3-sqs-v2.1.1")
       .withExposedPorts(5984, 4984)
       .withEnvironment({
         COUCHDB_PASSWORD: "budibase",
diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile
index dfcfe566bd..ded0bc17dc 100644
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASEIMG=budibase/couchdb:v3.3.3
+ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1
 FROM node:20-slim as build
 
 # install node-gyp dependencies
diff --git a/scripts/build-single-image-sqs.sh b/scripts/build-single-image-sqs.sh
index 502ba5fa14..40b97013a1 100644
--- a/scripts/build-single-image-sqs.sh
+++ b/scripts/build-single-image-sqs.sh
@@ -2,4 +2,4 @@
 
 yarn build:apps
 version=$(./scripts/getCurrentVersion.sh)
-docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs .
+docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 .