Improve fetch performance

packages/server/src/integrations/googlesheets.ts

@@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
 import sdk from "../sdk"
 
@@ -277,17 +277,21 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    for (let sheet of sheets) {
+    await utils.parallelForeach(
-      // must fetch rows to determine schema
+      sheets,
-      await sheet.getRows()
+      async sheet => {
+        // must fetch rows to determine schema
+        await sheet.getRows({ limit: 0, offset: 0 })
 
-      const id = buildExternalTableId(datasourceId, sheet.title)
+        const id = buildExternalTableId(datasourceId, sheet.title)
-      tables[sheet.title] = this.getTableSchema(
+        tables[sheet.title] = this.getTableSchema(
-        sheet.title,
+          sheet.title,
-        sheet.headerValues,
+          sheet.headerValues,
-        id
+          id
-      )
+        )
-    }
+      },
+      10
+    )
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors

packages/shared-core/src/utils.ts

@@ -4,3 +4,42 @@ export function unreachable(
 ) {
   throw new Error(message)
 }
+
+export async function parallelForeach<T>(
+  items: T[],
+  task: (item: T) => Promise<void>,
+  maxConcurrency: number
+): Promise<void> {
+  const promises: Promise<void>[] = []
+  let index = 0
+
+  const processItem = async (item: T) => {
+    try {
+      await task(item)
+    } finally {
+      processNext()
+    }
+  }
+
+  const processNext = () => {
+    if (index >= items.length) {
+      // No more items to process
+      return
+    }
+
+    const item = items[index]
+    index++
+
+    const promise = processItem(item)
+    promises.push(promise)
+
+    if (promises.length >= maxConcurrency) {
+      Promise.race(promises).then(processNext)
+    } else {
+      processNext()
+    }
+  }
+  processNext()
+
+  await Promise.all(promises)
+}