From 27f6fa7de46ba2f5c482cdcc20b4a866181606e1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 16:36:17 +0100 Subject: [PATCH 1/9] Add a test for row exports on Google Sheets. --- .../src/integrations/tests/googlesheets.spec.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 62d56bb2c2..cef4decfa7 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -10,6 +10,7 @@ import { TableSourceType, } from "@budibase/types" import { GoogleSheetsMock } from "./utils/googlesheets" +import rows from "src/sdk/app/rows" describe("Google Sheets Integration", () => { const config = new TestConfiguration() @@ -244,6 +245,20 @@ describe("Google Sheets Integration", () => { expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`)) ) }) + + it("can export rows", async () => { + const resp = await config.api.row.exportRows(table._id!, {}) + const parsed = JSON.parse(resp) + expect(parsed.length).toEqual(2) + expect(parsed[0]).toMatchObject({ + name: "Test Contact 1", + description: "original description 1", + }) + expect(parsed[1]).toMatchObject({ + name: "Test Contact 2", + description: "original description 2", + }) + }) }) describe("update", () => { From da122d2ac157bc26ba69bfdec32a184469033539 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 18 Sep 2024 14:55:09 +0100 Subject: [PATCH 2/9] Add table ID to row deletion requests that only contain string IDs --- packages/server/src/api/controllers/row/index.ts | 2 +- packages/server/src/api/routes/tests/row.spec.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index cd85f57982..2e5785157d 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx) { const { tableId } = utils.getSourceId(ctx) const processedRows = request.rows.map(row => { - let processedRow: Row = typeof row == "string" ? { _id: row } : row + let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row return !processedRow._rev ? addRev(fixRow(processedRow, ctx.params), tableId) : fixRow(processedRow, ctx.params) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9790703806..dc03a21d6d 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1138,6 +1138,18 @@ describe.each([ await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) }) + it("should be able to delete a row with ID only", async () => { + const createdRow = await config.api.row.save(table._id!, {}) + const rowUsage = await getRowUsage() + + const res = await config.api.row.bulkDelete(table._id!, { + rows: [createdRow._id!], + }) + expect(res[0]._id).toEqual(createdRow._id) + expect(res[0].tableId).toEqual(table._id!) + await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) + }) + it("should be able to bulk delete rows, including a row that doesn't exist", async () => { const createdRow = await config.api.row.save(table._id!, {}) const createdRow2 = await config.api.row.save(table._id!, {}) From f2f13bb84eda9645275598580443ee54c60a3194 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 19 Sep 2024 09:25:25 +0100 Subject: [PATCH 3/9] Fix conditions with empty values falsely evaluating to true --- packages/client/src/utils/conditions.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js index a23d879e45..b68afa4ab3 100644 --- a/packages/client/src/utils/conditions.js +++ b/packages/client/src/utils/conditions.js @@ -1,4 +1,5 @@ import { QueryUtils } from "@budibase/frontend-core" +import { EmptyFilterOption } from "@budibase/types" export const getActiveConditions = conditions => { if (!conditions?.length) { @@ -33,7 +34,8 @@ export const getActiveConditions = conditions => { value: condition.referenceValue, } - const query = QueryUtils.buildQuery([luceneCondition]) + let query = QueryUtils.buildQuery([luceneCondition]) + query.onEmptyFilter = EmptyFilterOption.RETURN_NONE const result = QueryUtils.runQuery([luceneCondition], query) return result.length > 0 }) From fd353b2ecf6c292a4ba6bcd5e18ba5c40ddfc194 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 10:09:21 +0100 Subject: [PATCH 4/9] wip --- .../server/src/integrations/googlesheets.ts | 63 ++++++++------ .../integrations/tests/googlesheets.spec.ts | 19 ++++- .../integrations/tests/utils/googlesheets.ts | 82 ++++++++++++------- packages/shared-core/src/utils.ts | 53 ++++++------ 4 files changed, 133 insertions(+), 84 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index dd9bef84ab..65606ad75c 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -25,7 +25,11 @@ import { checkExternalTables, finaliseExternalTables, } from "./utils" -import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" +import { + GoogleSpreadsheet, + GoogleSpreadsheetRow, + GoogleSpreadsheetWorksheet, +} from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" import { dataFilters, utils } from "@budibase/shared-core" @@ -330,15 +334,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return { tables: {}, errors: {} } } await this.connect() + const sheets = this.client.sheetsByIndex const tables: Record = {} let errors: Record = {} - await utils.parallelForeach( - sheets, - async sheet => { - // must fetch rows to determine schema + + let promises: Promise[] = [] + for (const sheet of sheets) { + const f = async (sheet: GoogleSpreadsheetWorksheet) => { try { - await sheet.getRows() + await sheet.getRows({ limit: 1 }) } catch (err) { // We expect this to always be an Error so if it's not, rethrow it to // make sure we don't fail quietly. @@ -346,26 +351,36 @@ export class GoogleSheetsIntegration implements DatasourcePlus { throw err } - if (err.message.startsWith("No values in the header row")) { - errors[sheet.title] = err.message - } else { - // If we get an error we don't expect, rethrow to avoid failing - // quietly. - throw err + if ( + err.message.startsWith("No values in the header row") || + err.message.startsWith("Header values are not yet loaded") + ) { + errors[ + sheet.title + ] = `Failed to find a header row in sheet "${sheet.title}", is the first row blank?` + return } - return - } - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - datasourceId, - id - ) - }, - 10 - ) + // If we get an error we don't expect, rethrow to avoid failing + // quietly. + throw err + } + } + promises.push(f(sheet)) + } + + await Promise.all(promises) + + for (const sheet of sheets) { + const id = buildExternalTableId(datasourceId, sheet.title) + tables[sheet.title] = this.getTableSchema( + sheet.title, + sheet.headerValues, + datasourceId, + id + ) + } + let externalTables = finaliseExternalTables(tables, entities) errors = { ...errors, ...checkExternalTables(externalTables) } return { tables: externalTables, errors } diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index cef4decfa7..fe2da479c3 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -10,7 +10,6 @@ import { TableSourceType, } from "@budibase/types" import { GoogleSheetsMock } from "./utils/googlesheets" -import rows from "src/sdk/app/rows" describe("Google Sheets Integration", () => { const config = new TestConfiguration() @@ -506,4 +505,22 @@ describe("Google Sheets Integration", () => { expect(emptyRows.length).toEqual(0) }) }) + + describe("import spreadsheet", () => { + it.only("should fail to import a completely blank sheet", async () => { + mock.createSheet({ title: "Sheet1" }) + await config.api.datasource.fetchSchema( + { + datasourceId: datasource._id!, + tablesFilter: ["Sheet1"], + }, + { + status: 400, + body: { + message: "", + }, + } + ) + }) + }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 4b17c25b01..4747f5f9bf 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -22,6 +22,7 @@ import type { CellPadding, Color, GridRange, + DataSourceSheetProperties, } from "google-spreadsheet/src/lib/types/sheets-types" const BLACK: Color = { red: 0, green: 0, blue: 0 } @@ -91,7 +92,7 @@ interface UpdateValuesResponse { // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest interface AddSheetRequest { - properties: WorksheetProperties + properties: Partial } // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse @@ -236,6 +237,38 @@ export class GoogleSheetsMock { this.mockAPI() } + public cell(cell: string): Value | undefined { + const cellData = this.cellData(cell) + if (!cellData) { + return undefined + } + return this.cellValue(cellData) + } + + public set(cell: string, value: Value): void { + const cellData = this.cellData(cell) + if (!cellData) { + throw new Error(`Cell ${cell} not found`) + } + cellData.userEnteredValue = this.createValue(value) + } + + public sheet(name: string | number): Sheet | undefined { + if (typeof name === "number") { + return this.getSheetById(name) + } + return this.getSheetByName(name) + } + + public createSheet(opts: Partial): Sheet { + const properties = this.defaultWorksheetProperties(opts) + if (this.getSheetByName(properties.title)) { + throw new Error(`Sheet ${properties.title} already exists`) + } + const resp = this.handleAddSheet({ properties }) + return this.getSheetById(resp.properties.sheetId)! + } + private route( method: "get" | "put" | "post", path: string | RegExp, @@ -462,35 +495,39 @@ export class GoogleSheetsMock { return response } - private handleAddSheet(request: AddSheetRequest): AddSheetResponse { - const properties: Omit = { + private defaultWorksheetProperties( + opts: Partial + ): WorksheetProperties { + return { index: this.spreadsheet.sheets.length, hidden: false, rightToLeft: false, tabColor: BLACK, tabColorStyle: { rgbColor: BLACK }, sheetType: "GRID", - title: request.properties.title, + title: "Sheet", sheetId: this.spreadsheet.sheets.length, gridProperties: { rowCount: 100, columnCount: 26, - frozenRowCount: 0, - frozenColumnCount: 0, - hideGridlines: false, - rowGroupControlAfter: false, - columnGroupControlAfter: false, }, + dataSourceSheetProperties: {} as DataSourceSheetProperties, + ...opts, } + } + private handleAddSheet(request: AddSheetRequest): AddSheetResponse { + const properties = this.defaultWorksheetProperties(request.properties) this.spreadsheet.sheets.push({ - properties: properties as WorksheetProperties, - data: [this.createEmptyGrid(100, 26)], + properties, + data: [ + this.createEmptyGrid( + properties.gridProperties.rowCount, + properties.gridProperties.columnCount + ), + ], }) - - // dataSourceSheetProperties is only returned by the API if the sheet type is - // DATA_SOURCE, which we aren't using, so sadly we need to cast here. - return { properties: properties as WorksheetProperties } + return { properties } } private handleDeleteRange(request: DeleteRangeRequest) { @@ -767,21 +804,6 @@ export class GoogleSheetsMock { return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex) } - public cell(cell: string): Value | undefined { - const cellData = this.cellData(cell) - if (!cellData) { - return undefined - } - return this.cellValue(cellData) - } - - public sheet(name: string | number): Sheet | undefined { - if (typeof name === "number") { - return this.getSheetById(name) - } - return this.getSheetByName(name) - } - private getCellNumericIndexes( sheet: Sheet | number, row: number, diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b69a059745..31ce8778cd 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -7,43 +7,38 @@ export function unreachable( throw new Error(message) } +interface PromiseWithId { + promise: Promise + id: number +} + export async function parallelForeach( items: T[], task: (item: T) => Promise, maxConcurrency: number ): Promise { - const promises: Promise[] = [] - let index = 0 + try { + let next = 0 + let inProgress: PromiseWithId[] = [] + while (next < items.length) { + if (inProgress.length === maxConcurrency) { + const finished = await Promise.race(inProgress.map(t => t.promise)) + inProgress = inProgress.filter(task => task.id !== finished) + } - const processItem = async (item: T) => { - try { - await task(item) - } finally { - processNext() + const promise = async (next: number) => { + await task(items[next]) + return next + } + + inProgress.push({ promise: promise(next), id: next }) + next++ } + await Promise.all(inProgress.map(t => t.promise)) + } catch (e) { + console.error(e) + throw e } - - const processNext = () => { - if (index >= items.length) { - // No more items to process - return - } - - const item = items[index] - index++ - - const promise = processItem(item) - promises.push(promise) - - if (promises.length >= maxConcurrency) { - Promise.race(promises).then(processNext) - } else { - processNext() - } - } - processNext() - - await Promise.all(promises) } export function filterValueToLabel() { From 5d945b90ff8a650239eac13b5d9a2b7f54c6c973 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 19 Sep 2024 12:58:57 +0000 Subject: [PATCH 5/9] Bump version to 2.32.6 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c710d888c7..d695869907 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.5", + "version": "2.32.6", "npmClient": "yarn", "packages": [ "packages/*", From aecd4f9e4d205411e89084574c03c670a4426827 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 16:48:50 +0100 Subject: [PATCH 6/9] Fetch schema tests. --- packages/server/package.json | 2 +- .../server/src/integrations/googlesheets.ts | 17 ++-- .../integrations/tests/googlesheets.spec.ts | 83 +++++++++++++++++- packages/shared-core/src/utils.ts | 40 ++++----- yarn.lock | 85 ++++++++++++++++++- 5 files changed, 187 insertions(+), 40 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 6dfd528963..0b36d49c2e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -80,7 +80,7 @@ "dotenv": "8.2.0", "form-data": "4.0.0", "global-agent": "3.0.0", - "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3", + "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5", "ioredis": "5.3.2", "isolated-vm": "^4.7.2", "jimp": "0.22.12", diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 65606ad75c..ea06ca8d69 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -339,9 +339,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus { const tables: Record = {} let errors: Record = {} - let promises: Promise[] = [] - for (const sheet of sheets) { - const f = async (sheet: GoogleSpreadsheetWorksheet) => { + await utils.parallelForEach( + sheets, + async sheet => { try { await sheet.getRows({ limit: 1 }) } catch (err) { @@ -353,7 +353,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus { if ( err.message.startsWith("No values in the header row") || - err.message.startsWith("Header values are not yet loaded") + err.message.startsWith("Header values are not yet loaded") || + err.message.startsWith("All your header cells are blank") ) { errors[ sheet.title @@ -365,11 +366,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus { // quietly. throw err } - } - promises.push(f(sheet)) - } - - await Promise.all(promises) + }, + { maxConcurrency: 2 } + ) for (const sheet of sheets) { const id = buildExternalTableId(datasourceId, sheet.title) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index fe2da479c3..dcf4a61b50 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -506,8 +506,8 @@ describe("Google Sheets Integration", () => { }) }) - describe("import spreadsheet", () => { - it.only("should fail to import a completely blank sheet", async () => { + describe("fetch schema", () => { + it("should fail to import a completely blank sheet", async () => { mock.createSheet({ title: "Sheet1" }) await config.api.datasource.fetchSchema( { @@ -515,12 +515,87 @@ describe("Google Sheets Integration", () => { tablesFilter: ["Sheet1"], }, { - status: 400, + status: 200, body: { - message: "", + errors: { + Sheet1: + 'Failed to find a header row in sheet "Sheet1", is the first row blank?', + }, }, } ) }) + + it("should fail to import multiple sheets with blank headers", async () => { + mock.createSheet({ title: "Sheet1" }) + mock.createSheet({ title: "Sheet2" }) + + await config.api.datasource.fetchSchema( + { + datasourceId: datasource!._id!, + tablesFilter: ["Sheet1", "Sheet2"], + }, + { + status: 200, + body: { + errors: { + Sheet1: + 'Failed to find a header row in sheet "Sheet1", is the first row blank?', + Sheet2: + 'Failed to find a header row in sheet "Sheet2", is the first row blank?', + }, + }, + } + ) + }) + + it("should only fail the sheet with missing headers", async () => { + mock.createSheet({ title: "Sheet1" }) + mock.createSheet({ title: "Sheet2" }) + mock.createSheet({ title: "Sheet3" }) + + mock.set("Sheet1!A1", "name") + mock.set("Sheet1!B1", "dob") + mock.set("Sheet2!A1", "name") + mock.set("Sheet2!B1", "dob") + + await config.api.datasource.fetchSchema( + { + datasourceId: datasource!._id!, + tablesFilter: ["Sheet1", "Sheet2", "Sheet3"], + }, + { + status: 200, + body: { + errors: { + Sheet3: + 'Failed to find a header row in sheet "Sheet3", is the first row blank?', + }, + }, + } + ) + }) + + it("should only succeed if sheet with missing headers is not being imported", async () => { + mock.createSheet({ title: "Sheet1" }) + mock.createSheet({ title: "Sheet2" }) + mock.createSheet({ title: "Sheet3" }) + + mock.set("Sheet1!A1", "name") + mock.set("Sheet1!B1", "dob") + mock.set("Sheet2!A1", "name") + mock.set("Sheet2!B1", "dob") + + await config.api.datasource.fetchSchema( + { + datasourceId: datasource!._id!, + tablesFilter: ["Sheet1", "Sheet2"], + }, + { + status: 200, + body: { errors: {} }, + } + ) + }) }) }) diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 31ce8778cd..9ead522192 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -12,33 +12,29 @@ interface PromiseWithId { id: number } -export async function parallelForeach( +export async function parallelForEach( items: T[], task: (item: T) => Promise, - maxConcurrency: number + opts?: { maxConcurrency?: number } ): Promise { - try { - let next = 0 - let inProgress: PromiseWithId[] = [] - while (next < items.length) { - if (inProgress.length === maxConcurrency) { - const finished = await Promise.race(inProgress.map(t => t.promise)) - inProgress = inProgress.filter(task => task.id !== finished) - } - - const promise = async (next: number) => { - await task(items[next]) - return next - } - - inProgress.push({ promise: promise(next), id: next }) - next++ + const { maxConcurrency = 10 } = opts || {} + let next = 0 + let inProgress: PromiseWithId[] = [] + while (next < items.length) { + if (inProgress.length === maxConcurrency) { + const finished = await Promise.race(inProgress.map(t => t.promise)) + inProgress = inProgress.filter(task => task.id !== finished) } - await Promise.all(inProgress.map(t => t.promise)) - } catch (e) { - console.error(e) - throw e + + const promise = async (next: number) => { + await task(items[next]) + return next + } + + inProgress.push({ promise: promise(next), id: next }) + next++ } + await Promise.all(inProgress.map(t => t.promise)) } export function filterValueToLabel() { diff --git a/yarn.lock b/yarn.lock index 110cbd7a15..34c9cb15d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2053,6 +2053,44 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/backend-core@2.32.6": + version "0.0.0" + dependencies: + "@budibase/nano" "10.1.5" + "@budibase/pouchdb-replication-stream" "1.2.11" + "@budibase/shared-core" "0.0.0" + "@budibase/types" "0.0.0" + aws-cloudfront-sign "3.0.2" + aws-sdk "2.1030.0" + bcrypt "5.1.0" + bcryptjs "2.4.3" + bull "4.10.1" + correlation-id "4.0.0" + dd-trace "5.2.0" + dotenv "16.0.1" + ioredis "5.3.2" + joi "17.6.0" + jsonwebtoken "9.0.2" + knex "2.4.2" + koa-passport "^6.0.0" + koa-pino-logger "4.0.0" + lodash "4.17.21" + node-fetch "2.6.7" + passport-google-oauth "2.0.0" + passport-local "1.0.0" + passport-oauth2-refresh "^2.1.0" + pino "8.11.0" + pino-http "8.3.3" + posthog-node "4.0.1" + pouchdb "7.3.0" + pouchdb-find "7.2.2" + redlock "4.2.0" + rotating-file-stream "3.1.0" + sanitize-s3-objectkey "0.0.1" + semver "^7.5.4" + tar-fs "2.1.1" + uuid "^8.3.2" + "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2095,6 +2133,45 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" +"@budibase/pro@npm:@budibase/pro@latest": + version "2.32.6" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f" + integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ== + dependencies: + "@budibase/backend-core" "2.32.6" + "@budibase/shared-core" "2.32.6" + "@budibase/string-templates" "2.32.6" + "@budibase/types" "2.32.6" + "@koa/router" "8.0.8" + bull "4.10.1" + dd-trace "5.2.0" + joi "17.6.0" + jsonwebtoken "9.0.2" + lru-cache "^7.14.1" + memorystream "^0.3.1" + node-fetch "2.6.7" + scim-patch "^0.8.1" + scim2-parse-filter "^0.2.8" + +"@budibase/shared-core@2.32.6": + version "0.0.0" + dependencies: + "@budibase/types" "0.0.0" + cron-validate "1.4.5" + +"@budibase/string-templates@2.32.6": + version "0.0.0" + dependencies: + "@budibase/handlebars-helpers" "^0.13.2" + dayjs "^1.10.8" + handlebars "^4.7.8" + lodash.clonedeep "^4.5.0" + +"@budibase/types@2.32.6": + version "0.0.0" + dependencies: + scim-patch "^0.8.1" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -12277,10 +12354,10 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad" - integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w== +"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f" + integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ== dependencies: axios "^1.4.0" lodash "^4.17.21" From ce105d8f4e590d68008427ce3702e10ae3398a62 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 16:51:00 +0100 Subject: [PATCH 7/9] Revert unnecessary change. --- .../server/src/integrations/googlesheets.ts | 4 +- packages/shared-core/src/utils.ts | 53 +++++++++++-------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index ea06ca8d69..d5667c276b 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -339,7 +339,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { const tables: Record = {} let errors: Record = {} - await utils.parallelForEach( + await utils.parallelForeach( sheets, async sheet => { try { @@ -367,7 +367,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { throw err } }, - { maxConcurrency: 2 } + 10 ) for (const sheet of sheets) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 9ead522192..b69a059745 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -7,34 +7,43 @@ export function unreachable( throw new Error(message) } -interface PromiseWithId { - promise: Promise - id: number -} - -export async function parallelForEach( +export async function parallelForeach( items: T[], task: (item: T) => Promise, - opts?: { maxConcurrency?: number } + maxConcurrency: number ): Promise { - const { maxConcurrency = 10 } = opts || {} - let next = 0 - let inProgress: PromiseWithId[] = [] - while (next < items.length) { - if (inProgress.length === maxConcurrency) { - const finished = await Promise.race(inProgress.map(t => t.promise)) - inProgress = inProgress.filter(task => task.id !== finished) - } + const promises: Promise[] = [] + let index = 0 - const promise = async (next: number) => { - await task(items[next]) - return next + const processItem = async (item: T) => { + try { + await task(item) + } finally { + processNext() } - - inProgress.push({ promise: promise(next), id: next }) - next++ } - await Promise.all(inProgress.map(t => t.promise)) + + const processNext = () => { + if (index >= items.length) { + // No more items to process + return + } + + const item = items[index] + index++ + + const promise = processItem(item) + promises.push(promise) + + if (promises.length >= maxConcurrency) { + Promise.race(promises).then(processNext) + } else { + processNext() + } + } + processNext() + + await Promise.all(promises) } export function filterValueToLabel() { From d0a0e74f393903143896d4c7ffda596bfb717705 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 16:51:26 +0100 Subject: [PATCH 8/9] Remove unused type. --- packages/server/src/integrations/googlesheets.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d5667c276b..8710932ef6 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -25,11 +25,7 @@ import { checkExternalTables, finaliseExternalTables, } from "./utils" -import { - GoogleSpreadsheet, - GoogleSpreadsheetRow, - GoogleSpreadsheetWorksheet, -} from "google-spreadsheet" +import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" import { dataFilters, utils } from "@budibase/shared-core" From bd618f2b006055ec70dcfce8e3233f335085bd8f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 16:53:00 +0100 Subject: [PATCH 9/9] Remove unneeded error message check. --- packages/server/src/integrations/googlesheets.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 8710932ef6..6012ff7789 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -349,7 +349,6 @@ export class GoogleSheetsIntegration implements DatasourcePlus { if ( err.message.startsWith("No values in the header row") || - err.message.startsWith("Header values are not yet loaded") || err.message.startsWith("All your header cells are blank") ) { errors[