From 213315e14f63793a484757201252ee9cb8a03942 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jun 2023 18:03:43 +0100 Subject: [PATCH 1/3] Fixing issue with row deletion not returning the _id of the row which was deleted in googlesheets. --- packages/server/src/integrations/googlesheets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index aa1c2fb66f..66702c73c8 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -535,7 +535,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { const row = rows[query.rowIndex] if (row) { await row.delete() - return [{ deleted: query.rowIndex }] + return [{ deleted: query.rowIndex, [GOOGLE_SHEETS_PRIMARY_KEY]: query.rowIndex }] } else { throw new Error("Row does not exist.") } From e7649374d26279af9c387408fad82a2152f3dbcb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jun 2023 18:25:35 +0100 Subject: [PATCH 2/3] Fixing googlesheets filtering by _id - there was an issue with the response API giving _rowNumber rather than the expected primary key of rowNumber, need to update the filters accordingly. --- .../server/src/integrations/googlesheets.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 66702c73c8..59d47a0afa 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -472,6 +472,20 @@ class GoogleSheetsIntegration implements DatasourcePlus { } else { rows = await sheet.getRows() } + // this is a special case - need to handle the _id, it doesn't exist + // we cannot edit the returned structure from google, it does not have + // setter functions and is immutable, easier to update the filters + // to look for the _rowNumber property rather than rowNumber + if (query.filters?.equal) { + const idFilterKeys = Object.keys(query.filters.equal).filter(filter => + filter.includes(GOOGLE_SHEETS_PRIMARY_KEY) + ) + for (let idFilterKey of idFilterKeys) { + const id = query.filters.equal[idFilterKey] + delete query.filters.equal[idFilterKey] + query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id + } + } const filtered = dataFilters.runLuceneQuery(rows, query.filters) const headerValues = sheet.headerValues let response = [] @@ -535,7 +549,12 @@ class GoogleSheetsIntegration implements DatasourcePlus { const row = rows[query.rowIndex] if (row) { await row.delete() - return [{ deleted: query.rowIndex, [GOOGLE_SHEETS_PRIMARY_KEY]: query.rowIndex }] + return [ + { + deleted: query.rowIndex, + [GOOGLE_SHEETS_PRIMARY_KEY]: query.rowIndex, + }, + ] } else { throw new Error("Row does not exist.") } From 0506096e721e6f68542b0e4589a7473bb3819cb1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jun 2023 10:03:47 +0100 Subject: [PATCH 3/3] Fixing filtering plus pagination in googlesheets, the pagination previously was first which broke the filtering. --- .../server/src/integrations/googlesheets.ts | 21 ++++++++++++------- packages/shared-core/src/filters.ts | 16 ++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 59d47a0afa..8863aa0b3a 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -457,17 +457,19 @@ class GoogleSheetsIntegration implements DatasourcePlus { }) { try { await this.connect() + const hasFilters = dataFilters.hasFilters(query.filters) + const limit = query.paginate?.limit || 100 + const page: number = + typeof query.paginate?.page === "number" + ? query.paginate.page + : parseInt(query.paginate?.page || "1") + const offset = (page - 1) * limit const sheet = this.client.sheetsByTitle[query.sheet] let rows: GoogleSpreadsheetRow[] = [] - if (query.paginate) { - const limit = query.paginate.limit || 100 - let page: number = - typeof query.paginate.page === "number" - ? query.paginate.page - : parseInt(query.paginate.page || "1") + if (query.paginate && !hasFilters) { rows = await sheet.getRows({ limit, - offset: (page - 1) * limit, + offset, }) } else { rows = await sheet.getRows() @@ -486,7 +488,10 @@ class GoogleSheetsIntegration implements DatasourcePlus { query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id } } - const filtered = dataFilters.runLuceneQuery(rows, query.filters) + let filtered = dataFilters.runLuceneQuery(rows, query.filters) + if (hasFilters && query.paginate) { + filtered = filtered.slice(offset, offset + limit) + } const headerValues = sheet.headerValues let response = [] for (let row of filtered) { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 1ffcebdd2b..7c2c34bf6b 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -455,3 +455,19 @@ export const luceneLimit = (docs: any[], limit: string) => { } return docs.slice(0, numLimit) } + +export const hasFilters = (query?: Query) => { + if (!query) { + return false + } + const skipped = ["allOr"] + for (let [key, value] of Object.entries(query)) { + if (skipped.includes(key) || typeof value !== "object") { + continue + } + if (Object.keys(value).length !== 0) { + return true + } + } + return false +}