From 945070488bbc133f0f9156cd51228c8fcf046911 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 13:00:33 +0200 Subject: [PATCH 01/14] Validate url --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 01fbc86700..86c32b80e0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 01fbc8670021c5a275c2a1a36ee18b984eeafad5 +Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21 From c47656d423c81715175b824ecde95c11379407d6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 14:29:45 +0200 Subject: [PATCH 02/14] Save datasource --- packages/server/src/api/controllers/datasource.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index bbbcf96538..5f96536799 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -315,6 +315,12 @@ const preSaveAction: Partial> = { }, } +const preSaveAction: Partial> = { + [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { + await googleSetupCreationAuth(datasource.config as any) + }, +} + export async function save( ctx: UserCtx ) { From faf886f446811664603b6f425e2604dfd7fd6169 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 15:51:42 +0200 Subject: [PATCH 03/14] Change success message --- .../modals/GoogleDatasourceConfigModal.svelte | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 7b4808967d..40b625380d 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -47,7 +47,9 @@ [GoogleDatasouceConfigStep.SET_URL]: { confirmButtonText: "Connect", onConfirm: async () => { - if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) { + const checkConnection = + integration.features[DatasourceFeature.CONNECTION_CHECKING] + if (checkConnection) { const resp = await validateDatasourceConfig(datasource) if (!resp.connected) { notifications.error(`Unable to connect - ${resp.error}`) @@ -58,7 +60,11 @@ try { const resp = await saveDatasource(datasource) $goto(`./datasource/${resp._id}`) - notifications.success(`Datasource created successfully.`) + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) } catch (err) { notifications.error(err?.message ?? "Error saving datasource") // prevent the modal from closing From e9d4dd51bbc7a415ee973705c5cdc01bafa48f51 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 16:00:55 +0200 Subject: [PATCH 04/14] Add sheets step --- .../modals/GoogleDatasourceConfigModal.svelte | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 40b625380d..09b17536f3 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -32,8 +32,9 @@ const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS] export const GoogleDatasouceConfigStep = { - AUTH: "Auth", - SET_URL: "Set_url", + AUTH: "auth", + SET_URL: "set_url", + SET_SHEETS: "set_sheets", } let step = continueSetupId @@ -43,8 +44,11 @@ let isValid = false const modalConfig = { - [GoogleDatasouceConfigStep.AUTH]: {}, + [GoogleDatasouceConfigStep.AUTH]: { + title: `Connect to ${integrationName}`, + }, [GoogleDatasouceConfigStep.SET_URL]: { + title: `Connect your spreadsheet`, confirmButtonText: "Connect", onConfirm: async () => { const checkConnection = @@ -57,14 +61,25 @@ } } + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) + + // prevent the modal from closing + return false + }, + }, + [GoogleDatasouceConfigStep.SET_SHEETS]: { + title: `Choose your sheets`, + confirmButtonText: "Fetch sheets", + onConfirm: async () => { try { const resp = await saveDatasource(datasource) $goto(`./datasource/${resp._id}`) - notifications.success( - checkConnection - ? "Connection Successful" - : `Datasource created successfully.` - ) + notifications.success(`Datasource created successfully.`) } catch (err) { notifications.error(err?.message ?? "Error saving datasource") // prevent the modal from closing @@ -76,7 +91,7 @@ {/if} + {#if step === GoogleDatasouceConfigStep.SET_SHEETS} + + Select which spreadsheet you want to connect. + + {/if} From 95c66edb657a08556cfbaa96e73eb8f6af6c0c44 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 11:48:11 +0100 Subject: [PATCH 05/14] Clean code --- packages/server/src/api/controllers/datasource.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 5f96536799..d765d96dcd 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -309,11 +309,6 @@ export async function update(ctx: UserCtx) { builderSocket?.emitDatasourceUpdate(ctx, datasource) } -const preSaveAction: Partial> = { - [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { - await googleSetupCreationAuth(datasource.config as any) - }, -} const preSaveAction: Partial> = { [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { From 258b3daf20411ca78f0d3a831df37209d2c8fb08 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 11:48:53 +0100 Subject: [PATCH 06/14] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 86c32b80e0..01fbc86700 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21 +Subproject commit 01fbc8670021c5a275c2a1a36ee18b984eeafad5 From 77bd2365573d18541c060657cee382337d1102f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 12:13:56 +0100 Subject: [PATCH 07/14] Fetch info when enabled --- .../builder/src/builderStore/datasource.js | 6 +++ .../modals/GoogleDatasourceConfigModal.svelte | 52 ++++++++++++------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 49250d2628..427e03edbb 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -49,3 +49,9 @@ export async function getDatasourceInfo(config) { const datasource = prepareData(config) return await API.fetchInfoForDatasource(datasource) } + +export async function getDatasourceInfo(config) { + const datasource = prepareData(config) + const resp = await API.fetchTablesForDatasource(datasource) + return resp +} diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 09b17536f3..c74616fb3f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -10,10 +10,14 @@ import GoogleButton from "../_components/GoogleButton.svelte" import { organisation } from "stores/portal" import { onMount } from "svelte" - import { validateDatasourceConfig } from "builderStore/datasource" + import { + validateDatasourceConfig, + getDatasourceInfo, + } from "builderStore/datasource" import cloneDeep from "lodash/cloneDeepWith" import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte" import { goto } from "@roxi/routify" + import { API } from "api" import { saveDatasource } from "builderStore/datasource" import { DatasourceFeature } from "@budibase/types" @@ -43,6 +47,18 @@ let isValid = false + const saveDatasourceAndRedirect = async () => { + try { + const resp = await saveDatasource(datasource) + $goto(`./datasource/${resp._id}`) + notifications.success(`Datasource created successfully.`) + } catch (err) { + notifications.error(err?.message ?? "Error saving datasource") + // prevent the modal from closing + return false + } + } + const modalConfig = { [GoogleDatasouceConfigStep.AUTH]: { title: `Connect to ${integrationName}`, @@ -61,12 +77,22 @@ } } - step = GoogleDatasouceConfigStep.SET_SHEETS - notifications.success( - checkConnection - ? "Connection Successful" - : `Datasource created successfully.` - ) + if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) { + saveDatasourceAndRedirect() + return + } + + getDatasourceInfo(datasource).then(i => { + debugger + console.log(i) + + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) + }) // prevent the modal from closing return false @@ -75,17 +101,7 @@ [GoogleDatasouceConfigStep.SET_SHEETS]: { title: `Choose your sheets`, confirmButtonText: "Fetch sheets", - onConfirm: async () => { - try { - const resp = await saveDatasource(datasource) - $goto(`./datasource/${resp._id}`) - notifications.success(`Datasource created successfully.`) - } catch (err) { - notifications.error(err?.message ?? "Error saving datasource") - // prevent the modal from closing - return false - } - }, + onConfirm: saveDatasourceAndRedirect, }, } From 5135ac36b7660dd5d2627de7490395bbb5a41c96 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 12:27:49 +0100 Subject: [PATCH 08/14] Fix fetching tables --- packages/server/src/integrations/googlesheets.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a792f49b57..a0c10d3655 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { async testConnection(): Promise { try { - await setupCreationAuth(this.config) await this.connect() return { connected: true } } catch (e: any) { @@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus { async connect() { try { + await setupCreationAuth(this.config) + // Initialise oAuth client let googleConfig = await configs.getGoogleDatasourceConfig() if (!googleConfig) { From 39848eef07b261e066acdb68a0fac020c725d024 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 13:52:44 +0100 Subject: [PATCH 09/14] Fetch sheets --- .../modals/GoogleDatasourceConfigModal.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index c74616fb3f..d01ae3de05 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -47,6 +47,8 @@ let isValid = false + let sheets + const saveDatasourceAndRedirect = async () => { try { const resp = await saveDatasource(datasource) @@ -82,9 +84,8 @@ return } - getDatasourceInfo(datasource).then(i => { - debugger - console.log(i) + const info = await getDatasourceInfo(datasource) + sheets = info.tableNames step = GoogleDatasouceConfigStep.SET_SHEETS notifications.success( @@ -92,7 +93,6 @@ ? "Connection Successful" : `Datasource created successfully.` ) - }) // prevent the modal from closing return false @@ -146,7 +146,7 @@ {/if} {#if step === GoogleDatasouceConfigStep.SET_SHEETS} - Select which spreadsheet you want to connect. + Select which spreadsheets you want to connect. {/if} From b128bdece5f24fb28f6e744a8f8850719210f18e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 13:55:30 +0100 Subject: [PATCH 10/14] Display sheets --- .../modals/GoogleDatasourceConfigModal.svelte | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index d01ae3de05..a98b76483d 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -47,7 +47,7 @@ let isValid = false - let sheets + let allSheets const saveDatasourceAndRedirect = async () => { try { @@ -85,14 +85,14 @@ } const info = await getDatasourceInfo(datasource) - sheets = info.tableNames + allSheets = info.tableNames - step = GoogleDatasouceConfigStep.SET_SHEETS - notifications.success( - checkConnection - ? "Connection Successful" - : `Datasource created successfully.` - ) + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) // prevent the modal from closing return false @@ -101,7 +101,9 @@ [GoogleDatasouceConfigStep.SET_SHEETS]: { title: `Choose your sheets`, confirmButtonText: "Fetch sheets", - onConfirm: saveDatasourceAndRedirect, + onConfirm: async () => { + await saveDatasourceAndRedirect() + }, }, } @@ -147,6 +149,10 @@ {#if step === GoogleDatasouceConfigStep.SET_SHEETS} Select which spreadsheets you want to connect. + + {#each allSheets as sheet} + {sheet} + {/each} {/if} From f10d3d69733b136069744ea29351ccc2f841195f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 13:56:45 +0100 Subject: [PATCH 11/14] Update --- packages/builder/src/builderStore/datasource.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 427e03edbb..2ed74f2e09 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -52,6 +52,6 @@ export async function getDatasourceInfo(config) { export async function getDatasourceInfo(config) { const datasource = prepareData(config) - const resp = await API.fetchTablesForDatasource(datasource) + const resp = await API.fetchInfoForDatasource(datasource) return resp } From bb901d94d2a14bcee35573bc46db03a9c2ec5d63 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 11:32:42 +0100 Subject: [PATCH 12/14] Clean code --- packages/builder/src/builderStore/datasource.js | 5 ----- packages/server/src/api/controllers/datasource.ts | 1 - 2 files changed, 6 deletions(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 2ed74f2e09..6f9bd29fd8 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -45,11 +45,6 @@ export async function validateDatasourceConfig(config) { return await API.validateDatasource(datasource) } -export async function getDatasourceInfo(config) { - const datasource = prepareData(config) - return await API.fetchInfoForDatasource(datasource) -} - export async function getDatasourceInfo(config) { const datasource = prepareData(config) const resp = await API.fetchInfoForDatasource(datasource) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index d765d96dcd..bbbcf96538 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -309,7 +309,6 @@ export async function update(ctx: UserCtx) { builderSocket?.emitDatasourceUpdate(ctx, datasource) } - const preSaveAction: Partial> = { [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { await googleSetupCreationAuth(datasource.config as any) From dd58aaf5cb3c1053652b2b46abf718efd7e97500 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 11:44:32 +0100 Subject: [PATCH 13/14] Use fancycheckboxgroup --- .../modals/GoogleDatasourceConfigModal.svelte | 11 ++++++----- packages/server/src/integrations/googlesheets.ts | 4 ---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index a98b76483d..93f8953cb6 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -5,6 +5,7 @@ Layout, Link, notifications, + FancyCheckboxGroup, } from "@budibase/bbui" import { IntegrationNames, IntegrationTypes } from "constants/backend" import GoogleButton from "../_components/GoogleButton.svelte" @@ -17,7 +18,6 @@ import cloneDeep from "lodash/cloneDeepWith" import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte" import { goto } from "@roxi/routify" - import { API } from "api" import { saveDatasource } from "builderStore/datasource" import { DatasourceFeature } from "@budibase/types" @@ -48,10 +48,13 @@ let isValid = false let allSheets + let selectedSheets const saveDatasourceAndRedirect = async () => { try { - const resp = await saveDatasource(datasource) + const resp = await saveDatasource(datasource, { + tablesFilter: selectedSheets, + }) $goto(`./datasource/${resp._id}`) notifications.success(`Datasource created successfully.`) } catch (err) { @@ -150,9 +153,7 @@ Select which spreadsheets you want to connect. - {#each allSheets as sheet} - {sheet} - {/each} + {/if} diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a0c10d3655..3af564d059 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -274,10 +274,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { } async buildSchema(datasourceId: string, entities: Record) { - // not fully configured yet - if (!this.config.auth) { - return - } await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} From a14b29efac17adc5ce4efa7999a3f6294b45fb48 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 12:29:36 +0100 Subject: [PATCH 14/14] Improve fetch performance --- .../server/src/integrations/googlesheets.ts | 26 +++++++------ packages/shared-core/src/utils.ts | 39 +++++++++++++++++++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 3af564d059..91bc310a44 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils" import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, utils } from "@budibase/shared-core" import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" import sdk from "../sdk" @@ -277,17 +277,21 @@ class GoogleSheetsIntegration implements DatasourcePlus { await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} - for (let sheet of sheets) { - // must fetch rows to determine schema - await sheet.getRows() + await utils.parallelForeach( + sheets, + async sheet => { + // must fetch rows to determine schema + await sheet.getRows({ limit: 0, offset: 0 }) - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - id - ) - } + const id = buildExternalTableId(datasourceId, sheet.title) + tables[sheet.title] = this.getTableSchema( + sheet.title, + sheet.headerValues, + id + ) + }, + 10 + ) const final = finaliseExternalTables(tables, entities) this.tables = final.tables this.schemaErrors = final.errors diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 720027d6a7..46a6585f89 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -4,3 +4,42 @@ export function unreachable( ) { throw new Error(message) } + +export async function parallelForeach( + items: T[], + task: (item: T) => Promise, + maxConcurrency: number +): Promise { + const promises: Promise[] = [] + let index = 0 + + const processItem = async (item: T) => { + try { + await task(item) + } finally { + processNext() + } + } + + const processNext = () => { + if (index >= items.length) { + // No more items to process + return + } + + const item = items[index] + index++ + + const promise = processItem(item) + promises.push(promise) + + if (promises.length >= maxConcurrency) { + Promise.race(promises).then(processNext) + } else { + processNext() + } + } + processNext() + + await Promise.all(promises) +}