diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js
index 49250d2628..6f9bd29fd8 100644
--- a/packages/builder/src/builderStore/datasource.js
+++ b/packages/builder/src/builderStore/datasource.js
@@ -47,5 +47,6 @@ export async function validateDatasourceConfig(config) {
export async function getDatasourceInfo(config) {
const datasource = prepareData(config)
- return await API.fetchInfoForDatasource(datasource)
+ const resp = await API.fetchInfoForDatasource(datasource)
+ return resp
}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
index 7b4808967d..93f8953cb6 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
@@ -5,12 +5,16 @@
Layout,
Link,
notifications,
+ FancyCheckboxGroup,
} from "@budibase/bbui"
import { IntegrationNames, IntegrationTypes } from "constants/backend"
import GoogleButton from "../_components/GoogleButton.svelte"
import { organisation } from "stores/portal"
import { onMount } from "svelte"
- import { validateDatasourceConfig } from "builderStore/datasource"
+ import {
+ validateDatasourceConfig,
+ getDatasourceInfo,
+ } from "builderStore/datasource"
import cloneDeep from "lodash/cloneDeepWith"
import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
import { goto } from "@roxi/routify"
@@ -32,8 +36,9 @@
const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
export const GoogleDatasouceConfigStep = {
- AUTH: "Auth",
- SET_URL: "Set_url",
+ AUTH: "auth",
+ SET_URL: "set_url",
+ SET_SHEETS: "set_sheets",
}
let step = continueSetupId
@@ -42,12 +47,34 @@
let isValid = false
+ let allSheets
+ let selectedSheets
+
+ const saveDatasourceAndRedirect = async () => {
+ try {
+ const resp = await saveDatasource(datasource, {
+ tablesFilter: selectedSheets,
+ })
+ $goto(`./datasource/${resp._id}`)
+ notifications.success(`Datasource created successfully.`)
+ } catch (err) {
+ notifications.error(err?.message ?? "Error saving datasource")
+ // prevent the modal from closing
+ return false
+ }
+ }
+
const modalConfig = {
- [GoogleDatasouceConfigStep.AUTH]: {},
+ [GoogleDatasouceConfigStep.AUTH]: {
+ title: `Connect to ${integrationName}`,
+ },
[GoogleDatasouceConfigStep.SET_URL]: {
+ title: `Connect your spreadsheet`,
confirmButtonText: "Connect",
onConfirm: async () => {
- if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
+ const checkConnection =
+ integration.features[DatasourceFeature.CONNECTION_CHECKING]
+ if (checkConnection) {
const resp = await validateDatasourceConfig(datasource)
if (!resp.connected) {
notifications.error(`Unable to connect - ${resp.error}`)
@@ -55,22 +82,37 @@
}
}
- try {
- const resp = await saveDatasource(datasource)
- $goto(`./datasource/${resp._id}`)
- notifications.success(`Datasource created successfully.`)
- } catch (err) {
- notifications.error(err?.message ?? "Error saving datasource")
- // prevent the modal from closing
- return false
+ if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) {
+ saveDatasourceAndRedirect()
+ return
}
+
+ const info = await getDatasourceInfo(datasource)
+ allSheets = info.tableNames
+
+ step = GoogleDatasouceConfigStep.SET_SHEETS
+ notifications.success(
+ checkConnection
+ ? "Connection Successful"
+ : `Datasource created successfully.`
+ )
+
+ // prevent the modal from closing
+ return false
+ },
+ },
+ [GoogleDatasouceConfigStep.SET_SHEETS]: {
+ title: `Choose your sheets`,
+ confirmButtonText: "Fetch sheets",
+ onConfirm: async () => {
+ await saveDatasourceAndRedirect()
},
},
}
{/if}
+ {#if step === GoogleDatasouceConfigStep.SET_SHEETS}
+
+ Select which spreadsheets you want to connect.
+
+
+
+ {/if}
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index a792f49b57..91bc310a44 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
import fetch from "node-fetch"
import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, utils } from "@budibase/shared-core"
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
import sdk from "../sdk"
@@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
async testConnection(): Promise {
try {
- await setupCreationAuth(this.config)
await this.connect()
return { connected: true }
} catch (e: any) {
@@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
async connect() {
try {
+ await setupCreationAuth(this.config)
+
// Initialise oAuth client
let googleConfig = await configs.getGoogleDatasourceConfig()
if (!googleConfig) {
@@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
async buildSchema(datasourceId: string, entities: Record) {
- // not fully configured yet
- if (!this.config.auth) {
- return
- }
await this.connect()
const sheets = this.client.sheetsByIndex
const tables: Record = {}
- for (let sheet of sheets) {
- // must fetch rows to determine schema
- await sheet.getRows()
+ await utils.parallelForeach(
+ sheets,
+ async sheet => {
+ // must fetch rows to determine schema
+ await sheet.getRows({ limit: 0, offset: 0 })
- const id = buildExternalTableId(datasourceId, sheet.title)
- tables[sheet.title] = this.getTableSchema(
- sheet.title,
- sheet.headerValues,
- id
- )
- }
+ const id = buildExternalTableId(datasourceId, sheet.title)
+ tables[sheet.title] = this.getTableSchema(
+ sheet.title,
+ sheet.headerValues,
+ id
+ )
+ },
+ 10
+ )
const final = finaliseExternalTables(tables, entities)
this.tables = final.tables
this.schemaErrors = final.errors
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index 720027d6a7..46a6585f89 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -4,3 +4,42 @@ export function unreachable(
) {
throw new Error(message)
}
+
+export async function parallelForeach(
+ items: T[],
+ task: (item: T) => Promise,
+ maxConcurrency: number
+): Promise {
+ const promises: Promise[] = []
+ let index = 0
+
+ const processItem = async (item: T) => {
+ try {
+ await task(item)
+ } finally {
+ processNext()
+ }
+ }
+
+ const processNext = () => {
+ if (index >= items.length) {
+ // No more items to process
+ return
+ }
+
+ const item = items[index]
+ index++
+
+ const promise = processItem(item)
+ promises.push(promise)
+
+ if (promises.length >= maxConcurrency) {
+ Promise.race(promises).then(processNext)
+ } else {
+ processNext()
+ }
+ }
+ processNext()
+
+ await Promise.all(promises)
+}