diff --git a/lerna.json b/lerna.json index eded0d214d..03bbdbca7e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.7-alpha.2", + "version": "2.7.7-alpha.4", "npmClient": "yarn", "packages": [ "packages/backend-core", diff --git a/packages/bbui/src/FancyForm/FancyCheckbox.svelte b/packages/bbui/src/FancyForm/FancyCheckbox.svelte index 191cc79485..0a2e5ac159 100644 --- a/packages/bbui/src/FancyForm/FancyCheckbox.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckbox.svelte @@ -8,6 +8,8 @@ export let disabled = false export let error = null export let validate = null + export let indeterminate = false + export let compact = false const dispatch = createEventDispatcher() @@ -21,11 +23,19 @@ } - + - + -
+
{#if text} {text} {/if} @@ -47,6 +57,10 @@ line-clamp: 2; -webkit-box-orient: vertical; } + .text.compact { + font-size: 13px; + line-height: 15px; + } .text > :global(*) { font-size: inherit !important; } diff --git a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte new file mode 100644 index 0000000000..aaea388c36 --- /dev/null +++ b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte @@ -0,0 +1,68 @@ + + +{#if options && Array.isArray(options)} +
+ + {#if showSelectAll} + + {/if} + {#each options as option, i} + + {/each} + +
+{/if} + + diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 0c99394599..455f4b38fb 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -11,6 +11,7 @@ export let value export let ref export let autoHeight + export let compact = false const formContext = getContext("fancy-form") const id = Math.random() @@ -42,6 +43,7 @@ class:disabled class:focused class:clickable + class:compact class:auto-height={autoHeight} >
@@ -61,7 +63,6 @@ diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 18aa361570..2486942dea 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -44,6 +44,9 @@ export default ICONS export function getIcon(integrationType, schema) { const integrationList = get(integrations) + if (!integrationList) { + return + } if (integrationList[integrationType]?.iconUrl) { return { url: integrationList[integrationType].iconUrl } } else if (schema?.custom || !ICONS[integrationType]) { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 21c7e07a25..fbc38e2daa 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -1,12 +1,19 @@ saveDatasource()} - confirmText={datasource.plus ? "Connect" : "Save and continue to query"} - cancelText="Back" - showSecondaryButton={datasource.plus} + {title} + onConfirm={() => nextStep()} + {confirmText} + cancelText={fetchTableStep ? "Cancel" : "Back"} + showSecondaryButton={datasourcePlus} size="L" disabled={!isValid} > - Connect your database to Budibase using the config below. + + {#if !fetchTableStep} + Connect your database to Budibase using the config below + {:else} + Choose what tables you want to sync with Budibase + {/if} - (isValid = e.detail)} - /> + {#if !fetchTableStep} + (isValid = e.detail)} + /> + {:else} +
+ +
+ {/if} + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 7b4808967d..14f81f915c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -1,22 +1,27 @@ Add the URL of the sheet you want to connect. (isValid = e.detail)} /> {/if} + {#if step === GoogleDatasouceConfigStep.SET_SHEETS} + + Select which spreadsheets you want to connect. + + + + {#if setSheetsErrorTitle || setSheetsErrorMessage} + + {/if} + + {/if} diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index 952cf8a499..3418206611 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -5,6 +5,7 @@ Body, InlineAlert, Input, + notifications, } from "@budibase/bbui" import { createValidationStore } from "helpers/validation/yup" @@ -50,14 +51,49 @@ }, } - const exportApp = password => { + const exportApp = async () => { const id = published ? app.prodId : app.devId - const appName = encodeURIComponent(app.name) - let url = `/api/backups/export?appId=${id}&appname=${appName}&excludeRows=${!includeInternalTablesRows}` - if (password) { - url += `&encryptPassword=${password}` + const url = `/api/backups/export?appId=${id}` + await downloadFile(url, { excludeRows, encryptPassword: password }) + } + + export async function downloadFile(url, body) { + try { + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }) + + if (response.ok) { + const contentDisposition = response.headers.get("Content-Disposition") + + const matches = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/.exec( + contentDisposition + ) + + const filename = matches[1].replace(/['"]/g, "") + + const url = URL.createObjectURL(await response.blob()) + + const link = document.createElement("a") + link.href = url + link.download = filename + link.click() + + URL.revokeObjectURL(url) + } else { + notifications.error("Error exporting the app.") + } + } catch (error) { + let message = "Error downloading the exported app" + if (error.message) { + message += `: ${error.message}` + } + notifications.error("Error downloading the exported app", message) } - window.location = url } diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 0815f9d766..e774aae8c6 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -57,7 +57,10 @@ export function createDatasourcesStore() { return updateDatasource(response) } - const save = async (body, fetchSchema = false) => { + const save = async (body, { fetchSchema, tablesFilter } = {}) => { + if (fetchSchema == null) { + fetchSchema = false + } let response if (body._id) { response = await API.updateDatasource(body) @@ -65,6 +68,7 @@ export function createDatasourcesStore() { response = await API.createDatasource({ datasource: body, fetchSchema, + tablesFilter, }) } return updateDatasource(response) diff --git a/packages/frontend-core/src/api/datasources.js b/packages/frontend-core/src/api/datasources.js index 16d19c512f..1ba17cc610 100644 --- a/packages/frontend-core/src/api/datasources.js +++ b/packages/frontend-core/src/api/datasources.js @@ -26,13 +26,16 @@ export const buildDatasourceEndpoints = API => ({ * Creates a datasource * @param datasource the datasource to create * @param fetchSchema whether to fetch the schema or not + * @param tablesFilter a list of tables to actually fetch rather than simply + * all that are accessible. */ - createDatasource: async ({ datasource, fetchSchema }) => { + createDatasource: async ({ datasource, fetchSchema, tablesFilter }) => { return await API.post({ url: "/api/datasources", body: { datasource, fetchSchema, + tablesFilter, }, }) }, @@ -69,4 +72,15 @@ export const buildDatasourceEndpoints = API => ({ body: { datasource }, }) }, + + /** + * Fetch table names available within the datasource, for filtering out undesired tables + * @param datasource the datasource configuration to use for fetching tables + */ + fetchInfoForDatasource: async datasource => { + return await API.post({ + url: `/api/datasources/info`, + body: { datasource }, + }) + }, }) diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index e3b98f0058..2a7921d354 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -1,17 +1,25 @@ import sdk from "../../sdk" -import { events, context } from "@budibase/backend-core" +import { events, context, db } from "@budibase/backend-core" import { DocumentType } from "../../db/utils" -import { isQsTrue } from "../../utilities" +import { Ctx } from "@budibase/types" + +interface ExportAppDumpRequest { + excludeRows: boolean + encryptPassword?: string +} + +export async function exportAppDump(ctx: Ctx) { + const { appId } = ctx.query as any + const { excludeRows, encryptPassword } = ctx.request.body + + const [app] = await db.getAppsByIDs([appId]) + const appName = app.name -export async function exportAppDump(ctx: any) { - let { appId, excludeRows = false, encryptPassword } = ctx.query // remove the 120 second limit for the request ctx.req.setTimeout(0) - const appName = decodeURI(ctx.query.appname) - excludeRows = isQsTrue(excludeRows) - const backupIdentifier = `${appName}-export-${new Date().getTime()}${ - encryptPassword ? ".enc" : "" - }.tar.gz` + + const extension = encryptPassword ? "enc.tar.gz" : "tar.gz" + const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}` ctx.attachment(backupIdentifier) ctx.body = await sdk.backups.streamExportApp({ appId, diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 75b33083fb..bbbcf96538 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -103,6 +103,22 @@ async function buildSchemaHelper(datasource: Datasource) { return { tables: connector.tables, error } } +async function buildFilteredSchema(datasource: Datasource, filter?: string[]) { + let { tables, error } = await buildSchemaHelper(datasource) + let finalTables = tables + if (filter) { + finalTables = {} + for (let key in tables) { + if ( + filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase()) + ) { + finalTables[key] = tables[key] + } + } + } + return { tables: finalTables, error } +} + export async function fetch(ctx: UserCtx) { // Get internal tables const db = context.getAppDB() @@ -174,43 +190,28 @@ export async function information( } const tableNames = await connector.getTableNames() ctx.body = { - tableNames, + tableNames: tableNames.sort(), } } export async function buildSchemaFromDb(ctx: UserCtx) { const db = context.getAppDB() - const datasource = await sdk.datasources.get(ctx.params.datasourceId) const tablesFilter = ctx.request.body.tablesFilter + const datasource = await sdk.datasources.get(ctx.params.datasourceId) - let { tables, error } = await buildSchemaHelper(datasource) - if (tablesFilter) { - if (!datasource.entities) { - datasource.entities = {} - } - for (let key in tables) { - if ( - tablesFilter.some( - (filter: any) => filter.toLowerCase() === key.toLowerCase() - ) - ) { - datasource.entities[key] = tables[key] - } - } - } else { - datasource.entities = tables - } + const { tables, error } = await buildFilteredSchema(datasource, tablesFilter) + datasource.entities = tables setDefaultDisplayColumns(datasource) const dbResp = await db.put(datasource) datasource._rev = dbResp.rev const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource) - const response: any = { datasource: cleanedDatasource } + const res: any = { datasource: cleanedDatasource } if (error) { - response.error = error + res.error = error } - ctx.body = response + ctx.body = res } /** @@ -320,6 +321,7 @@ export async function save( const db = context.getAppDB() const plus = ctx.request.body.datasource.plus const fetchSchema = ctx.request.body.fetchSchema + const tablesFilter = ctx.request.body.tablesFilter const datasource = { _id: generateDatasourceID({ plus }), @@ -329,7 +331,10 @@ export async function save( let schemaError = null if (fetchSchema) { - const { tables, error } = await buildSchemaHelper(datasource) + const { tables, error } = await buildFilteredSchema( + datasource, + tablesFilter + ) schemaError = error datasource.entities = tables setDefaultDisplayColumns(datasource) diff --git a/packages/server/src/api/routes/backup.ts b/packages/server/src/api/routes/backup.ts index fb455250a7..94e4cf957f 100644 --- a/packages/server/src/api/routes/backup.ts +++ b/packages/server/src/api/routes/backup.ts @@ -5,7 +5,7 @@ import { permissions } from "@budibase/backend-core" const router: Router = new Router() -router.get( +router.post( "/api/backups/export", authorized(permissions.BUILDER), controller.exportAppDump diff --git a/packages/server/src/events/docUpdates/syncUsers.ts b/packages/server/src/events/docUpdates/syncUsers.ts index 7957178168..5777445249 100644 --- a/packages/server/src/events/docUpdates/syncUsers.ts +++ b/packages/server/src/events/docUpdates/syncUsers.ts @@ -26,6 +26,10 @@ export default function process(updateCb?: UpdateCallback) { // if something not found - no changes to perform if (err?.status === 404) { return + } + // The user has already been sync in another process + else if (err?.status === 409) { + return } else { logging.logAlert("Failed to perform user/group app sync", err) } diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a792f49b57..91bc310a44 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils" import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, utils } from "@budibase/shared-core" import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" import sdk from "../sdk" @@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { async testConnection(): Promise { try { - await setupCreationAuth(this.config) await this.connect() return { connected: true } } catch (e: any) { @@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus { async connect() { try { + await setupCreationAuth(this.config) + // Initialise oAuth client let googleConfig = await configs.getGoogleDatasourceConfig() if (!googleConfig) { @@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus { } async buildSchema(datasourceId: string, entities: Record) { - // not fully configured yet - if (!this.config.auth) { - return - } await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} - for (let sheet of sheets) { - // must fetch rows to determine schema - await sheet.getRows() + await utils.parallelForeach( + sheets, + async sheet => { + // must fetch rows to determine schema + await sheet.getRows({ limit: 0, offset: 0 }) - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - id - ) - } + const id = buildExternalTableId(datasourceId, sheet.title) + tables[sheet.title] = this.getTableSchema( + sheet.title, + sheet.headerValues, + id + ) + }, + 10 + ) const final = finaliseExternalTables(tables, entities) this.tables = final.tables this.schemaErrors = final.errors diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 47295716e6..33ecf45982 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -322,7 +322,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus { await this.openConnection() const columnsResponse: { rows: PostgresColumn[] } = await this.client.query(this.COLUMNS_SQL) - return columnsResponse.rows.map(row => row.table_name) + const names = columnsResponse.rows.map(row => row.table_name) + return [...new Set(names)] } finally { await this.closeConnection() } diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 0f629630e2..3be8c64159 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -187,7 +187,7 @@ export async function streamExportApp({ }: { appId: string excludeRows: boolean - encryptPassword: string + encryptPassword?: string }) { const tmpPath = await exportApp(appId, { excludeRows, diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index c886e6a15f..028f8e7e93 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -164,5 +164,6 @@ export function mergeConfigs(update: Datasource, old: Datasource) { delete update.config[key] } } + return update } diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 720027d6a7..46a6585f89 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -4,3 +4,42 @@ export function unreachable( ) { throw new Error(message) } + +export async function parallelForeach( + items: T[], + task: (item: T) => Promise, + maxConcurrency: number +): Promise { + const promises: Promise[] = [] + let index = 0 + + const processItem = async (item: T) => { + try { + await task(item) + } finally { + processNext() + } + } + + const processNext = () => { + if (index >= items.length) { + // No more items to process + return + } + + const item = items[index] + index++ + + const promise = processItem(item) + promises.push(promise) + + if (promises.length >= maxConcurrency) { + Promise.race(promises).then(processNext) + } else { + processNext() + } + } + processNext() + + await Promise.all(promises) +} diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index d692f17421..d0688a24d3 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -12,6 +12,7 @@ export interface UpdateDatasourceResponse { export interface CreateDatasourceRequest { datasource: Datasource fetchSchema?: boolean + tablesFilter: string[] } export interface VerifyDatasourceRequest { diff --git a/qa-core/scripts/testResultsWebhook.js b/qa-core/scripts/testResultsWebhook.js index b4725cae56..40cc42082d 100644 --- a/qa-core/scripts/testResultsWebhook.js +++ b/qa-core/scripts/testResultsWebhook.js @@ -15,6 +15,12 @@ async function generateReport() { return JSON.parse(report) } +const env = process.argv.slice(2)[0] + +if (!env) { + throw new Error("environment argument is required") +} + async function discordResultsNotification(report) { const { numTotalTestSuites, @@ -39,8 +45,8 @@ async function discordResultsNotification(report) { content: `**Nightly Tests Status**: ${OUTCOME}`, embeds: [ { - title: "Budi QA Bot", - description: `Nightly Tests`, + title: `Budi QA Bot - ${env}`, + description: `API Integration Tests`, url: GITHUB_ACTIONS_RUN_URL, color: OUTCOME === "success" ? 3066993 : 15548997, timestamp: new Date(), diff --git a/qa-core/src/account-api/api/apis/AccountAPI.ts b/qa-core/src/account-api/api/apis/AccountAPI.ts index c18dde3d4a..fc6f6caecb 100644 --- a/qa-core/src/account-api/api/apis/AccountAPI.ts +++ b/qa-core/src/account-api/api/apis/AccountAPI.ts @@ -60,8 +60,16 @@ export default class AccountAPI { } async delete(accountID: string) { - const [response, json] = await this.client.del(`/api/accounts/${accountID}`) - expect(response).toHaveStatusCode(200) + const [response, json] = await this.client.del( + `/api/accounts/${accountID}`, + { + internal: true, + } + ) + // can't use expect here due to use in global teardown + if (response.status !== 204) { + throw new Error(`Could not delete accountId=${accountID}`) + } return response } } diff --git a/qa-core/src/jest/globalTeardown.ts b/qa-core/src/jest/globalTeardown.ts index 51f6046f4f..52696a72f8 100644 --- a/qa-core/src/jest/globalTeardown.ts +++ b/qa-core/src/jest/globalTeardown.ts @@ -10,6 +10,7 @@ const API_OPTS: APIRequestOpts = { doExpect: false } async function deleteAccount() { // @ts-ignore const accountID = global.qa.accountId + // can't run 'expect' blocks in teardown await accountsApi.accounts.delete(accountID) }