Merge branch 'budi-7010/frontend-encrypt-app-exports' into budi-7010/frontend-encrypt-app-imports

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.7.7-alpha.2",
+  "version": "2.7.7-alpha.4",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

packages/bbui/src/FancyForm/FancyCheckbox.svelte

@@ -8,6 +8,8 @@
   export let disabled = false
   export let error = null
   export let validate = null
+  export let indeterminate = false
+  export let compact = false
 
   const dispatch = createEventDispatcher()
 
@@ -21,11 +23,19 @@
   }
 </script>
 
-<FancyField {error} {value} {validate} {disabled} clickable on:click={onChange}>
+<FancyField
+  {error}
+  {value}
+  {validate}
+  {disabled}
+  {compact}
+  clickable
+  on:click={onChange}
+>
   <span>
-    <Checkbox {disabled} {value} />
+    <Checkbox {disabled} {value} {indeterminate} />
   </span>
-  <div class="text">
+  <div class="text" class:compact>
     {#if text}
       {text}
     {/if}
@@ -47,6 +57,10 @@
     line-clamp: 2;
     -webkit-box-orient: vertical;
   }
+  .text.compact {
+    font-size: 13px;
+    line-height: 15px;
+  }
   .text > :global(*) {
     font-size: inherit !important;
   }

packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte

@@ -0,0 +1,68 @@
+<script>
+  import FancyCheckbox from "./FancyCheckbox.svelte"
+  import FancyForm from "./FancyForm.svelte"
+  import { createEventDispatcher } from "svelte"
+
+  export let options = []
+  export let selected = []
+  export let showSelectAll = true
+  export let selectAllText = "Select all"
+
+  let selectedBooleans = reset()
+  const dispatch = createEventDispatcher()
+
+  $: updateSelected(selectedBooleans)
+  $: dispatch("change", selected)
+  $: allSelected = selected?.length === options.length
+  $: noneSelected = !selected?.length
+
+  function reset() {
+    return Array(options.length).fill(true)
+  }
+
+  function updateSelected(selectedArr) {
+    const array = []
+    for (let [i, isSelected] of Object.entries(selectedArr)) {
+      if (isSelected) {
+        array.push(options[i])
+      }
+    }
+    selected = array
+  }
+
+  function toggleSelectAll() {
+    if (allSelected === true) {
+      selectedBooleans = []
+    } else {
+      selectedBooleans = reset()
+    }
+  }
+</script>
+
+{#if options && Array.isArray(options)}
+  <div class="checkbox-group" class:has-select-all={showSelectAll}>
+    <FancyForm on:change>
+      {#if showSelectAll}
+        <FancyCheckbox
+          bind:value={allSelected}
+          on:change={toggleSelectAll}
+          text={selectAllText}
+          indeterminate={!allSelected && !noneSelected}
+          compact
+        />
+      {/if}
+      {#each options as option, i}
+        <FancyCheckbox bind:value={selectedBooleans[i]} text={option} compact />
+      {/each}
+    </FancyForm>
+  </div>
+{/if}
+
+<style>
+  .checkbox-group.has-select-all :global(.fancy-field:first-of-type) {
+    background: var(--spectrum-global-color-gray-100);
+  }
+  .checkbox-group.has-select-all :global(.fancy-field:first-of-type:hover) {
+    background: var(--spectrum-global-color-gray-200);
+  }
+</style>

packages/bbui/src/FancyForm/FancyField.svelte

@@ -11,6 +11,7 @@
   export let value
   export let ref
   export let autoHeight
+  export let compact = false
 
   const formContext = getContext("fancy-form")
   const id = Math.random()
@@ -42,6 +43,7 @@
   class:disabled
   class:focused
   class:clickable
+  class:compact
   class:auto-height={autoHeight}
 >
   <div class="content" on:click>
@@ -61,7 +63,6 @@
 
 <style>
   .fancy-field {
-    max-width: 400px;
     background: var(--spectrum-global-color-gray-75);
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px;
@@ -69,6 +70,12 @@
     transition: border-color 130ms ease-out, background 130ms ease-out,
       background 130ms ease-out;
     color: var(--spectrum-global-color-gray-800);
+    --padding: 16px;
+    --height: 64px;
+  }
+  .fancy-field.compact {
+    --padding: 8px;
+    --height: 36px;
   }
   .fancy-field:hover {
     border-color: var(--spectrum-global-color-gray-400);
@@ -91,8 +98,8 @@
   }
   .content {
     position: relative;
-    height: 64px;
-    padding: 0 16px;
+    height: var(--height);
+    padding: 0 var(--padding);
   }
   .fancy-field.auto-height .content {
     height: auto;
@@ -103,7 +110,7 @@
     flex-direction: row;
     justify-content: flex-start;
     align-items: center;
-    gap: 16px;
+    gap: var(--padding);
   }
   .field {
     flex: 1 1 auto;

packages/bbui/src/FancyForm/index.js

@@ -4,4 +4,5 @@ export { default as FancySelect } from "./FancySelect.svelte"
 export { default as FancyButton } from "./FancyButton.svelte"
 export { default as FancyForm } from "./FancyForm.svelte"
 export { default as FancyButtonRadio } from "./FancyButtonRadio.svelte"
+export { default as FancyCheckboxGroup } from "./FancyCheckboxGroup.svelte"
 export { default as ErrorMessage } from "./ErrorMessage.svelte"

packages/bbui/src/Form/Core/Checkbox.svelte

@@ -9,6 +9,7 @@
   export let text = null
   export let disabled = false
   export let size
+  export let indeterminate = false
 
   const dispatch = createEventDispatcher()
   const onChange = event => {
@@ -22,6 +23,7 @@
   class="spectrum-Checkbox spectrum-Checkbox--emphasized {sizeClass}"
   class:is-invalid={!!error}
   class:checked={value}
+  class:is-indeterminate={indeterminate}
 >
   <input
     checked={value}

packages/builder/src/builderStore/datasource.js

@@ -23,10 +23,11 @@ function prepareData(config) {
   return datasource
 }
 
-export async function saveDatasource(config, skipFetch = false) {
+export async function saveDatasource(config, { skipFetch, tablesFilter } = {}) {
   const datasource = prepareData(config)
   // Create datasource
-  const resp = await datasources.save(datasource, !skipFetch && datasource.plus)
+  const fetchSchema = !skipFetch && datasource.plus
+  const resp = await datasources.save(datasource, { fetchSchema, tablesFilter })
 
   // update the tables incase datasource plus
   await tables.fetch()
@@ -41,6 +42,13 @@ export async function createRestDatasource(integration) {
 
 export async function validateDatasourceConfig(config) {
   const datasource = prepareData(config)
-  const resp = await API.validateDatasource(datasource)
-  return resp
+  return await API.validateDatasource(datasource)
+}
+
+export async function getDatasourceInfo(config) {
+  let datasource = config
+  if (!config._id) {
+    datasource = prepareData(config)
+  }
+  return await API.fetchInfoForDatasource(datasource)
 }

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -8,7 +8,7 @@
     notifications,
     Modal,
     Table,
-    Toggle,
+    FancyCheckboxGroup,
   } from "@budibase/bbui"
   import { datasources, integrations, tables } from "stores/backend"
   import CreateEditRelationship from "components/backend/Datasources/CreateEditRelationship.svelte"
@@ -16,7 +16,7 @@
   import ArrayRenderer from "components/common/renderers/ArrayRenderer.svelte"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { goto } from "@roxi/routify"
-  import ValuesList from "components/common/ValuesList.svelte"
+  import { getDatasourceInfo } from "builderStore/datasource"
 
   export let datasource
   export let save
@@ -34,7 +34,7 @@
   let selectedFromRelationship, selectedToRelationship
   let confirmDialog
   let specificTables = null
-  let requireSpecificTables = false
+  let tableList
 
   $: integration = datasource && $integrations[datasource.source]
   $: plusTables = datasource?.plus
@@ -153,30 +153,28 @@
   warning={false}
   title="Confirm table fetch"
 >
-  <Toggle
-    bind:value={requireSpecificTables}
-    on:change={e => {
-      requireSpecificTables = e.detail
-      specificTables = null
-    }}
-    thin
-    text="Fetch listed tables only (one per line)"
-  />
-  {#if requireSpecificTables}
-    <ValuesList label="" bind:values={specificTables} />
-  {/if}
-  <br />
   <Body>
     If you have fetched tables from this database before, this action may
     overwrite any changes you made after your initial fetch.
   </Body>
+  <br />
+  <div class="table-checkboxes">
+    <FancyCheckboxGroup options={tableList} bind:selected={specificTables} />
+  </div>
 </ConfirmDialog>
 
 <Divider />
 <div class="query-header">
   <Heading size="S">Tables</Heading>
   <div class="table-buttons">
-    <Button secondary on:click={() => confirmDialog.show()}>
+    <Button
+      secondary
+      on:click={async () => {
+        const info = await getDatasourceInfo(datasource)
+        tableList = info.tableNames
+        confirmDialog.show()
+      }}
+    >
       Fetch tables
     </Button>
     <Button cta icon="Add" on:click={createNewTable}>New table</Button>
@@ -246,4 +244,8 @@
     display: flex;
     gap: var(--spacing-m);
   }
+
+  .table-checkboxes {
+    width: 100%;
+  }
 </style>

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -44,6 +44,9 @@ export default ICONS
 
 export function getIcon(integrationType, schema) {
   const integrationList = get(integrations)
+  if (!integrationList) {
+    return
+  }
   if (integrationList[integrationType]?.iconUrl) {
     return { url: integrationList[integrationType].iconUrl }
   } else if (schema?.custom || !ICONS[integrationType]) {

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -1,12 +1,19 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
+  import {
+    ModalContent,
+    notifications,
+    Body,
+    Layout,
+    FancyCheckboxGroup,
+  } from "@budibase/bbui"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
   import { IntegrationNames } from "constants/backend"
   import cloneDeep from "lodash/cloneDeepWith"
   import {
     saveDatasource as save,
     validateDatasourceConfig,
+    getDatasourceInfo,
   } from "builderStore/datasource"
   import { DatasourceFeature } from "@budibase/types"
 
@@ -15,11 +22,24 @@
   // kill the reference so the input isn't saved
   let datasource = cloneDeep(integration)
   let isValid = false
+  let fetchTableStep = false
+  let selectedTables = []
+  let tableList = []
 
   $: name =
-    IntegrationNames[datasource.type] || datasource.name || datasource.type
+    IntegrationNames[datasource?.type] || datasource?.name || datasource?.type
+  $: datasourcePlus = datasource?.plus
+  $: title = fetchTableStep ? "Fetch your tables" : `Connect to ${name}`
+  $: confirmText = fetchTableStep
+    ? "Continue"
+    : datasourcePlus
+    ? "Connect"
+    : "Save and continue to query"
 
   async function validateConfig() {
+    if (!integration.features?.[DatasourceFeature.CONNECTION_CHECKING]) {
+      return true
+    }
     const displayError = message =>
       notifications.error(message ?? "Error validating datasource")
 
@@ -37,45 +57,79 @@
   }
 
   async function saveDatasource() {
-    if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
-      const valid = await validateConfig()
-      if (!valid) {
-        return false
-      }
-    }
     try {
       if (!datasource.name) {
         datasource.name = name
       }
-      const resp = await save(datasource)
+      const opts = {}
+      if (datasourcePlus && selectedTables) {
+        opts.tablesFilter = selectedTables
+      }
+      const resp = await save(datasource, opts)
       $goto(`./datasource/${resp._id}`)
-      notifications.success(`Datasource created successfully.`)
+      notifications.success("Datasource created successfully.")
     } catch (err) {
       notifications.error(err?.message ?? "Error saving datasource")
       // prevent the modal from closing
       return false
     }
   }
+
+  async function nextStep() {
+    let connected = true
+    if (datasourcePlus) {
+      connected = await validateConfig()
+    }
+    if (!connected) {
+      return false
+    }
+    if (datasourcePlus && !fetchTableStep) {
+      notifications.success("Connected to datasource successfully.")
+      const info = await getDatasourceInfo(datasource)
+      tableList = info.tableNames
+      fetchTableStep = true
+      return false
+    } else {
+      await saveDatasource()
+      return true
+    }
+  }
 </script>
 
 <ModalContent
-  title={`Connect to ${name}`}
-  onConfirm={() => saveDatasource()}
-  confirmText={datasource.plus ? "Connect" : "Save and continue to query"}
-  cancelText="Back"
-  showSecondaryButton={datasource.plus}
+  {title}
+  onConfirm={() => nextStep()}
+  {confirmText}
+  cancelText={fetchTableStep ? "Cancel" : "Back"}
+  showSecondaryButton={datasourcePlus}
   size="L"
   disabled={!isValid}
 >
   <Layout noPadding>
-    <Body size="XS"
-      >Connect your database to Budibase using the config below.
+    <Body size="XS">
+      {#if !fetchTableStep}
+        Connect your database to Budibase using the config below
+      {:else}
+        Choose what tables you want to sync with Budibase
+      {/if}
     </Body>
   </Layout>
-  <IntegrationConfigForm
-    schema={datasource.schema}
-    bind:datasource
-    creating={true}
-    on:valid={e => (isValid = e.detail)}
-  />
+  {#if !fetchTableStep}
+    <IntegrationConfigForm
+      schema={datasource?.schema}
+      bind:datasource
+      creating={true}
+      on:valid={e => (isValid = e.detail)}
+    />
+  {:else}
+    <div class="table-checkboxes">
+      <FancyCheckboxGroup options={tableList} bind:selected={selectedTables} />
+    </div>
+  {/if}
 </ModalContent>
+
+<style>
+  .table-checkboxes {
+    width: 100%;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte

@@ -1,22 +1,27 @@
 <script>
   import {
-    ModalContent,
     Body,
+    FancyCheckboxGroup,
+    InlineAlert,
     Layout,
     Link,
+    ModalContent,
     notifications,
   } from "@budibase/bbui"
   import { IntegrationNames, IntegrationTypes } from "constants/backend"
   import GoogleButton from "../_components/GoogleButton.svelte"
   import { organisation } from "stores/portal"
-  import { onMount } from "svelte"
-  import { validateDatasourceConfig } from "builderStore/datasource"
+  import { onDestroy, onMount } from "svelte"
+  import {
+    getDatasourceInfo,
+    saveDatasource,
+    validateDatasourceConfig,
+  } from "builderStore/datasource"
   import cloneDeep from "lodash/cloneDeepWith"
   import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
   import { goto } from "@roxi/routify"
-
-  import { saveDatasource } from "builderStore/datasource"
   import { DatasourceFeature } from "@budibase/types"
+  import { API } from "api"
 
   export let integration
   export let continueSetupId = false
@@ -24,16 +29,20 @@
   let datasource = cloneDeep(integration)
   datasource.config.continueSetupId = continueSetupId
 
+  let { schema } = datasource
+
   $: isGoogleConfigured = !!$organisation.googleDatasourceConfigured
 
   onMount(async () => {
     await organisation.init()
   })
+
   const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
 
   export const GoogleDatasouceConfigStep = {
-    AUTH: "Auth",
-    SET_URL: "Set_url",
+    AUTH: "auth",
+    SET_URL: "set_url",
+    SET_SHEETS: "set_sheets",
   }
 
   let step = continueSetupId
@@ -42,12 +51,21 @@
 
   let isValid = false
 
-  const modalConfig = {
-    [GoogleDatasouceConfigStep.AUTH]: {},
+  let allSheets
+  let selectedSheets
+  let setSheetsErrorTitle, setSheetsErrorMessage
+
+  $: modalConfig = {
+    [GoogleDatasouceConfigStep.AUTH]: {
+      title: `Connect to ${integrationName}`,
+    },
     [GoogleDatasouceConfigStep.SET_URL]: {
+      title: `Connect your spreadsheet`,
       confirmButtonText: "Connect",
       onConfirm: async () => {
-        if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
+        const checkConnection =
+          integration.features[DatasourceFeature.CONNECTION_CHECKING]
+        if (checkConnection) {
           const resp = await validateDatasourceConfig(datasource)
           if (!resp.connected) {
             notifications.error(`Unable to connect - ${resp.error}`)
@@ -56,21 +74,81 @@
         }
 
         try {
-          const resp = await saveDatasource(datasource)
-          $goto(`./datasource/${resp._id}`)
-          notifications.success(`Datasource created successfully.`)
+          datasource = await saveDatasource(datasource, {
+            tablesFilter: selectedSheets,
+            skipFetch: true,
+          })
         } catch (err) {
           notifications.error(err?.message ?? "Error saving datasource")
           // prevent the modal from closing
           return false
         }
+
+        if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) {
+          notifications.success(`Datasource created successfully.`)
+          return
+        }
+
+        const info = await getDatasourceInfo(datasource)
+        allSheets = info.tableNames
+
+        step = GoogleDatasouceConfigStep.SET_SHEETS
+        notifications.success(
+          checkConnection
+            ? "Connection Successful"
+            : `Datasource created successfully.`
+        )
+
+        // prevent the modal from closing
+        return false
+      },
+    },
+    [GoogleDatasouceConfigStep.SET_SHEETS]: {
+      title: `Choose your sheets`,
+      confirmButtonText: selectedSheets?.length
+        ? "Fetch sheets"
+        : "Continue without fetching",
+      onConfirm: async () => {
+        try {
+          if (selectedSheets.length) {
+            await API.buildDatasourceSchema({
+              datasourceId: datasource._id,
+              tablesFilter: selectedSheets,
+            })
+          }
+
+          return
+        } catch (err) {
+          const message = err?.message ?? "Error fetching the sheets"
+          // Handling message with format: Error title - error description
+          const indexSeparator = message.indexOf(" - ")
+          if (indexSeparator >= 0) {
+            setSheetsErrorTitle = message.substr(0, indexSeparator)
+            setSheetsErrorMessage =
+              message[indexSeparator + 3].toUpperCase() +
+              message.substr(indexSeparator + 4)
+          } else {
+            setSheetsErrorTitle = null
+            setSheetsErrorMessage = message
+          }
+
+          // prevent the modal from closing
+          return false
+        }
       },
     },
   }
+
+  // This will handle the user closing the modal pressing outside the modal
+  onDestroy(() => {
+    if (step === GoogleDatasouceConfigStep.SET_SHEETS) {
+      $goto(`./datasource/${datasource._id}`)
+    }
+  })
 </script>
 
 <ModalContent
-  title={`Connect to ${integrationName}`}
+  title={modalConfig[step].title}
   cancelText="Cancel"
   size="L"
   confirmText={modalConfig[step].confirmButtonText}
@@ -100,11 +178,30 @@
       <Body size="S">Add the URL of the sheet you want to connect.</Body>
 
       <IntegrationConfigForm
-        schema={datasource.schema}
+        {schema}
         bind:datasource
         creating={true}
         on:valid={e => (isValid = e.detail)}
       />
     </Layout>
   {/if}
+  {#if step === GoogleDatasouceConfigStep.SET_SHEETS}
+    <Layout noPadding no>
+      <Body size="S">Select which spreadsheets you want to connect.</Body>
+
+      <FancyCheckboxGroup
+        options={allSheets}
+        bind:selected={selectedSheets}
+        selectAllText="Select all sheets"
+      />
+
+      {#if setSheetsErrorTitle || setSheetsErrorMessage}
+        <InlineAlert
+          type="error"
+          header={setSheetsErrorTitle}
+          message={setSheetsErrorMessage}
+        />
+      {/if}
+    </Layout>
+  {/if}
 </ModalContent>

packages/builder/src/components/start/ExportAppModal.svelte

@@ -5,6 +5,7 @@
     Body,
     InlineAlert,
     Input,
+    notifications,
   } from "@budibase/bbui"
   import { createValidationStore } from "helpers/validation/yup"
 
@@ -50,14 +51,49 @@
     },
   }
 
-  const exportApp = password => {
+  const exportApp = async () => {
     const id = published ? app.prodId : app.devId
-    const appName = encodeURIComponent(app.name)
-    let url = `/api/backups/export?appId=${id}&appname=${appName}&excludeRows=${!includeInternalTablesRows}`
-    if (password) {
-      url += `&encryptPassword=${password}`
+    const url = `/api/backups/export?appId=${id}`
+    await downloadFile(url, { excludeRows, encryptPassword: password })
+  }
+
+  export async function downloadFile(url, body) {
+    try {
+      const response = await fetch(url, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+      })
+
+      if (response.ok) {
+        const contentDisposition = response.headers.get("Content-Disposition")
+
+        const matches = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/.exec(
+          contentDisposition
+        )
+
+        const filename = matches[1].replace(/['"]/g, "")
+
+        const url = URL.createObjectURL(await response.blob())
+
+        const link = document.createElement("a")
+        link.href = url
+        link.download = filename
+        link.click()
+
+        URL.revokeObjectURL(url)
+      } else {
+        notifications.error("Error exporting the app.")
+      }
+    } catch (error) {
+      let message = "Error downloading the exported app"
+      if (error.message) {
+        message += `: ${error.message}`
+      }
+      notifications.error("Error downloading the exported app", message)
     }
-    window.location = url
   }
 </script>
 

packages/builder/src/stores/backend/datasources.js

@@ -57,7 +57,10 @@ export function createDatasourcesStore() {
     return updateDatasource(response)
   }
 
-  const save = async (body, fetchSchema = false) => {
+  const save = async (body, { fetchSchema, tablesFilter } = {}) => {
+    if (fetchSchema == null) {
+      fetchSchema = false
+    }
     let response
     if (body._id) {
       response = await API.updateDatasource(body)
@@ -65,6 +68,7 @@ export function createDatasourcesStore() {
       response = await API.createDatasource({
         datasource: body,
         fetchSchema,
+        tablesFilter,
       })
     }
     return updateDatasource(response)

packages/frontend-core/src/api/datasources.js

@@ -26,13 +26,16 @@ export const buildDatasourceEndpoints = API => ({
    * Creates a datasource
    * @param datasource the datasource to create
    * @param fetchSchema whether to fetch the schema or not
+   * @param tablesFilter a list of tables to actually fetch rather than simply
+   * all that are accessible.
    */
-  createDatasource: async ({ datasource, fetchSchema }) => {
+  createDatasource: async ({ datasource, fetchSchema, tablesFilter }) => {
     return await API.post({
       url: "/api/datasources",
       body: {
         datasource,
         fetchSchema,
+        tablesFilter,
       },
     })
   },
@@ -69,4 +72,15 @@ export const buildDatasourceEndpoints = API => ({
       body: { datasource },
     })
   },
+
+  /**
+   * Fetch table names available within the datasource, for filtering out undesired tables
+   * @param datasource the datasource configuration to use for fetching tables
+   */
+  fetchInfoForDatasource: async datasource => {
+    return await API.post({
+      url: `/api/datasources/info`,
+      body: { datasource },
+    })
+  },
 })

packages/server/src/api/controllers/backup.ts

@@ -1,17 +1,25 @@
 import sdk from "../../sdk"
-import { events, context } from "@budibase/backend-core"
+import { events, context, db } from "@budibase/backend-core"
 import { DocumentType } from "../../db/utils"
-import { isQsTrue } from "../../utilities"
+import { Ctx } from "@budibase/types"
+
+interface ExportAppDumpRequest {
+  excludeRows: boolean
+  encryptPassword?: string
+}
+
+export async function exportAppDump(ctx: Ctx<ExportAppDumpRequest>) {
+  const { appId } = ctx.query as any
+  const { excludeRows, encryptPassword } = ctx.request.body
+
+  const [app] = await db.getAppsByIDs([appId])
+  const appName = app.name
 
-export async function exportAppDump(ctx: any) {
-  let { appId, excludeRows = false, encryptPassword } = ctx.query
   // remove the 120 second limit for the request
   ctx.req.setTimeout(0)
-  const appName = decodeURI(ctx.query.appname)
-  excludeRows = isQsTrue(excludeRows)
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}${
-    encryptPassword ? ".enc" : ""
-  }.tar.gz`
+
+  const extension = encryptPassword ? "enc.tar.gz" : "tar.gz"
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}`
   ctx.attachment(backupIdentifier)
   ctx.body = await sdk.backups.streamExportApp({
     appId,

packages/server/src/api/controllers/datasource.ts

@@ -103,6 +103,22 @@ async function buildSchemaHelper(datasource: Datasource) {
   return { tables: connector.tables, error }
 }
 
+async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
+  let { tables, error } = await buildSchemaHelper(datasource)
+  let finalTables = tables
+  if (filter) {
+    finalTables = {}
+    for (let key in tables) {
+      if (
+        filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase())
+      ) {
+        finalTables[key] = tables[key]
+      }
+    }
+  }
+  return { tables: finalTables, error }
+}
+
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -174,43 +190,28 @@ export async function information(
   }
   const tableNames = await connector.getTableNames()
   ctx.body = {
-    tableNames,
+    tableNames: tableNames.sort(),
   }
 }
 
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  let { tables, error } = await buildSchemaHelper(datasource)
-  if (tablesFilter) {
-    if (!datasource.entities) {
-      datasource.entities = {}
-    }
-    for (let key in tables) {
-      if (
-        tablesFilter.some(
-          (filter: any) => filter.toLowerCase() === key.toLowerCase()
-        )
-      ) {
-        datasource.entities[key] = tables[key]
-      }
-    }
-  } else {
-    datasource.entities = tables
-  }
+  const { tables, error } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
   const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const response: any = { datasource: cleanedDatasource }
+  const res: any = { datasource: cleanedDatasource }
   if (error) {
-    response.error = error
+    res.error = error
   }
-  ctx.body = response
+  ctx.body = res
 }
 
 /**
@@ -320,6 +321,7 @@ export async function save(
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
+  const tablesFilter = ctx.request.body.tablesFilter
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
@@ -329,7 +331,10 @@ export async function save(
 
   let schemaError = null
   if (fetchSchema) {
-    const { tables, error } = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     schemaError = error
     datasource.entities = tables
     setDefaultDisplayColumns(datasource)

packages/server/src/api/routes/backup.ts

@@ -5,7 +5,7 @@ import { permissions } from "@budibase/backend-core"
 
 const router: Router = new Router()
 
-router.get(
+router.post(
   "/api/backups/export",
   authorized(permissions.BUILDER),
   controller.exportAppDump

packages/server/src/events/docUpdates/syncUsers.ts

@@ -26,6 +26,10 @@ export default function process(updateCb?: UpdateCallback) {
       // if something not found - no changes to perform
       if (err?.status === 404) {
         return
+      }
+      // The user has already been sync in another process
+      else if (err?.status === 409) {
+        return
       } else {
         logging.logAlert("Failed to perform user/group app sync", err)
       }

packages/server/src/integrations/googlesheets.ts

@@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
 import sdk from "../sdk"
 
@@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async testConnection(): Promise<ConnectionInfo> {
     try {
-      await setupCreationAuth(this.config)
       await this.connect()
       return { connected: true }
     } catch (e: any) {
@@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async connect() {
     try {
+      await setupCreationAuth(this.config)
+
       // Initialise oAuth client
       let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
@@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    // not fully configured yet
-    if (!this.config.auth) {
-      return
-    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    for (let sheet of sheets) {
-      // must fetch rows to determine schema
-      await sheet.getRows()
+    await utils.parallelForeach(
+      sheets,
+      async sheet => {
+        // must fetch rows to determine schema
+        await sheet.getRows({ limit: 0, offset: 0 })
 
-      const id = buildExternalTableId(datasourceId, sheet.title)
-      tables[sheet.title] = this.getTableSchema(
-        sheet.title,
-        sheet.headerValues,
-        id
-      )
-    }
+        const id = buildExternalTableId(datasourceId, sheet.title)
+        tables[sheet.title] = this.getTableSchema(
+          sheet.title,
+          sheet.headerValues,
+          id
+        )
+      },
+      10
+    )
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors

packages/server/src/integrations/postgres.ts

@@ -322,7 +322,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       await this.openConnection()
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
-      return columnsResponse.rows.map(row => row.table_name)
+      const names = columnsResponse.rows.map(row => row.table_name)
+      return [...new Set(names)]
     } finally {
       await this.closeConnection()
     }

packages/server/src/sdk/app/backups/exports.ts

@@ -187,7 +187,7 @@ export async function streamExportApp({
 }: {
   appId: string
   excludeRows: boolean
-  encryptPassword: string
+  encryptPassword?: string
 }) {
   const tmpPath = await exportApp(appId, {
     excludeRows,

packages/server/src/sdk/app/datasources/datasources.ts

@@ -164,5 +164,6 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
       delete update.config[key]
     }
   }
+
   return update
 }

packages/shared-core/src/utils.ts

@@ -4,3 +4,42 @@ export function unreachable(
 ) {
   throw new Error(message)
 }
+
+export async function parallelForeach<T>(
+  items: T[],
+  task: (item: T) => Promise<void>,
+  maxConcurrency: number
+): Promise<void> {
+  const promises: Promise<void>[] = []
+  let index = 0
+
+  const processItem = async (item: T) => {
+    try {
+      await task(item)
+    } finally {
+      processNext()
+    }
+  }
+
+  const processNext = () => {
+    if (index >= items.length) {
+      // No more items to process
+      return
+    }
+
+    const item = items[index]
+    index++
+
+    const promise = processItem(item)
+    promises.push(promise)
+
+    if (promises.length >= maxConcurrency) {
+      Promise.race(promises).then(processNext)
+    } else {
+      processNext()
+    }
+  }
+  processNext()
+
+  await Promise.all(promises)
+}

packages/types/src/api/web/app/datasource.ts

@@ -12,6 +12,7 @@ export interface UpdateDatasourceResponse {
 export interface CreateDatasourceRequest {
   datasource: Datasource
   fetchSchema?: boolean
+  tablesFilter: string[]
 }
 
 export interface VerifyDatasourceRequest {

qa-core/scripts/testResultsWebhook.js

@@ -15,6 +15,12 @@ async function generateReport() {
   return JSON.parse(report)
 }
 
+const env = process.argv.slice(2)[0]
+
+if (!env) {
+  throw new Error("environment argument is required")
+}
+
 async function discordResultsNotification(report) {
   const {
     numTotalTestSuites,
@@ -39,8 +45,8 @@ async function discordResultsNotification(report) {
       content: `**Nightly Tests Status**: ${OUTCOME}`,
       embeds: [
         {
-          title: "Budi QA Bot",
-          description: `Nightly Tests`,
+          title: `Budi QA Bot - ${env}`,
+          description: `API Integration Tests`,
           url: GITHUB_ACTIONS_RUN_URL,
           color: OUTCOME === "success" ? 3066993 : 15548997,
           timestamp: new Date(),

qa-core/src/account-api/api/apis/AccountAPI.ts

@@ -60,8 +60,16 @@ export default class AccountAPI {
   }
 
   async delete(accountID: string) {
-    const [response, json] = await this.client.del(`/api/accounts/${accountID}`)
-    expect(response).toHaveStatusCode(200)
+    const [response, json] = await this.client.del(
+      `/api/accounts/${accountID}`,
+      {
+        internal: true,
+      }
+    )
+    // can't use expect here due to use in global teardown
+    if (response.status !== 204) {
+      throw new Error(`Could not delete accountId=${accountID}`)
+    }
     return response
   }
 }

qa-core/src/jest/globalTeardown.ts

@@ -10,6 +10,7 @@ const API_OPTS: APIRequestOpts = { doExpect: false }
 async function deleteAccount() {
   // @ts-ignore
   const accountID = global.qa.accountId
+  // can't run 'expect' blocks in teardown
   await accountsApi.accounts.delete(accountID)
 }