Merge branch 'develop' into merge-master

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -1,10 +1,10 @@
 import * as google from "../sso/google"
 import { Cookie } from "../../../constants"
 import { clearCookie, getCookie } from "../../../utils"
-import { doWithDB } from "../../../db"
 import * as configs from "../../../configs"
-import { BBContext, Database, SSOProfile } from "@budibase/types"
+import { BBContext, SSOProfile } from "@budibase/types"
 import { ssoSaveUserNoOp } from "../sso/sso"
+import { cache, utils } from "../../../"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
 type Passport = {
@@ -36,8 +36,8 @@ export async function preAuth(
     ssoSaveUserNoOp
   )
 
-  if (!ctx.query.appId || !ctx.query.datasourceId) {
-    ctx.throw(400, "appId and datasourceId query params not present.")
+  if (!ctx.query.appId) {
+    ctx.throw(400, "appId query param not present.")
   }
 
   return passport.authenticate(strategy, {
@@ -69,7 +69,7 @@ export async function postAuth(
       (
         accessToken: string,
         refreshToken: string,
-        profile: SSOProfile,
+        _profile: SSOProfile,
         done: Function
       ) => {
         clearCookie(ctx, Cookie.DatasourceAuth)
@@ -79,23 +79,16 @@ export async function postAuth(
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, tokens: string[]) => {
       const baseUrl = `/builder/app/${authStateCookie.appId}/data`
-      // update the DB for the datasource with all the user info
-      await doWithDB(authStateCookie.appId, async (db: Database) => {
-        let datasource
-        try {
-          datasource = await db.get(authStateCookie.datasourceId)
-        } catch (err: any) {
-          if (err.status === 404) {
-            ctx.redirect(baseUrl)
-          }
+
+      const id = utils.newid()
+      await cache.store(
+        `datasource:creation:${authStateCookie.appId}:google:${id}`,
+        {
+          tokens,
         }
-        if (!datasource.config) {
-          datasource.config = {}
-        }
-        datasource.config.auth = { type: "google", ...tokens }
-        await db.put(datasource)
-        ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`)
-      })
+      )
+
+      ctx.redirect(`${baseUrl}/new?continue_google_setup=${id}`)
     }
   )(ctx, next)
 }

packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte

@@ -3,8 +3,6 @@
   import { store } from "builderStore"
   import { auth } from "stores/portal"
 
-  export let preAuthStep
-  export let datasource
   export let disabled
   export let samePage
 
@@ -15,18 +13,8 @@
   class:disabled
   {disabled}
   on:click={async () => {
-    let ds = datasource
     let appId = $store.appId
-    if (!ds) {
-      const resp = await preAuthStep()
-      if (resp.datasource && resp.appId) {
-        ds = resp.datasource
-        appId = resp.appId
-      } else {
-        ds = resp
-      }
-    }
-    const url = `/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${appId}`
+    const url = `/api/global/auth/${tenantId}/datasource/google?appId=${appId}`
     if (samePage) {
       window.location = url
     } else {

packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte

@@ -1,43 +1,110 @@
 <script>
-  import { ModalContent, Body, Layout, Link } from "@budibase/bbui"
-  import { IntegrationNames } from "constants/backend"
-  import cloneDeep from "lodash/cloneDeepWith"
+  import {
+    ModalContent,
+    Body,
+    Layout,
+    Link,
+    notifications,
+  } from "@budibase/bbui"
+  import { IntegrationNames, IntegrationTypes } from "constants/backend"
   import GoogleButton from "../_components/GoogleButton.svelte"
-  import { saveDatasource as save } from "builderStore/datasource"
   import { organisation } from "stores/portal"
   import { onMount } from "svelte"
+  import { validateDatasourceConfig } from "builderStore/datasource"
+  import cloneDeep from "lodash/cloneDeepWith"
+  import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
+  import { goto } from "@roxi/routify"
+
+  import { saveDatasource } from "builderStore/datasource"
+  import { DatasourceFeature } from "@budibase/types"
 
   export let integration
+  export let continueSetupId = false
 
-  // kill the reference so the input isn't saved
   let datasource = cloneDeep(integration)
+  datasource.config.continueSetupId = continueSetupId
+
   $: isGoogleConfigured = !!$organisation.googleDatasourceConfigured
 
   onMount(async () => {
     await organisation.init()
   })
+  const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
+
+  export const GoogleDatasouceConfigStep = {
+    AUTH: "Auth",
+    SET_URL: "Set_url",
+  }
+
+  let step = continueSetupId
+    ? GoogleDatasouceConfigStep.SET_URL
+    : GoogleDatasouceConfigStep.AUTH
+
+  let isValid = false
+
+  const modalConfig = {
+    [GoogleDatasouceConfigStep.AUTH]: {},
+    [GoogleDatasouceConfigStep.SET_URL]: {
+      confirmButtonText: "Connect",
+      onConfirm: async () => {
+        if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
+          const resp = await validateDatasourceConfig(datasource)
+          if (!resp.connected) {
+            notifications.error(`Unable to connect - ${resp.error}`)
+            return false
+          }
+        }
+
+        try {
+          const resp = await saveDatasource(datasource)
+          $goto(`./datasource/${resp._id}`)
+          notifications.success(`Datasource created successfully.`)
+        } catch (err) {
+          notifications.error(err?.message ?? "Error saving datasource")
+          // prevent the modal from closing
+          return false
+        }
+      },
+    },
+  }
 </script>
 
 <ModalContent
-  title={`Connect to ${IntegrationNames[datasource.type]}`}
-  cancelText="Back"
+  title={`Connect to ${integrationName}`}
+  cancelText="Cancel"
   size="L"
+  confirmText={modalConfig[step].confirmButtonText}
+  showConfirmButton={!!modalConfig[step].onConfirm}
+  onConfirm={modalConfig[step].onConfirm}
+  disabled={!isValid}
 >
-  <!-- check true and false directly, don't render until flag is set -->
-  {#if isGoogleConfigured === true}
-    <Layout noPadding>
+  {#if step === GoogleDatasouceConfigStep.AUTH}
+    <!-- check true and false directly, don't render until flag is set -->
+    {#if isGoogleConfigured === true}
+      <Layout noPadding>
+        <Body size="S"
+          >Authenticate with your google account to use the {integrationName} integration.</Body
+        >
+      </Layout>
+      <GoogleButton samePage />
+    {:else if isGoogleConfigured === false}
       <Body size="S"
-        >Authenticate with your google account to use the {IntegrationNames[
-          datasource.type
-        ]} integration.</Body
+        >Google authentication is not enabled, please complete Google SSO
+        configuration.</Body
       >
+      <Link href="/builder/portal/settings/auth">Configure Google SSO</Link>
+    {/if}
+  {/if}
+  {#if step === GoogleDatasouceConfigStep.SET_URL}
+    <Layout noPadding no>
+      <Body size="S">Add the URL of the sheet you want to connect.</Body>
+
+      <IntegrationConfigForm
+        schema={datasource.schema}
+        bind:datasource
+        creating={true}
+        on:valid={e => (isValid = e.detail)}
+      />
     </Layout>
-    <GoogleButton preAuthStep={() => save(datasource, true)} />
-  {:else if isGoogleConfigured === false}
-    <Body size="S"
-      >Google authentication is not enabled, please complete Google SSO
-      configuration.</Body
-    >
-    <Link href="/builder/portal/settings/auth">Configure Google SSO</Link>
   {/if}
 </ModalContent>

packages/builder/src/pages/builder/app/[application]/data/new.svelte

@@ -17,6 +17,7 @@
   import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte"
   import ICONS from "components/backend/DatasourceNavigator/icons/index.js"
   import FontAwesomeIcon from "components/common/FontAwesomeIcon.svelte"
+  import { onMount } from "svelte"
 
   let internalTableModal
   let externalDatasourceModal
@@ -129,9 +130,19 @@
     return integrationsArray
   }
 
+  let continueGoogleSetup
+  onMount(() => {
+    const urlParams = new URLSearchParams(window.location.search)
+    continueGoogleSetup = urlParams.get("continue_google_setup")
+  })
+
   const fetchIntegrations = async () => {
     const unsortedIntegrations = await API.getIntegrations()
     integrations = sortIntegrations(unsortedIntegrations)
+
+    if (continueGoogleSetup) {
+      handleIntegrationSelect(IntegrationTypes.GOOGLE_SHEETS)
+    }
   }
 
   $: fetchIntegrations()
@@ -141,9 +152,17 @@
   <CreateTableModal {promptUpload} afterSave={handleInternalTableSave} />
 </Modal>
 
-<Modal bind:this={externalDatasourceModal}>
+<Modal
+  bind:this={externalDatasourceModal}
+  on:hide={() => {
+    continueGoogleSetup = null
+  }}
+>
   {#if integration?.auth?.type === "google"}
-    <GoogleDatasourceConfigModal {integration} />
+    <GoogleDatasourceConfigModal
+      continueSetupId={continueGoogleSetup}
+      {integration}
+    />
   {:else}
     <DatasourceConfigModal {integration} />
   {/if}

packages/server/src/api/controllers/datasource.ts

@@ -11,7 +11,7 @@ import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
-import { db as dbCore, context, events } from "@budibase/backend-core"
+import { db as dbCore, context, events, cache } from "@budibase/backend-core"
 import {
   UserCtx,
   Datasource,
@@ -25,9 +25,11 @@ import {
   FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
+  SourceName,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
+import { setupCreationAuth as googleSetupCreationAuth } from "src/integrations/googlesheets"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -306,6 +308,12 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
+const preSaveAction: Partial<Record<SourceName, any>> = {
+  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
+    await googleSetupCreationAuth(datasource.config as any)
+  },
+}
+
 export async function save(
   ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
 ) {
@@ -327,6 +335,10 @@ export async function save(
     setDefaultDisplayColumns(datasource)
   }
 
+  if (preSaveAction[datasource.source]) {
+    await preSaveAction[datasource.source](datasource)
+  }
+
   const dbResp = await db.put(datasource)
   await events.datasource.created(datasource)
   datasource._rev = dbResp.rev

packages/server/src/integrations/googlesheets.ts

@@ -1,5 +1,6 @@
 import {
   ConnectionInfo,
+  Datasource,
   DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
@@ -19,13 +20,15 @@ import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
-import { configs, HTTPError } from "@budibase/backend-core"
+import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
+import sdk from "../sdk"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
   auth: OAuthClientConfig
+  continueSetupId?: string
 }
 
 interface OAuthClientConfig {
@@ -72,7 +75,7 @@ const SCHEMA: Integration = {
   },
   datasource: {
     spreadsheetId: {
-      display: "Google Sheet URL",
+      display: "Spreadsheet URL",
       type: DatasourceFieldType.STRING,
       required: true,
     },
@@ -147,6 +150,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async testConnection(): Promise<ConnectionInfo> {
     try {
+      await setupCreationAuth(this.config)
       await this.connect()
       return { connected: true }
     } catch (e: any) {
@@ -566,6 +570,18 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 }
 
+export async function setupCreationAuth(datasouce: GoogleSheetsConfig) {
+  if (datasouce.continueSetupId) {
+    const appId = context.getAppId()
+    const tokens = await cache.get(
+      `datasource:creation:${appId}:google:${datasouce.continueSetupId}`
+    )
+
+    datasouce.auth = tokens.tokens
+    delete datasouce.continueSetupId
+  }
+}
+
 export default {
   schema: SCHEMA,
   integration: GoogleSheetsIntegration,

packages/worker/src/api/controllers/global/auth.ts

@@ -140,7 +140,6 @@ export const datasourcePreAuth = async (ctx: any, next: any) => {
     {
       provider,
       appId: ctx.query.appId,
-      datasourceId: ctx.query.datasourceId,
     },
     Cookie.DatasourceAuth
   )