Merge pull request #10680 from Budibase/budi-6933-verify-data-source-connection-during

Verify data source connection before saves

packages/builder/src/builderStore/datasource.js

@@ -2,6 +2,7 @@ import { datasources, tables } from "../stores/backend"
 import { IntegrationNames } from "../constants/backend"
 import { get } from "svelte/store"
 import cloneDeep from "lodash/cloneDeepWith"
+import { API } from "api"
 
 function prepareData(config) {
   let datasource = {}
@@ -37,3 +38,9 @@ export async function createRestDatasource(integration) {
   const config = cloneDeep(integration)
   return saveDatasource(config)
 }
+
+export async function validateDatasourceConfig(config) {
+  const datasource = prepareData(config)
+  const resp = await API.validateDatasource(datasource)
+  return resp
+}

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -53,6 +53,7 @@
       config,
       schema: selected.datasource,
       auth: selected.auth,
+      features: selected.features || [],
     }
     if (selected.friendlyName) {
       integration.name = selected.friendlyName

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -4,55 +4,68 @@
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
   import { IntegrationNames } from "constants/backend"
   import cloneDeep from "lodash/cloneDeepWith"
-  import { saveDatasource as save } from "builderStore/datasource"
+  import {
-  import { onMount } from "svelte"
+    saveDatasource as save,
+    validateDatasourceConfig,
+  } from "builderStore/datasource"
+  import { DatasourceFeature } from "@budibase/types"
 
   export let integration
   export let modal
 
   // kill the reference so the input isn't saved
   let datasource = cloneDeep(integration)
-  let skipFetch = false
   let isValid = false
 
   $: name =
     IntegrationNames[datasource.type] || datasource.name || datasource.type
 
+  async function validateConfig() {
+    const displayError = message =>
+      notifications.error(message ?? "Error validating datasource")
+
+    let connected = false
+    try {
+      const resp = await validateDatasourceConfig(datasource)
+      if (!resp.connected) {
+        displayError(`Unable to connect - ${resp.error}`)
+      }
+      connected = resp.connected
+    } catch (err) {
+      displayError(err?.message)
+    }
+    return connected
+  }
+
   async function saveDatasource() {
+    if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
+      const valid = await validateConfig()
+      if (!valid) {
+        return false
+      }
+    }
     try {
       if (!datasource.name) {
         datasource.name = name
       }
-      const resp = await save(datasource, skipFetch)
+      const resp = await save(datasource)
       $goto(`./datasource/${resp._id}`)
-      notifications.success(`Datasource updated successfully.`)
+      notifications.success(`Datasource created successfully.`)
     } catch (err) {
       notifications.error(err?.message ?? "Error saving datasource")
       // prevent the modal from closing
       return false
     }
   }
-
-  onMount(() => {
-    skipFetch = false
-  })
 </script>
 
 <ModalContent
   title={`Connect to ${name}`}
   onConfirm={() => saveDatasource()}
   onCancel={() => modal.show()}
-  confirmText={datasource.plus
+  confirmText={datasource.plus ? "Connect" : "Save and continue to query"}
-    ? "Save and fetch tables"
-    : "Save and continue to query"}
   cancelText="Back"
   showSecondaryButton={datasource.plus}
-  secondaryButtonText={datasource.plus ? "Skip table fetch" : undefined}
-  secondaryAction={() => {
-    skipFetch = true
-    saveDatasource()
-    return true
-  }}
   size="L"
   disabled={!isValid}
 >

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte

@@ -20,6 +20,8 @@
   import { isEqual } from "lodash"
   import { cloneDeep } from "lodash/fp"
   import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
+  import { API } from "api"
+  import { DatasourceFeature } from "@budibase/types"
 
   const querySchema = {
     name: {},
@@ -45,7 +47,30 @@
     }
   }
 
+  async function validateConfig() {
+    const displayError = message =>
+      notifications.error(message ?? "Error validating datasource")
+
+    let connected = false
+    try {
+      const resp = await API.validateDatasource(datasource)
+      if (!resp.connected) {
+        displayError(`Unable to connect - ${resp.error}`)
+      }
+      connected = resp.connected
+    } catch (err) {
+      displayError(err?.message)
+    }
+    return connected
+  }
+
   const saveDatasource = async () => {
+    if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
+      const valid = await validateConfig()
+      if (!valid) {
+        return false
+      }
+    }
     try {
       // Create datasource
       await datasources.save(datasource)

packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte

@@ -31,6 +31,18 @@
       return "Invalid URL"
     }
   }
+
+  $: urlManuallySet = false
+
+  const updateUrl = event => {
+    const appName = event.detail
+    if (urlManuallySet) {
+      return
+    }
+
+    const parsedUrl = appName.toLowerCase().replace(/\s+/g, "-")
+    url = encodeURI(parsedUrl)
+  }
 </script>
 
 <div>
@@ -43,11 +55,13 @@
       bind:value={name}
       bind:error={nameError}
       validate={validateName}
+      on:change={updateUrl}
       label="Name"
     />
     <FancyInput
       bind:value={url}
       bind:error={urlError}
+      on:change={() => (urlManuallySet = true)}
       validate={validateUrl}
       label="URL"
     />

packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte

@@ -18,6 +18,8 @@
   import { Roles } from "constants/backend"
   import Spinner from "components/common/Spinner.svelte"
   import { helpers } from "@budibase/shared-core"
+  import { validateDatasourceConfig } from "builderStore/datasource"
+  import { DatasourceFeature } from "@budibase/types"
 
   let name = "My first app"
   let url = "my-first-app"
@@ -108,7 +110,24 @@
     isGoogle,
   }) => {
     let app
+
     try {
+      if (
+        datasourceConfig &&
+        plusIntegrations[stage].features[DatasourceFeature.CONNECTION_CHECKING]
+      ) {
+        const resp = await validateDatasourceConfig({
+          config: datasourceConfig,
+          type: stage,
+        })
+        if (!resp.connected) {
+          notifications.error(
+            `Unable to connect - ${resp.error ?? "Error validating datasource"}`
+          )
+          return false
+        }
+      }
+
       app = await createApp(useSampleData)
 
       let datasource

packages/builder/src/stores/portal/features.js

@@ -1,7 +1,7 @@
 import { writable } from "svelte/store"
 import { API } from "api"
 import { licensing } from "./licensing"
-import { ConfigType } from "../../../../types/src/documents"
+import { ConfigType } from "@budibase/types"
 
 export const createFeatureStore = () => {
   const internalStore = writable({

packages/frontend-core/src/api/datasources.js

@@ -58,4 +58,15 @@ export const buildDatasourceEndpoints = API => ({
       url: `/api/datasources/${datasourceId}/${datasourceRev}`,
     })
   },
+
+  /**
+   * Validate a datasource configuration
+   * @param datasource the datasource configuration to validate
+   */
+  validateDatasource: async datasource => {
+    return await API.post({
+      url: `/api/datasources/verify`,
+      body: { datasource },
+    })
+  },
 })

packages/server/src/integrations/airtable.ts

@@ -20,7 +20,9 @@ const SCHEMA: Integration = {
     "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
   friendlyName: "Airtable",
   type: "Spreadsheet",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     apiKey: {
       type: DatasourceFieldType.PASSWORD,

packages/server/src/integrations/arangodb.ts

@@ -23,7 +23,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/couchdb.ts

@@ -20,7 +20,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/dynamodb.ts

@@ -25,7 +25,9 @@ const SCHEMA: Integration = {
     "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
   friendlyName: "DynamoDB",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     region: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/elasticsearch.ts

@@ -22,7 +22,9 @@ const SCHEMA: Integration = {
     "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
   friendlyName: "ElasticSearch",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/firebase.ts

@@ -20,7 +20,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     email: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/googlesheets.ts

@@ -66,10 +66,10 @@ const SCHEMA: Integration = {
     "Create and collaborate on online spreadsheets in real-time and from any device.",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
-  features: [
+  features: {
-    DatasourceFeature.CONNECTION_CHECKING,
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
-    DatasourceFeature.FETCH_TABLE_NAMES,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
-  ],
+  },
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",

packages/server/src/integrations/microsoftSqlServer.ts

@@ -40,10 +40,10 @@ const SCHEMA: Integration = {
     "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
   friendlyName: "MS SQL Server",
   type: "Relational",
-  features: [
+  features: {
-    DatasourceFeature.CONNECTION_CHECKING,
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
-    DatasourceFeature.FETCH_TABLE_NAMES,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
-  ],
+  },
   datasource: {
     user: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/mongodb.ts

@@ -40,7 +40,9 @@ const getSchema = () => {
     type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
-    features: [DatasourceFeature.CONNECTION_CHECKING],
+    features: {
+      [DatasourceFeature.CONNECTION_CHECKING]: true,
+    },
     datasource: {
       connectionString: {
         type: DatasourceFieldType.STRING,

packages/server/src/integrations/mysql.ts

@@ -36,10 +36,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
-  features: [
+  features: {
-    DatasourceFeature.CONNECTION_CHECKING,
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
-    DatasourceFeature.FETCH_TABLE_NAMES,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
-  ],
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/oracle.ts

@@ -50,10 +50,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "Oracle Database is an object-relational database management system developed by Oracle Corporation",
-  features: [
+  features: {
-    DatasourceFeature.CONNECTION_CHECKING,
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
-    DatasourceFeature.FETCH_TABLE_NAMES,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
-  ],
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/postgres.ts

@@ -52,10 +52,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
-  features: [
+  features: {
-    DatasourceFeature.CONNECTION_CHECKING,
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
-    DatasourceFeature.FETCH_TABLE_NAMES,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
-  ],
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

packages/server/src/integrations/redis.ts

@@ -21,7 +21,9 @@ const SCHEMA: Integration = {
     "Redis is a caching tool, providing powerful key-value store capabilities.",
   friendlyName: "Redis",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     host: {
       type: "string",

packages/server/src/integrations/s3.ts

@@ -24,7 +24,9 @@ const SCHEMA: Integration = {
     "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
   friendlyName: "Amazon S3",
   type: "Object store",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     region: {
       type: "string",

packages/server/src/integrations/snowflake.ts

@@ -22,7 +22,9 @@ const SCHEMA: Integration = {
     "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
   friendlyName: "Snowflake",
   type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     account: {
       type: "string",

packages/types/src/sdk/datasources.ts

@@ -116,7 +116,7 @@ export interface Integration {
   docs: string
   plus?: boolean
   auth?: { type: string }
-  features?: DatasourceFeature[]
+  features?: Partial<Record<DatasourceFeature, boolean>>
   relationships?: boolean
   description: string
   friendlyName: string