Merge pull request #2735 from Budibase/feature/datasource-wizard

Datasource wizard

packages/bbui/src/Modal/ModalContent.svelte

@@ -14,6 +14,7 @@
   export let showConfirmButton = true
   export let showCloseIcon = true
   export let onConfirm = undefined
+  export let onCancel = undefined
   export let disabled = false
   export let showDivider = true
 
@@ -28,6 +29,14 @@
     }
     loading = false
   }
+
+  async function close() {
+    loading = true
+    if (!onCancel || (await onCancel()) !== false) {
+      cancel()
+    }
+    loading = false
+  }
 </script>
 
 <div
@@ -65,7 +74,7 @@
       >
         <slot name="footer" />
         {#if showCancelButton}
-          <Button group secondary on:click={cancel}>{cancelText}</Button>
+          <Button group secondary on:click={close}>{cancelText}</Button>
         {/if}
         {#if showConfirmButton}
           <Button

packages/builder/cypress/integration/createTable.spec.js

@@ -6,7 +6,7 @@ context("Create a Table", () => {
 
   it("should create a new Table", () => {
     cy.createTable("dog")
-
+    cy.wait(1000)
     // Check if Table exists
     cy.get(".table-title h1").should("have.text", "dog")
   })
@@ -36,7 +36,8 @@ context("Create a Table", () => {
   it("edits a row", () => {
     cy.contains("button", "Edit").click({ force: true })
     cy.wait(1000)
-    cy.get(".spectrum-Modal input").type("Updated")
+    cy.get(".spectrum-Modal input").clear()
+    cy.get(".spectrum-Modal input").type("RoverUpdated")
     cy.contains("Save").click()
     cy.contains("RoverUpdated").should("have.text", "RoverUpdated")
   })
@@ -62,7 +63,7 @@ context("Create a Table", () => {
 
   it("deletes a table", () => {
     cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
-      .first()
+      .eq(1)
       .click({ force: true })
     cy.get(".spectrum-Menu > :nth-child(2)").click()
     cy.contains("Delete Table").click()

packages/builder/cypress/support/commands.js

@@ -35,8 +35,11 @@ Cypress.Commands.add("createApp", name => {
     .within(() => {
       cy.get("input").eq(0).type(name).should("have.value", name).blur()
       cy.get(".spectrum-ButtonGroup").contains("Create app").click()
+      cy.wait(7000)
     })
     .then(() => {
+      // Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
+      cy.createInitialDatasource("initialTable")
       cy.expandBudibaseConnection()
       cy.get(".nav-item.selected > .content").should("be.visible")
     })
@@ -69,11 +72,28 @@ Cypress.Commands.add("createTestTableWithData", () => {
   cy.addColumn("dog", "age", "Number")
 })
 
-Cypress.Commands.add("createTable", tableName => {
+Cypress.Commands.add("createInitialDatasource", tableName => {
   // Enter table name
+  cy.get(".spectrum-Modal").within(() => {
+    cy.contains("Budibase DB").trigger("mouseover").click().click()
+    cy.wait(1000)
+    cy.contains("Continue").click()
+  })
+
+  cy.get(".spectrum-Modal").within(() => {
+    cy.wait(1000)
+    cy.get("input").first().type(tableName).blur()
+    cy.get(".spectrum-ButtonGroup").contains("Create").click()
+  })
+  cy.contains(tableName).should("be.visible")
+})
+
+Cypress.Commands.add("createTable", tableName => {
   cy.contains("Budibase DB").click()
   cy.contains("Create new table").click()
+
   cy.get(".spectrum-Modal").within(() => {
+    cy.wait(1000)
     cy.get("input").first().type(tableName).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create").click()
   })

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -1,20 +1,28 @@
 <script>
-  import { Label, Input, Layout, Toggle } from "@budibase/bbui"
+  import { Label, Input, Layout, Toggle, Button } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { capitalise } from "helpers"
 
   export let integration
   export let schema
+  let addButton
 </script>
 
 <form>
   <Layout gap="S">
     {#each Object.keys(schema) as configKey}
       {#if schema[configKey].type === "object"}
-        <Label>{capitalise(configKey)}</Label>
+        <div class="form-row ssl">
+          <Label>{capitalise(configKey)}</Label>
+          <Button secondary thin outline on:click={addButton.addEntry()}
+            >Add</Button
+          >
+        </div>
         <KeyValueBuilder
+          bind:this={addButton}
           defaults={schema[configKey].default}
           bind:object={integration[configKey]}
+          noAddButton={true}
         />
       {:else if schema[configKey].type === "boolean"}
         <div class="form-row">
@@ -42,4 +50,11 @@
     grid-gap: var(--spacing-l);
     align-items: center;
   }
+
+  .form-row.ssl {
+    display: grid;
+    grid-template-columns: 20% 20%;
+    grid-gap: var(--spacing-l);
+    align-items: center;
+  }
 </style>

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -1,74 +1,160 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { ModalContent, Modal, Body, Layout, Detail } from "@budibase/bbui"
-  import { datasources } from "stores/backend"
+  import { onMount } from "svelte"
-  import { notifications } from "@budibase/bbui"
+  import ICONS from "../icons"
-  import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui"
+  import api from "builderStore/api"
-  import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
+  import { IntegrationNames } from "constants"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
-  import analytics, { Events } from "analytics"
+  import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
-  import { getContext } from "svelte"
 
-  const modalContext = getContext(Context.Modal)
+  export let modal
+  let integrations = []
+  let integration = {}
+  let internalTableModal
+  let externalDatasourceModal
 
-  let tableModal
+  const INTERNAL = "BUDIBASE"
-  let name
-  let error = ""
-  let integration
 
-  $: checkOpenModal(integration && integration.type === "BUDIBASE")
+  onMount(() => {
+    fetchIntegrations()
+  })
 
-  function checkValid(evt) {
+  function selectIntegration(integrationType) {
-    const datasourceName = evt.target.value
+    const selected = integrations[integrationType]
-    if (
+
-      $datasources?.list.some(datasource => datasource.name === datasourceName)
+    // build the schema
-    ) {
+    const config = {}
-      error = `Datasource with name ${datasourceName} already exists. Please choose another name.`
+    for (let key of Object.keys(selected.datasource)) {
-      return
+      config[key] = selected.datasource[key].default
     }
-    error = ""
+    integration = {
-  }
+      type: integrationType,
-
+      plus: selected.plus,
-  function checkOpenModal(isInternal) {
-    if (isInternal) {
-      tableModal.show()
-    }
-  }
-
-  async function saveDatasource() {
-    const { type, plus, ...config } = integration
-
-    // Create datasource
-    const response = await datasources.save({
-      name,
-      source: type,
       config,
-      plus,
+      schema: selected.datasource,
-    })
+    }
-    notifications.success(`Datasource ${name} created successfully.`)
+  }
-    analytics.captureEvent(Events.DATASOURCE.CREATED, { name, type })
 
-    // Navigate to new datasource
+  function chooseNextModal() {
-    $goto(`./datasource/${response._id}`)
+    if (integration.type === INTERNAL) {
+      externalDatasourceModal.hide()
+      internalTableModal.show()
+    } else {
+      externalDatasourceModal.show()
+    }
+  }
+
+  async function fetchIntegrations() {
+    const response = await api.get("/api/integrations")
+    const json = await response.json()
+    integrations = {
+      [INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
+      ...json,
+    }
+    return json
   }
 </script>
 
-<Modal bind:this={tableModal} on:hide={modalContext.hide}>
+<Modal bind:this={internalTableModal}>
-  <CreateTableModal bind:name />
+  <CreateTableModal />
 </Modal>
-<ModalContent
+
-  title="Create Datasource"
+<Modal bind:this={externalDatasourceModal}>
-  size="L"
+  <DatasourceConfigModal {integration} />
-  confirmText="Create"
+</Modal>
-  onConfirm={saveDatasource}
+
-  disabled={error || !name || !integration?.type}
+<Modal bind:this={modal}>
->
+  <ModalContent
-  <Input
+    disabled={!Object.keys(integration).length}
-    data-cy="datasource-name-input"
+    title="Data"
-    label="Datasource Name"
+    confirmText="Continue"
-    on:input={checkValid}
+    showCancelButton={false}
-    bind:value={name}
+    size="M"
-    {error}
+    onConfirm={() => {
-  />
+      chooseNextModal()
-  <Label>Datasource Type</Label>
+    }}
-  <TableIntegrationMenu bind:integration />
+  >
-</ModalContent>
+    <Layout noPadding>
+      <Body size="XS"
+        >All apps need data. You can connect to a data source below, or add data
+        to your app using Budibase's built-in database.
+      </Body>
+      <div
+        class:selected={integration.type === INTERNAL}
+        on:click={() => selectIntegration(INTERNAL)}
+        class="item hoverable"
+      >
+        <div class="item-body">
+          <svelte:component this={ICONS.BUDIBASE} height="18" width="18" />
+          <span class="icon-spacing"> <Body size="S">Budibase DB</Body></span>
+        </div>
+      </div>
+    </Layout>
+
+    <Layout gap="XS" noPadding>
+      <div class="title-spacing">
+        <Detail size="S">Connect to data source</Detail>
+      </div>
+      <div class="item-list">
+        {#each Object.entries(integrations).filter(([key]) => key !== INTERNAL) as [integrationType, schema]}
+          <div
+            class:selected={integration.type === integrationType}
+            on:click={() => selectIntegration(integrationType)}
+            class="item hoverable"
+          >
+            <div class="item-body">
+              <svelte:component
+                this={ICONS[integrationType]}
+                height="18"
+                width="18"
+              />
+
+              <span class="icon-spacing">
+                <Body size="S"
+                  >{schema.name || IntegrationNames[integrationType]}</Body
+                ></span
+              >
+            </div>
+          </div>
+        {/each}
+      </div>
+    </Layout>
+  </ModalContent>
+</Modal>
+
+<style>
+  .icon-spacing {
+    margin-left: var(--spacing-m);
+  }
+  .item-body {
+    display: flex;
+    margin-left: var(--spacing-m);
+  }
+  .item-list {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+    grid-gap: var(--spectrum-alias-grid-baseline);
+  }
+
+  .item {
+    cursor: pointer;
+    display: grid;
+    grid-gap: var(--spectrum-alias-grid-margin-xsmall);
+    padding: var(--spectrum-alias-item-padding-s);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: 0.3s all;
+    border: solid var(--spectrum-alias-border-color);
+    border-radius: 5px;
+    box-sizing: border-box;
+    border-width: 2px;
+  }
+
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+
+  .item:hover,
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -0,0 +1,72 @@
+<script>
+  import { goto } from "@roxi/routify"
+  import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
+  import analytics, { Events } from "analytics"
+  import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
+  import { datasources } from "stores/backend"
+  import { IntegrationNames } from "constants"
+
+  export let integration
+
+  function prepareData() {
+    let datasource = {}
+    let existingTypeCount = $datasources.list.filter(
+      ds => ds.source == integration.type
+    ).length
+
+    let baseName = IntegrationNames[integration.type]
+    let name =
+      existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
+
+    datasource.type = "datasource"
+    datasource.source = integration.type
+    datasource.config = integration.config
+    datasource.name = name
+    datasource.plus = integration.plus
+
+    return datasource
+  }
+  async function saveDatasource() {
+    const datasource = prepareData()
+    try {
+      // Create datasource
+      const resp = await datasources.save(datasource, datasource.plus)
+
+      await datasources.select(resp._id)
+      $goto(`./datasource/${resp._id}`)
+      notifications.success(`Datasource updated successfully.`)
+      analytics.captureEvent(Events.DATASOURCE.CREATED, {
+        name: resp.name,
+        source: resp.source,
+      })
+      return true
+    } catch (err) {
+      notifications.error(`Error saving datasource: ${err}`)
+      return false
+    }
+  }
+</script>
+
+<ModalContent
+  title={`Connect to ${IntegrationNames[integration.type]}`}
+  onConfirm={() => saveDatasource()}
+  confirmText={integration.plus
+    ? "Fetch tables from database"
+    : "Save and continue to query"}
+  cancelText="Back"
+  size="M"
+>
+  <Layout noPadding>
+    <Body size="XS"
+      >Connect your database to Budibase using the config below.
+    </Body>
+  </Layout>
+
+  <IntegrationConfigForm
+    schema={integration.schema}
+    bind:integration={integration.config}
+  />
+</ModalContent>
+
+<style>
+</style>

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -4,6 +4,7 @@
   export let defaults
   export let object = defaults || {}
   export let readOnly
+  export let noAddButton
 
   let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
 
@@ -12,7 +13,7 @@
     {}
   )
 
-  function addEntry() {
+  export function addEntry() {
     fields = [...fields, {}]
   }
 
@@ -32,7 +33,7 @@
     {/if}
   {/each}
 </div>
-{#if !readOnly}
+{#if !readOnly && !noAddButton}
   <div>
     <Button secondary thin outline on:click={addEntry}>Add</Button>
   </div>

packages/builder/src/constants/index.js

@@ -15,6 +15,20 @@ export const AppStatus = {
   DEPLOYED: "published",
 }
 
+export const IntegrationNames = {
+  POSTGRES: "PostgreSQL",
+  MONGODB: "MongoDB",
+  COUCHDB: "CouchDB",
+  S3: "S3",
+  MYSQL: "MySQL",
+  REST: "REST",
+  DYNAMODB: "DynamoDB",
+  ELASTICSEARCH: "ElasticSearch",
+  SQL_SERVER: "SQL Server",
+  AIRTABLE: "Airtable",
+  ARANGODB: "ArangoDB",
+}
+
 // fields on the user table that cannot be edited
 export const UNEDITABLE_USER_FIELDS = [
   "email",

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -1,12 +1,14 @@
 <script>
   import { goto, params } from "@roxi/routify"
-  import { Icon, Modal, Tabs, Tab } from "@budibase/bbui"
+  import { Icon, Tabs, Tab } from "@budibase/bbui"
   import { BUDIBASE_INTERNAL_DB } from "constants"
   import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
 
   let selected = "Sources"
+
   let modal
+
   $: isExternal =
     $params.selectedDatasource &&
     $params.selectedDatasource !== BUDIBASE_INTERNAL_DB
@@ -23,9 +25,7 @@
       <Tab title="Sources">
         <div class="tab-content-padding">
           <DatasourceNavigator />
-          <Modal bind:this={modal}>
+          <CreateDatasourceModal bind:modal />
-            <CreateDatasourceModal />
-          </Modal>
         </div>
       </Tab>
     </Tabs>

packages/builder/src/pages/builder/app/[application]/data/index.svelte

@@ -1,6 +1,22 @@
 <script>
   import { goto } from "@roxi/routify"
-  $goto("./table")
+  import { onMount } from "svelte"
+  import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
+  import { datasources } from "stores/backend"
+
+  let modal
+  $: setupComplete =
+    $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 ||
+    $datasources.list.length > 1
+
+  onMount(() => {
+    if (!setupComplete) {
+      modal.show()
+    } else {
+      $goto("./table")
+    }
+  })
 </script>
 
+<CreateDatasourceModal bind:modal />
 <!-- routify:options index=false -->

packages/builder/src/stores/backend/datasources.js

@@ -67,7 +67,7 @@ export function createDatasourcesStore() {
       })
       return json
     },
-    save: async datasource => {
+    save: async (datasource, fetchSchema = false) => {
       let response
       if (datasource._id) {
         response = await api.put(
@@ -75,7 +75,10 @@ export function createDatasourcesStore() {
           datasource
         )
       } else {
-        response = await api.post("/api/datasources", datasource)
+        response = await api.post("/api/datasources", {
+          datasource: datasource,
+          fetchSchema,
+        })
       }
 
       const json = await response.json()

packages/server/src/api/controllers/datasource.js

@@ -41,15 +41,10 @@ exports.fetch = async function (ctx) {
 
 exports.buildSchemaFromDb = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  const datasourceId = ctx.params.datasourceId
+  const datasource = await db.get(ctx.params.datasourceId)
-  const datasource = await db.get(datasourceId)
 
-  const Connector = integrations[datasource.source]
+  const tables = await buildSchemaHelper(datasource)
-
+  datasource.entities = tables
-  // Connect to the DB and build the schema
-  const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id, datasource.entities)
-  datasource.entities = connector.tables
 
   const response = await db.put(datasource)
   datasource._rev = response.rev
@@ -81,12 +76,18 @@ exports.update = async function (ctx) {
 
 exports.save = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  const plus = ctx.request.body.plus
+  const plus = ctx.request.body.datasource.plus
+  const fetchSchema = ctx.request.body.fetchSchema
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
     type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE,
-    ...ctx.request.body,
+    ...ctx.request.body.datasource,
+  }
+
+  if (fetchSchema) {
+    let tables = await buildSchemaHelper(datasource)
+    datasource.entities = tables
   }
 
   const response = await db.put(datasource)
@@ -133,3 +134,14 @@ exports.query = async function (ctx) {
     ctx.throw(400, err)
   }
 }
+
+const buildSchemaHelper = async datasource => {
+  const Connector = integrations[datasource.source]
+
+  // Connect to the DB and build the schema
+  const connector = new Connector(datasource.config)
+  await connector.buildSchema(datasource._id, datasource.entities)
+  datasource.entities = connector.tables
+
+  return connector.tables
+}

packages/server/src/api/routes/tests/query.spec.js

@@ -1,6 +1,7 @@
 // mock out postgres for this
 jest.mock("pg")
 
+const { findLastKey } = require("lodash/fp")
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures
@@ -19,10 +20,10 @@ describe("/queries", () => {
   })
 
   async function createInvalidIntegration() {
-    const datasource = await config.createDatasource({
+    const datasource = await config.createDatasource({datasource: {
-      ...basicDatasource(),
+      ...basicDatasource().datasource,
       source: "INVALID_INTEGRATION",
-    })
+    }})
     const query = await config.createQuery()
     return { datasource, query }
   }
@@ -183,11 +184,14 @@ describe("/queries", () => {
     })
 
     it("should fail with invalid integration type", async () => {
-      const { query } = await createInvalidIntegration()
+      const { query, datasource } = await createInvalidIntegration()
       await request
         .post(`/api/queries/${query._id}`)
         .send({
+          datasourceId: datasource._id,
           parameters: {},
+          fields: {},
+          queryVerb: "read",
         })
         .set(config.defaultHeaders())
         .expect(400)

packages/server/src/tests/utilities/structures.js

@@ -70,10 +70,12 @@ exports.basicRole = () => {
 
 exports.basicDatasource = () => {
   return {
-    type: "datasource",
+    datasource: {
-    name: "Test",
+      type: "datasource",
-    source: "POSTGRES",
+      name: "Test",
-    config: {},
+      source: "POSTGRES",
+      config: {},
+    },
   }
 }