MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2735 from Budibase/feature/datasource-wizard 

Datasource wizard
This commit is contained in:
Peter Clement 2021-09-30 11:05:22 +01:00 committed by GitHub
parent 9afad2b744 978f08f30d
commit c84e70c919
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 350 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/bbui/src/Modal/ModalContent.svelte
View File

@ -14,6 +14,7 @@
export let showConfirmButton = true
export let showCloseIcon = true
export let onConfirm = undefined
export let onCancel = undefined
export let disabled = false
export let showDivider = true
@ -28,6 +29,14 @@
}
loading = false
}
async function close() {
loading = true
if (!onCancel || (await onCancel()) !== false) {
cancel()
}
loading = false
}
</script>
<div
@ -65,7 +74,7 @@
>
<slot name="footer" />
{#if showCancelButton}
<Button group secondary on:click={cancel}>{cancelText}</Button>
<Button group secondary on:click={close}>{cancelText}</Button>
{/if}
{#if showConfirmButton}
<Button

7
packages/builder/cypress/integration/createTable.spec.js
View File

@ -6,7 +6,7 @@ context("Create a Table", () => {
it("should create a new Table", () => {
cy.createTable("dog")
cy.wait(1000)
// Check if Table exists
cy.get(".table-title h1").should("have.text", "dog")
})
@ -36,7 +36,8 @@ context("Create a Table", () => {
it("edits a row", () => {
cy.contains("button", "Edit").click({ force: true })
cy.wait(1000)
cy.get(".spectrum-Modal input").type("Updated")
cy.get(".spectrum-Modal input").clear()
cy.get(".spectrum-Modal input").type("RoverUpdated")
cy.contains("Save").click()
cy.contains("RoverUpdated").should("have.text", "RoverUpdated")
})
@ -62,7 +63,7 @@ context("Create a Table", () => {
it("deletes a table", () => {
cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
.first()
.eq(1)
.click({ force: true })
cy.get(".spectrum-Menu > :nth-child(2)").click()
cy.contains("Delete Table").click()

22
packages/builder/cypress/support/commands.js
View File

@ -35,8 +35,11 @@ Cypress.Commands.add("createApp", name => {
.within(() => {
cy.get("input").eq(0).type(name).should("have.value", name).blur()
cy.get(".spectrum-ButtonGroup").contains("Create app").click()
cy.wait(7000)
})
.then(() => {
// Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
cy.createInitialDatasource("initialTable")
cy.expandBudibaseConnection()
cy.get(".nav-item.selected > .content").should("be.visible")
})
@ -69,11 +72,28 @@ Cypress.Commands.add("createTestTableWithData", () => {
cy.addColumn("dog", "age", "Number")
})
Cypress.Commands.add("createTable", tableName => {
Cypress.Commands.add("createInitialDatasource", tableName => {
// Enter table name
cy.get(".spectrum-Modal").within(() => {
cy.contains("Budibase DB").trigger("mouseover").click().click()
cy.wait(1000)
cy.contains("Continue").click()
})
cy.get(".spectrum-Modal").within(() => {
cy.wait(1000)
cy.get("input").first().type(tableName).blur()
cy.get(".spectrum-ButtonGroup").contains("Create").click()
})
cy.contains(tableName).should("be.visible")
})
Cypress.Commands.add("createTable", tableName => {
cy.contains("Budibase DB").click()
cy.contains("Create new table").click()
cy.get(".spectrum-Modal").within(() => {
cy.wait(1000)
cy.get("input").first().type(tableName).blur()
cy.get(".spectrum-ButtonGroup").contains("Create").click()
})

19
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte
View File

@ -1,20 +1,28 @@
<script>
import { Label, Input, Layout, Toggle } from "@budibase/bbui"
import { Label, Input, Layout, Toggle, Button } from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import { capitalise } from "helpers"
export let integration
export let schema
let addButton
</script>
<form>
<Layout gap="S">
{#each Object.keys(schema) as configKey}
{#if schema[configKey].type === "object"}
<Label>{capitalise(configKey)}</Label>
<div class="form-row ssl">
<Label>{capitalise(configKey)}</Label>
<Button secondary thin outline on:click={addButton.addEntry()}
>Add</Button
>
</div>
<KeyValueBuilder
bind:this={addButton}
defaults={schema[configKey].default}
bind:object={integration[configKey]}
noAddButton={true}
/>
{:else if schema[configKey].type === "boolean"}
<div class="form-row">
@ -42,4 +50,11 @@
grid-gap: var(--spacing-l);
align-items: center;
}
.form-row.ssl {
display: grid;
grid-template-columns: 20% 20%;
grid-gap: var(--spacing-l);
align-items: center;
}
</style>

208
packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte
View File

@ -1,74 +1,160 @@
<script>
import { goto } from "@roxi/routify"
import { datasources } from "stores/backend"
import { notifications } from "@budibase/bbui"
import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui"
import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
import { ModalContent, Modal, Body, Layout, Detail } from "@budibase/bbui"
import { onMount } from "svelte"
import ICONS from "../icons"
import api from "builderStore/api"
import { IntegrationNames } from "constants"
import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
import analytics, { Events } from "analytics"
import { getContext } from "svelte"
import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
const modalContext = getContext(Context.Modal)
export let modal
let integrations = []
let integration = {}
let internalTableModal
let externalDatasourceModal
let tableModal
let name
let error = ""
let integration
const INTERNAL = "BUDIBASE"
$: checkOpenModal(integration && integration.type === "BUDIBASE")
onMount(() => {
fetchIntegrations()
})
function checkValid(evt) {
const datasourceName = evt.target.value
if (
$datasources?.list.some(datasource => datasource.name === datasourceName)
) {
error = `Datasource with name ${datasourceName} already exists. Please choose another name.`
return
function selectIntegration(integrationType) {
const selected = integrations[integrationType]
// build the schema
const config = {}
for (let key of Object.keys(selected.datasource)) {
config[key] = selected.datasource[key].default
}
error = ""
}
function checkOpenModal(isInternal) {
if (isInternal) {
tableModal.show()
}
}
async function saveDatasource() {
const { type, plus, ...config } = integration
// Create datasource
const response = await datasources.save({
name,
source: type,
integration = {
type: integrationType,
plus: selected.plus,
config,
plus,
})
notifications.success(`Datasource ${name} created successfully.`)
analytics.captureEvent(Events.DATASOURCE.CREATED, { name, type })
schema: selected.datasource,
}
}
// Navigate to new datasource
$goto(`./datasource/${response._id}`)
function chooseNextModal() {
if (integration.type === INTERNAL) {
externalDatasourceModal.hide()
internalTableModal.show()
} else {
externalDatasourceModal.show()
}
}
async function fetchIntegrations() {
const response = await api.get("/api/integrations")
const json = await response.json()
integrations = {
[INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
...json,
}
return json
}
</script>
<Modal bind:this={tableModal} on:hide={modalContext.hide}>
<CreateTableModal bind:name />
<Modal bind:this={internalTableModal}>
<CreateTableModal />
</Modal>
<ModalContent
title="Create Datasource"
size="L"
confirmText="Create"
onConfirm={saveDatasource}
disabled={error || !name || !integration?.type}
>
<Input
data-cy="datasource-name-input"
label="Datasource Name"
on:input={checkValid}
bind:value={name}
{error}
/>
<Label>Datasource Type</Label>
<TableIntegrationMenu bind:integration />
</ModalContent>
<Modal bind:this={externalDatasourceModal}>
<DatasourceConfigModal {integration} />
</Modal>
<Modal bind:this={modal}>
<ModalContent
disabled={!Object.keys(integration).length}
title="Data"
confirmText="Continue"
showCancelButton={false}
size="M"
onConfirm={() => {
chooseNextModal()
}}
>
<Layout noPadding>
<Body size="XS"
>All apps need data. You can connect to a data source below, or add data
to your app using Budibase's built-in database.
</Body>
<div
class:selected={integration.type === INTERNAL}
on:click={() => selectIntegration(INTERNAL)}
class="item hoverable"
>
<div class="item-body">
<svelte:component this={ICONS.BUDIBASE} height="18" width="18" />
<span class="icon-spacing"> <Body size="S">Budibase DB</Body></span>
</div>
</div>
</Layout>
<Layout gap="XS" noPadding>
<div class="title-spacing">
<Detail size="S">Connect to data source</Detail>
</div>
<div class="item-list">
{#each Object.entries(integrations).filter(([key]) => key !== INTERNAL) as [integrationType, schema]}
<div
class:selected={integration.type === integrationType}
on:click={() => selectIntegration(integrationType)}
class="item hoverable"
>
<div class="item-body">
<svelte:component
this={ICONS[integrationType]}
height="18"
width="18"
/>
<span class="icon-spacing">
<Body size="S"
>{schema.name || IntegrationNames[integrationType]}</Body
></span
>
</div>
</div>
{/each}
</div>
</Layout>
</ModalContent>
</Modal>
<style>
.icon-spacing {
margin-left: var(--spacing-m);
}
.item-body {
display: flex;
margin-left: var(--spacing-m);
}
.item-list {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
grid-gap: var(--spectrum-alias-grid-baseline);
}
.item {
cursor: pointer;
display: grid;
grid-gap: var(--spectrum-alias-grid-margin-xsmall);
padding: var(--spectrum-alias-item-padding-s);
background: var(--spectrum-alias-background-color-secondary);
transition: 0.3s all;
border: solid var(--spectrum-alias-border-color);
border-radius: 5px;
box-sizing: border-box;
border-width: 2px;
}
.selected {
background: var(--spectrum-alias-background-color-tertiary);
}
.item:hover,
.selected {
background: var(--spectrum-alias-background-color-tertiary);
}
</style>

72
packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte Normal file
View File

@ -0,0 +1,72 @@
<script>
import { goto } from "@roxi/routify"
import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
import analytics, { Events } from "analytics"
import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
import { datasources } from "stores/backend"
import { IntegrationNames } from "constants"
export let integration
function prepareData() {
let datasource = {}
let existingTypeCount = $datasources.list.filter(
ds => ds.source == integration.type
).length
let baseName = IntegrationNames[integration.type]
let name =
existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
datasource.type = "datasource"
datasource.source = integration.type
datasource.config = integration.config
datasource.name = name
datasource.plus = integration.plus
return datasource
}
async function saveDatasource() {
const datasource = prepareData()
try {
// Create datasource
const resp = await datasources.save(datasource, datasource.plus)
await datasources.select(resp._id)
$goto(`./datasource/${resp._id}`)
notifications.success(`Datasource updated successfully.`)
analytics.captureEvent(Events.DATASOURCE.CREATED, {
name: resp.name,
source: resp.source,
})
return true
} catch (err) {
notifications.error(`Error saving datasource: ${err}`)
return false
}
}
</script>
<ModalContent
title={`Connect to ${IntegrationNames[integration.type]}`}
onConfirm={() => saveDatasource()}
confirmText={integration.plus
? "Fetch tables from database"
: "Save and continue to query"}
cancelText="Back"
size="M"
>
<Layout noPadding>
<Body size="XS"
>Connect your database to Budibase using the config below.
</Body>
</Layout>
<IntegrationConfigForm
schema={integration.schema}
bind:integration={integration.config}
/>
</ModalContent>
<style>
</style>

5
packages/builder/src/components/integration/KeyValueBuilder.svelte
View File

@ -4,6 +4,7 @@
export let defaults
export let object = defaults || {}
export let readOnly
export let noAddButton
let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
@ -12,7 +13,7 @@
{}
)
function addEntry() {
export function addEntry() {
fields = [...fields, {}]
}
@ -32,7 +33,7 @@
{/if}
{/each}
</div>
{#if !readOnly}
{#if !readOnly && !noAddButton}
<div>
<Button secondary thin outline on:click={addEntry}>Add</Button>
</div>

14
packages/builder/src/constants/index.js
View File

@ -15,6 +15,20 @@ export const AppStatus = {
DEPLOYED: "published",
}
export const IntegrationNames = {
POSTGRES: "PostgreSQL",
MONGODB: "MongoDB",
COUCHDB: "CouchDB",
S3: "S3",
MYSQL: "MySQL",
REST: "REST",
DYNAMODB: "DynamoDB",
ELASTICSEARCH: "ElasticSearch",
SQL_SERVER: "SQL Server",
AIRTABLE: "Airtable",
ARANGODB: "ArangoDB",
}
// fields on the user table that cannot be edited
export const UNEDITABLE_USER_FIELDS = [
"email",

8
packages/builder/src/pages/builder/app/[application]/data/_layout.svelte
View File

@ -1,12 +1,14 @@
<script>
import { goto, params } from "@roxi/routify"
import { Icon, Modal, Tabs, Tab } from "@budibase/bbui"
import { Icon, Tabs, Tab } from "@budibase/bbui"
import { BUDIBASE_INTERNAL_DB } from "constants"
import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
let selected = "Sources"
let modal
$: isExternal =
$params.selectedDatasource &&
$params.selectedDatasource !== BUDIBASE_INTERNAL_DB
@ -23,9 +25,7 @@
<Tab title="Sources">
<div class="tab-content-padding">
<DatasourceNavigator />
<Modal bind:this={modal}>
<CreateDatasourceModal />
</Modal>
<CreateDatasourceModal bind:modal />
</div>
</Tab>
</Tabs>

18
packages/builder/src/pages/builder/app/[application]/data/index.svelte
View File

@ -1,6 +1,22 @@
<script>
import { goto } from "@roxi/routify"
$goto("./table")
import { onMount } from "svelte"
import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
import { datasources } from "stores/backend"
let modal
$: setupComplete =
$datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 ||
$datasources.list.length > 1
onMount(() => {
if (!setupComplete) {
modal.show()
} else {
$goto("./table")
}
})
</script>
<CreateDatasourceModal bind:modal />
<!-- routify:options index=false -->

7
packages/builder/src/stores/backend/datasources.js
View File

@ -67,7 +67,7 @@ export function createDatasourcesStore() {
})
return json
},
save: async datasource => {
save: async (datasource, fetchSchema = false) => {
let response
if (datasource._id) {
response = await api.put(
@ -75,7 +75,10 @@ export function createDatasourcesStore() {
datasource
)
} else {
response = await api.post("/api/datasources", datasource)
response = await api.post("/api/datasources", {
datasource: datasource,
fetchSchema,
})
}
const json = await response.json()

32
packages/server/src/api/controllers/datasource.js
View File

@ -41,15 +41,10 @@ exports.fetch = async function (ctx) {
exports.buildSchemaFromDb = async function (ctx) {
const db = new CouchDB(ctx.appId)
const datasourceId = ctx.params.datasourceId
const datasource = await db.get(datasourceId)
const datasource = await db.get(ctx.params.datasourceId)
const Connector = integrations[datasource.source]
// Connect to the DB and build the schema
const connector = new Connector(datasource.config)
await connector.buildSchema(datasource._id, datasource.entities)
datasource.entities = connector.tables
const tables = await buildSchemaHelper(datasource)
datasource.entities = tables
const response = await db.put(datasource)
datasource._rev = response.rev
@ -81,12 +76,18 @@ exports.update = async function (ctx) {
exports.save = async function (ctx) {
const db = new CouchDB(ctx.appId)
const plus = ctx.request.body.plus
const plus = ctx.request.body.datasource.plus
const fetchSchema = ctx.request.body.fetchSchema
const datasource = {
_id: generateDatasourceID({ plus }),
type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE,
...ctx.request.body,
...ctx.request.body.datasource,
}
if (fetchSchema) {
let tables = await buildSchemaHelper(datasource)
datasource.entities = tables
}
const response = await db.put(datasource)
@ -133,3 +134,14 @@ exports.query = async function (ctx) {
ctx.throw(400, err)
}
}
const buildSchemaHelper = async datasource => {
const Connector = integrations[datasource.source]
// Connect to the DB and build the schema
const connector = new Connector(datasource.config)
await connector.buildSchema(datasource._id, datasource.entities)
datasource.entities = connector.tables
return connector.tables
}

12
packages/server/src/api/routes/tests/query.spec.js
View File

@ -1,6 +1,7 @@
// mock out postgres for this
jest.mock("pg")
const { findLastKey } = require("lodash/fp")
const setup = require("./utilities")
const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
const { basicQuery, basicDatasource } = setup.structures
@ -19,10 +20,10 @@ describe("/queries", () => {
})
async function createInvalidIntegration() {
const datasource = await config.createDatasource({
...basicDatasource(),
const datasource = await config.createDatasource({datasource: {
...basicDatasource().datasource,
source: "INVALID_INTEGRATION",
})
}})
const query = await config.createQuery()
return { datasource, query }
}
@ -183,11 +184,14 @@ describe("/queries", () => {
})
it("should fail with invalid integration type", async () => {
const { query } = await createInvalidIntegration()
const { query, datasource } = await createInvalidIntegration()
await request
.post(`/api/queries/${query._id}`)
.send({
datasourceId: datasource._id,
parameters: {},
fields: {},
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect(400)

10
packages/server/src/tests/utilities/structures.js
View File

@ -70,10 +70,12 @@ exports.basicRole = () => {
exports.basicDatasource = () => {
return {
type: "datasource",
name: "Test",
source: "POSTGRES",
config: {},
datasource: {
type: "datasource",
name: "Test",
source: "POSTGRES",
config: {},
},
}
}