MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2735 from Budibase/feature/datasource-wizard 

Datasource wizard
This commit is contained in:
Peter Clement 2021-09-30 11:05:22 +01:00 committed by GitHub
parent 4d873a00bc 86428f3998
commit afa3cd53f6
14 changed files with 350 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/bbui/src/Modal/ModalContent.svelte
View File

@ -14,6 +14,7 @@
export let showConfirmButton = true
export let showCloseIcon = true
export let onConfirm = undefined
export let onCancel = undefined
export let disabled = false
export let showDivider = true
@ -28,6 +29,14 @@
}
loading = false
}
async function close() {
loading = true
if (!onCancel || (await onCancel()) !== false) {
cancel()
}
loading = false
}
</script>
<div
@ -65,7 +74,7 @@
>
<slot name="footer" />
{#if showCancelButton}
<Button group secondary on:click={cancel}>{cancelText}</Button>
<Button group secondary on:click={close}>{cancelText}</Button>
{/if}
{#if showConfirmButton}
<Button

7
packages/builder/cypress/integration/createTable.spec.js
View File

@ -6,7 +6,7 @@ context("Create a Table", () => {
it("should create a new Table", () => {
cy.createTable("dog")
cy.wait(1000)
// Check if Table exists
cy.get(".table-title h1").should("have.text", "dog")
})
@ -36,7 +36,8 @@ context("Create a Table", () => {
it("edits a row", () => {
cy.contains("button", "Edit").click({ force: true })
cy.wait(1000)
cy.get(".spectrum-Modal input").type("Updated")
cy.get(".spectrum-Modal input").clear()
cy.get(".spectrum-Modal input").type("RoverUpdated")
cy.contains("Save").click()
cy.contains("RoverUpdated").should("have.text", "RoverUpdated")
})
@ -62,7 +63,7 @@ context("Create a Table", () => {
it("deletes a table", () => {
cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
.first()
.eq(1)
.click({ force: true })
cy.get(".spectrum-Menu > :nth-child(2)").click()
cy.contains("Delete Table").click()

22
packages/builder/cypress/support/commands.js
View File

@ -35,8 +35,11 @@ Cypress.Commands.add("createApp", name => {
.within(() => {
cy.get("input").eq(0).type(name).should("have.value", name).blur()
cy.get(".spectrum-ButtonGroup").contains("Create app").click()
cy.wait(7000)
})
.then(() => {
// Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
cy.createInitialDatasource("initialTable")
cy.expandBudibaseConnection()
cy.get(".nav-item.selected > .content").should("be.visible")
})
@ -69,11 +72,28 @@ Cypress.Commands.add("createTestTableWithData", () => {
cy.addColumn("dog", "age", "Number")
})
Cypress.Commands.add("createTable", tableName => {
Cypress.Commands.add("createInitialDatasource", tableName => {
// Enter table name
cy.get(".spectrum-Modal").within(() => {
cy.contains("Budibase DB").trigger("mouseover").click().click()
cy.wait(1000)
cy.contains("Continue").click()
})
cy.get(".spectrum-Modal").within(() => {
cy.wait(1000)
cy.get("input").first().type(tableName).blur()
cy.get(".spectrum-ButtonGroup").contains("Create").click()
})
cy.contains(tableName).should("be.visible")
})
Cypress.Commands.add("createTable", tableName => {
cy.contains("Budibase DB").click()
cy.contains("Create new table").click()
cy.get(".spectrum-Modal").within(() => {
cy.wait(1000)
cy.get("input").first().type(tableName).blur()
cy.get(".spectrum-ButtonGroup").contains("Create").click()
})

17
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte
View File

@ -1,20 +1,28 @@
<script>
import { Label, Input, Layout, Toggle } from "@budibase/bbui"
import { Label, Input, Layout, Toggle, Button } from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import { capitalise } from "helpers"
export let integration
export let schema
let addButton
</script>
<form>
<Layout gap="S">
{#each Object.keys(schema) as configKey}
{#if schema[configKey].type === "object"}
<div class="form-row ssl">
<Label>{capitalise(configKey)}</Label>
<Button secondary thin outline on:click={addButton.addEntry()}
>Add</Button
>
</div>
<KeyValueBuilder
bind:this={addButton}
defaults={schema[configKey].default}
bind:object={integration[configKey]}
noAddButton={true}
/>
{:else if schema[configKey].type === "boolean"}
<div class="form-row">
@ -42,4 +50,11 @@
grid-gap: var(--spacing-l);
align-items: center;
}
.form-row.ssl {
display: grid;
grid-template-columns: 20% 20%;
grid-gap: var(--spacing-l);
align-items: center;
}
</style>

210
packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte
View File

@ -1,74 +1,160 @@
<script>
import { goto } from "@roxi/routify"
import { datasources } from "stores/backend"
import { notifications } from "@budibase/bbui"
import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui"
import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
import { ModalContent, Modal, Body, Layout, Detail } from "@budibase/bbui"
import { onMount } from "svelte"
import ICONS from "../icons"
import api from "builderStore/api"
import { IntegrationNames } from "constants"
import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
import analytics, { Events } from "analytics"
import { getContext } from "svelte"
import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
const modalContext = getContext(Context.Modal)
export let modal
let integrations = []
let integration = {}
let internalTableModal
let externalDatasourceModal
let tableModal
let name
let error = ""
let integration
const INTERNAL = "BUDIBASE"
$: checkOpenModal(integration && integration.type === "BUDIBASE")
function checkValid(evt) {
const datasourceName = evt.target.value
if (
$datasources?.list.some(datasource => datasource.name === datasourceName)
) {
error = `Datasource with name ${datasourceName} already exists. Please choose another name.`
return
}
error = ""
}
function checkOpenModal(isInternal) {
if (isInternal) {
tableModal.show()
}
}
async function saveDatasource() {
const { type, plus, ...config } = integration
// Create datasource
const response = await datasources.save({
name,
source: type,
config,
plus,
onMount(() => {
fetchIntegrations()
})
notifications.success(`Datasource ${name} created successfully.`)
analytics.captureEvent(Events.DATASOURCE.CREATED, { name, type })
// Navigate to new datasource
$goto(`./datasource/${response._id}`)
function selectIntegration(integrationType) {
const selected = integrations[integrationType]
// build the schema
const config = {}
for (let key of Object.keys(selected.datasource)) {
config[key] = selected.datasource[key].default
}
integration = {
type: integrationType,
plus: selected.plus,
config,
schema: selected.datasource,
}
}
function chooseNextModal() {
if (integration.type === INTERNAL) {
externalDatasourceModal.hide()
internalTableModal.show()
} else {
externalDatasourceModal.show()
}
}
async function fetchIntegrations() {
const response = await api.get("/api/integrations")
const json = await response.json()
integrations = {
[INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
...json,
}
return json
}
</script>
<Modal bind:this={tableModal} on:hide={modalContext.hide}>
<CreateTableModal bind:name />
<Modal bind:this={internalTableModal}>
<CreateTableModal />
</Modal>
<ModalContent
title="Create Datasource"
size="L"
confirmText="Create"
onConfirm={saveDatasource}
disabled={error || !name || !integration?.type}
>
<Input
data-cy="datasource-name-input"
label="Datasource Name"
on:input={checkValid}
bind:value={name}
{error}
<Modal bind:this={externalDatasourceModal}>
<DatasourceConfigModal {integration} />
</Modal>
<Modal bind:this={modal}>
<ModalContent
disabled={!Object.keys(integration).length}
title="Data"
confirmText="Continue"
showCancelButton={false}
size="M"
onConfirm={() => {
chooseNextModal()
}}
>
<Layout noPadding>
<Body size="XS"
>All apps need data. You can connect to a data source below, or add data
to your app using Budibase's built-in database.
</Body>
<div
class:selected={integration.type === INTERNAL}
on:click={() => selectIntegration(INTERNAL)}
class="item hoverable"
>
<div class="item-body">
<svelte:component this={ICONS.BUDIBASE} height="18" width="18" />
<span class="icon-spacing"> <Body size="S">Budibase DB</Body></span>
</div>
</div>
</Layout>
<Layout gap="XS" noPadding>
<div class="title-spacing">
<Detail size="S">Connect to data source</Detail>
</div>
<div class="item-list">
{#each Object.entries(integrations).filter(([key]) => key !== INTERNAL) as [integrationType, schema]}
<div
class:selected={integration.type === integrationType}
on:click={() => selectIntegration(integrationType)}
class="item hoverable"
>
<div class="item-body">
<svelte:component
this={ICONS[integrationType]}
height="18"
width="18"
/>
<Label>Datasource Type</Label>
<TableIntegrationMenu bind:integration />
</ModalContent>
<span class="icon-spacing">
<Body size="S"
>{schema.name || IntegrationNames[integrationType]}</Body
></span
>
</div>
</div>
{/each}
</div>
</Layout>
</ModalContent>
</Modal>
<style>
.icon-spacing {
margin-left: var(--spacing-m);
}
.item-body {
display: flex;
margin-left: var(--spacing-m);
}
.item-list {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
grid-gap: var(--spectrum-alias-grid-baseline);
}
.item {
cursor: pointer;
display: grid;
grid-gap: var(--spectrum-alias-grid-margin-xsmall);
padding: var(--spectrum-alias-item-padding-s);
background: var(--spectrum-alias-background-color-secondary);
transition: 0.3s all;
border: solid var(--spectrum-alias-border-color);
border-radius: 5px;
box-sizing: border-box;
border-width: 2px;
}
.selected {
background: var(--spectrum-alias-background-color-tertiary);
}
.item:hover,
.selected {
background: var(--spectrum-alias-background-color-tertiary);
}
</style>

72
packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte Normal file
View File

@ -0,0 +1,72 @@
<script>
import { goto } from "@roxi/routify"
import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
import analytics, { Events } from "analytics"
import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
import { datasources } from "stores/backend"
import { IntegrationNames } from "constants"
export let integration
function prepareData() {
let datasource = {}
let existingTypeCount = $datasources.list.filter(
ds => ds.source == integration.type
).length
let baseName = IntegrationNames[integration.type]
let name =
existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
datasource.type = "datasource"
datasource.source = integration.type
datasource.config = integration.config
datasource.name = name
datasource.plus = integration.plus
return datasource
}
async function saveDatasource() {
const datasource = prepareData()
try {
// Create datasource
const resp = await datasources.save(datasource, datasource.plus)
await datasources.select(resp._id)
$goto(`./datasource/${resp._id}`)
notifications.success(`Datasource updated successfully.`)
analytics.captureEvent(Events.DATASOURCE.CREATED, {
name: resp.name,
source: resp.source,
})
return true
} catch (err) {
notifications.error(`Error saving datasource: ${err}`)
return false
}
}
</script>
<ModalContent
title={`Connect to ${IntegrationNames[integration.type]}`}
onConfirm={() => saveDatasource()}
confirmText={integration.plus
? "Fetch tables from database"
: "Save and continue to query"}
cancelText="Back"
size="M"
>
<Layout noPadding>
<Body size="XS"
>Connect your database to Budibase using the config below.
</Body>
</Layout>
<IntegrationConfigForm
schema={integration.schema}
bind:integration={integration.config}
/>
</ModalContent>
<style>
</style>

5
packages/builder/src/components/integration/KeyValueBuilder.svelte
View File

@ -4,6 +4,7 @@
export let defaults
export let object = defaults || {}
export let readOnly
export let noAddButton
let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
@ -12,7 +13,7 @@
{}
)
function addEntry() {
export function addEntry() {
fields = [...fields, {}]
}
@ -32,7 +33,7 @@
{/if}
{/each}
</div>
{#if !readOnly}
{#if !readOnly && !noAddButton}
<div>
<Button secondary thin outline on:click={addEntry}>Add</Button>
</div>

14
packages/builder/src/constants/index.js
View File

@ -15,6 +15,20 @@ export const AppStatus = {
DEPLOYED: "published",
}
export const IntegrationNames = {
POSTGRES: "PostgreSQL",
MONGODB: "MongoDB",
COUCHDB: "CouchDB",
S3: "S3",
MYSQL: "MySQL",
REST: "REST",
DYNAMODB: "DynamoDB",
ELASTICSEARCH: "ElasticSearch",
SQL_SERVER: "SQL Server",
AIRTABLE: "Airtable",
ARANGODB: "ArangoDB",
}
// fields on the user table that cannot be edited
export const UNEDITABLE_USER_FIELDS = [
"email",

8
packages/builder/src/pages/builder/app/[application]/data/_layout.svelte
View File

@ -1,12 +1,14 @@
<script>
import { goto, params } from "@roxi/routify"
import { Icon, Modal, Tabs, Tab } from "@budibase/bbui"
import { Icon, Tabs, Tab } from "@budibase/bbui"
import { BUDIBASE_INTERNAL_DB } from "constants"
import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
let selected = "Sources"
let modal
$: isExternal =
$params.selectedDatasource &&
$params.selectedDatasource !== BUDIBASE_INTERNAL_DB
@ -23,9 +25,7 @@
<Tab title="Sources">
<div class="tab-content-padding">
<DatasourceNavigator />
<Modal bind:this={modal}>
<CreateDatasourceModal />
</Modal>
<CreateDatasourceModal bind:modal />
</div>
</Tab>
</Tabs>

16
packages/builder/src/pages/builder/app/[application]/data/index.svelte
View File

@ -1,6 +1,22 @@
<script>
import { goto } from "@roxi/routify"
import { onMount } from "svelte"
import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
import { datasources } from "stores/backend"
let modal
$: setupComplete =
$datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 ||
$datasources.list.length > 1
onMount(() => {
if (!setupComplete) {
modal.show()
} else {
$goto("./table")
}
})
</script>
<CreateDatasourceModal bind:modal />
<!-- routify:options index=false -->

7
packages/builder/src/stores/backend/datasources.js
View File

@ -67,7 +67,7 @@ export function createDatasourcesStore() {
})
return json
},
save: async datasource => {
save: async (datasource, fetchSchema = false) => {
let response
if (datasource._id) {
response = await api.put(
@ -75,7 +75,10 @@ export function createDatasourcesStore() {
datasource
)
} else {
response = await api.post("/api/datasources", datasource)
response = await api.post("/api/datasources", {
datasource: datasource,
fetchSchema,
})
}
const json = await response.json()

32
packages/server/src/api/controllers/datasource.js
View File

@ -41,15 +41,10 @@ exports.fetch = async function (ctx) {
exports.buildSchemaFromDb = async function (ctx) {
const db = new CouchDB(ctx.appId)
const datasourceId = ctx.params.datasourceId
const datasource = await db.get(datasourceId)
const datasource = await db.get(ctx.params.datasourceId)
const Connector = integrations[datasource.source]
// Connect to the DB and build the schema
const connector = new Connector(datasource.config)
await connector.buildSchema(datasource._id, datasource.entities)
datasource.entities = connector.tables
const tables = await buildSchemaHelper(datasource)
datasource.entities = tables
const response = await db.put(datasource)
datasource._rev = response.rev
@ -81,12 +76,18 @@ exports.update = async function (ctx) {
exports.save = async function (ctx) {
const db = new CouchDB(ctx.appId)
const plus = ctx.request.body.plus
const plus = ctx.request.body.datasource.plus
const fetchSchema = ctx.request.body.fetchSchema
const datasource = {
_id: generateDatasourceID({ plus }),
type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE,
...ctx.request.body,
...ctx.request.body.datasource,
}
if (fetchSchema) {
let tables = await buildSchemaHelper(datasource)
datasource.entities = tables
}
const response = await db.put(datasource)
@ -133,3 +134,14 @@ exports.query = async function (ctx) {
ctx.throw(400, err)
}
}
const buildSchemaHelper = async datasource => {
const Connector = integrations[datasource.source]
// Connect to the DB and build the schema
const connector = new Connector(datasource.config)
await connector.buildSchema(datasource._id, datasource.entities)
datasource.entities = connector.tables
return connector.tables
}

12
packages/server/src/api/routes/tests/query.spec.js
View File

@ -1,6 +1,7 @@
// mock out postgres for this
jest.mock("pg")
const { findLastKey } = require("lodash/fp")
const setup = require("./utilities")
const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
const { basicQuery, basicDatasource } = setup.structures
@ -19,10 +20,10 @@ describe("/queries", () => {
})
async function createInvalidIntegration() {
const datasource = await config.createDatasource({
...basicDatasource(),
const datasource = await config.createDatasource({datasource: {
...basicDatasource().datasource,
source: "INVALID_INTEGRATION",
})
}})
const query = await config.createQuery()
return { datasource, query }
}
@ -183,11 +184,14 @@ describe("/queries", () => {
})
it("should fail with invalid integration type", async () => {
const { query } = await createInvalidIntegration()
const { query, datasource } = await createInvalidIntegration()
await request
.post(`/api/queries/${query._id}`)
.send({
datasourceId: datasource._id,
parameters: {},
fields: {},
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect(400)

2
packages/server/src/tests/utilities/structures.js
View File

@ -70,10 +70,12 @@ exports.basicRole = () => {
exports.basicDatasource = () => {
return {
datasource: {
type: "datasource",
name: "Test",
source: "POSTGRES",
config: {},
},
}
}