From b5ff5a7f7647ee96544054e1d336f2f7984f0f6e Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 23 Sep 2021 16:06:57 +0100 Subject: [PATCH 01/48] Remove duplicate routes which are never used --- .../table/[selectedTable]/_layout.svelte | 13 ------------ .../table/[selectedTable]/index.svelte | 16 -------------- .../[selectedField]/index.svelte | 10 --------- .../relationship/[selectedRow]/index.svelte | 6 ------ .../[selectedTable]/relationship/index.svelte | 6 ------ .../bb_internal/table/_layout.svelte | 19 ----------------- .../datasource/bb_internal/table/index.svelte | 21 ------------------- 7 files changed, 91 deletions(-) delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/_layout.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/[selectedField]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/_layout.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/index.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/_layout.svelte deleted file mode 100644 index 14f6303e5f..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/_layout.svelte +++ /dev/null @@ -1,13 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/index.svelte deleted file mode 100644 index a68c0dc651..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/index.svelte +++ /dev/null @@ -1,16 +0,0 @@ - - -{#if $database?._id && $tables?.selected?.name} - -{:else}Create your first table to start building{/if} - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/[selectedField]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/[selectedField]/index.svelte deleted file mode 100644 index eddb5ab598..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/[selectedField]/index.svelte +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/index.svelte deleted file mode 100644 index 8e195ddb12..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/index.svelte +++ /dev/null @@ -1,6 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/index.svelte deleted file mode 100644 index 7d081b6976..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/index.svelte +++ /dev/null @@ -1,6 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/_layout.svelte deleted file mode 100644 index f957355c5c..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/_layout.svelte +++ /dev/null @@ -1,19 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/index.svelte deleted file mode 100644 index 6d61614145..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/index.svelte +++ /dev/null @@ -1,21 +0,0 @@ - - -{#if $tables.list.length === 0} - Create your first table to start building -{:else}Select a table to edit{/if} - - From 05cfd87613799c471318c738e9efab913af190c8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 23 Sep 2021 16:08:09 +0100 Subject: [PATCH 02/48] Broadcast a sort event from BBUI table, and add prop to disable table sorting of data --- packages/bbui/src/Table/Table.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 11284b8917..63f20ac1ff 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -27,6 +27,7 @@ export let selectedRows = [] export let editColumnTitle = "Edit" export let customRenderers = [] + export let disableSorting = false const dispatch = createEventDispatcher() @@ -107,7 +108,7 @@ } const sortRows = (rows, sortColumn, sortOrder) => { - if (!sortColumn || !sortOrder) { + if (!sortColumn || !sortOrder || disableSorting) { return rows } return rows.slice().sort((a, b) => { @@ -131,6 +132,7 @@ sortColumn = fieldSchema.name sortOrder = "Descending" } + dispatch("sort", { column: sortColumn, order: sortOrder }) } const getDisplayName = schema => { From 7ac4144903661cac7e9853b7373827f673c67159 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 23 Sep 2021 16:08:47 +0100 Subject: [PATCH 03/48] Add new core implementation of fetching paginated table data and enable pagination in backend UI for tables --- .../backend/DataTable/DataTable.svelte | 121 ++++++----- .../components/backend/DataTable/Table.svelte | 76 +++---- .../builder/src/helpers/fetchTableData.js | 192 ++++++++++++++++++ 3 files changed, 301 insertions(+), 88 deletions(-) create mode 100644 packages/builder/src/helpers/fetchTableData.js diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 3293c694b6..88f754b254 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -1,6 +1,5 @@ - - {#if isInternal} - - {/if} - {#if schema && Object.keys(schema).length > 0} - {#if !isUsersTable} - - {/if} +
+
{#if isInternal} - + {/if} - - {#if isUsersTable} - + {#if schema && Object.keys(schema).length > 0} + {#if !isUsersTable} + + {/if} + {#if isInternal} + + {/if} + + {#if isUsersTable} + + {/if} + + + {/if} - - - - {/if} -
+ + + + + diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte index fcb17a774d..38d580e273 100644 --- a/packages/builder/src/components/backend/DataTable/Table.svelte +++ b/packages/builder/src/components/backend/DataTable/Table.svelte @@ -1,8 +1,7 @@ -
-
- {#if title} - {title} - {/if} - {#if loading} -
- -
- {/if} + +
+
+ {#if title} + {title} + {/if} + {#if loading} +
+ +
+ {/if} +
+
+ + {#if !isUsersTable && selectedRows.length > 0} + + {/if} +
-
- - {#if !isUsersTable && selectedRows.length > 0} - - {/if} -
-
-{#key tableId} - editColumn(e.detail)} - on:editrow={e => editRow(e.detail)} - on:clickrelationship={e => selectRelationship(e.detail)} - /> -{/key} + {#key tableId} +
editColumn(e.detail)} + on:editrow={e => editRow(e.detail)} + on:clickrelationship={e => selectRelationship(e.detail)} + on:sort + /> + {/key} + diff --git a/packages/builder/src/helpers/fetchTableData.js b/packages/builder/src/helpers/fetchTableData.js new file mode 100644 index 0000000000..ff7d61519f --- /dev/null +++ b/packages/builder/src/helpers/fetchTableData.js @@ -0,0 +1,192 @@ +import { writable, derived, get } from "svelte/store" +import * as API from "builderStore/api" +import { buildLuceneQuery } from "../../../client/src/utils/lucene" + +const defaultOptions = { + tableId: null, + filter: null, + limit: 10, + sortColumn: null, + sortOrder: "ascending", + paginate: true, + schema: null, +} + +export const fetchTableData = opts => { + // Save option set so we can override it later rather than relying on params + let options = { + ...defaultOptions, + ...opts, + } + + // Local non-observable state + let query + let sortType + + // Local observable state + const store = writable({ + rows: [], + schema: null, + loading: false, + loaded: false, + bookmarks: [], + pageNumber: 0, + }) + + // Derive certain properties to return + const derivedStore = derived(store, $store => { + return { + ...$store, + hasNextPage: $store.bookmarks[$store.pageNumber + 1] != null, + hasPrevPage: $store.pageNumber > 0, + } + }) + + const fetchPage = async bookmark => { + const { tableId, limit, sortColumn, sortOrder, paginate } = options + store.update($store => ({ ...$store, loading: true })) + const res = await API.post(`/api/${options.tableId}/search`, { + tableId, + query, + limit, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + sortType, + paginate, + bookmark, + }) + store.update($store => ({ ...$store, loading: false, loaded: true })) + return await res.json() + } + + // Fetches a fresh set of results from the server + const fetchData = async () => { + const { tableId, schema, sortColumn, filter } = options + + // Ensure table ID exists + if (!tableId) { + return + } + + // Get and enrich schema. + // Ensure there are "name" properties for all fields and that field schema + // are objects + let enrichedSchema = schema + if (!enrichedSchema) { + const definition = await API.get(`/api/tables/${tableId}`) + enrichedSchema = definition?.schema ?? null + } + if (enrichedSchema) { + Object.entries(schema).forEach(([fieldName, fieldSchema]) => { + if (typeof fieldSchema === "string") { + enrichedSchema[fieldName] = { + type: fieldSchema, + name: fieldName, + } + } else { + enrichedSchema[fieldName] = { + ...fieldSchema, + name: fieldName, + } + } + }) + + // Save fixed schema so we can provide it later + options.schema = enrichedSchema + } + + // Ensure schema exists + if (!schema) { + return + } + store.update($store => ({ ...$store, schema })) + + // Work out what sort type to use + if (!sortColumn || !schema[sortColumn]) { + sortType = "string" + } + const type = schema?.[sortColumn]?.type + sortType = type === "number" ? "number" : "string" + + // Build the lucene query + query = buildLuceneQuery(filter) + + // Actually fetch data + const page = await fetchPage() + store.update($store => ({ + ...$store, + loading: false, + loaded: true, + pageNumber: 0, + rows: page.rows, + bookmarks: page.hasNextPage ? [null, page.bookmark] : [null], + })) + } + + // Fetches the next page of data + const nextPage = async () => { + const state = get(derivedStore) + if (!options.paginate || !state.hasNextPage) { + return + } + + // Fetch next page + const page = await fetchPage(state.bookmarks[state.pageNumber + 1]) + + // Update state + store.update($store => { + let { bookmarks, pageNumber } = $store + if (page.hasNextPage) { + bookmarks[pageNumber + 2] = page.bookmark + } + return { + ...$store, + pageNumber: pageNumber + 1, + rows: page.rows, + bookmarks, + } + }) + } + + // Fetches the previous page of data + const prevPage = async () => { + const state = get(derivedStore) + if (!options.paginate || !state.hasPrevPage) { + return + } + + // Fetch previous page + const page = await fetchPage(state.bookmarks[state.pageNumber - 1]) + + // Update state + store.update($store => { + return { + ...$store, + pageNumber: $store.pageNumber - 1, + rows: page.rows, + } + }) + } + + // Resets the data set and updates options + const update = async newOptions => { + if (newOptions) { + options = { + ...options, + ...newOptions, + } + } + await fetchData() + } + + // Initially fetch data but don't bother waiting for the result + fetchData() + + // Return our derived store which will be updated over time + return { + subscribe: derivedStore.subscribe, + nextPage, + prevPage, + update, + } +} From 890749ee3ea5142540e0607853e31077eff473f8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 23 Sep 2021 17:04:12 +0100 Subject: [PATCH 04/48] Only refresh tables when table ID changes --- .../backend/DataTable/DataTable.svelte | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 88f754b254..e822692da5 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -13,22 +13,24 @@ import { fetchTableData } from "helpers/fetchTableData" import { Pagination } from "@budibase/bbui" - let hideAutocolumns = true const data = fetchTableData() + let hideAutocolumns = true $: isUsersTable = $tables.selected?._id === TableNames.USERS $: title = $tables.selected?.name $: schema = $tables.selected?.schema $: type = $tables.selected?.type $: isInternal = type !== "external" + $: fetchTable($tables.selected?._id) - // Fetch data whenever table changes - $: data.update({ - tableId: $tables.selected?._id, - schema, - limit: 10, - paginate: true, - }) + const fetchTable = tableId => { + data.update({ + tableId, + schema, + limit: 10, + paginate: true, + }) + } // Fetch data whenever sorting option changes const onSort = e => { From 39192e4e7f71df44dd01d76475ded1980aa74424 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 23 Sep 2021 21:47:22 +0100 Subject: [PATCH 05/48] add base wizard for datasources --- .../modals/CreateDatasourceModal.svelte | 207 ++++++++++++------ .../modals/DatasourceDetailsModal.svelte | 51 +++++ packages/builder/src/constants/index.js | 14 ++ .../app/[application]/data/_layout.svelte | 8 +- .../app/[application]/data/index.svelte | 11 +- 5 files changed, 224 insertions(+), 67 deletions(-) create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index 9cdd893230..bf8585d401 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -1,74 +1,159 @@ - - + + - - - - - + + + + + + + { + chooseNextModal() + }} + > + All apps need data. You can connect to a data source below, or add data + to your app using Budibase's built-in database - it's simple! + + + +
selectIntegration(INTERNAL)} + class="item hoverable" + > +
+ + + Budibase DB (no prior data required) +
+
+ + +
+ {#each Object.entries(integrations).filter(([key]) => key !== INTERNAL) as [integrationType, schema]} +
selectIntegration(integrationType)} + class="item hoverable" + > +
+ + + + {schema.name || IntegrationNames[integrationType]} +
+
+ {/each} +
+
+
+
+ + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte new file mode 100644 index 0000000000..dcdd2da497 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte @@ -0,0 +1,51 @@ + + + saveDatasource()} + confirmText="Continue" + cancelText="Start from scratch" + size="M" +> + + + + diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.js index a892eb2129..c0d283b0ea 100644 --- a/packages/builder/src/constants/index.js +++ b/packages/builder/src/constants/index.js @@ -15,6 +15,20 @@ export const AppStatus = { DEPLOYED: "published", } +export const IntegrationNames = { + POSTGRES: "PostgreSQL", + MONGODB: "MongoDB", + COUCHDB: "CouchDB", + S3: "S3", + MYSQL: "MySQL", + REST: "REST", + DYNAMODB: "DynamoDB", + ELASTICSEARCH: "ElasticSearch", + SQL_SERVER: "SQL Server", + AIRTABLE: "Airtable", + ARANGODB: "ArangoDB", +} + // fields on the user table that cannot be edited export const UNEDITABLE_USER_FIELDS = [ "email", diff --git a/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte index 5202bd45f2..f321a2c422 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/_layout.svelte @@ -1,12 +1,14 @@ +{#if !setupComplete} + +{/if} From 468a7caa07386f2aef45f22b0d124eea8e416cd0 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 23 Sep 2021 22:02:35 +0100 Subject: [PATCH 06/48] fix flow of wizard --- .../app/[application]/data/index.svelte | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/index.svelte index d3b5cff4f2..2d1d94d267 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/index.svelte @@ -1,13 +1,25 @@ -{#if !setupComplete} - -{/if} + From af5280fa363a98e33058e3094d553b09ef918608 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 24 Sep 2021 10:01:53 +0100 Subject: [PATCH 07/48] fetch tables for datasource plus --- .../modals/DatasourceDetailsModal.svelte | 28 +++++++++++++++---- .../app/[application]/data/index.svelte | 5 +--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte index dcdd2da497..6d494378b3 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte @@ -1,11 +1,10 @@ saveDatasource()} - confirmText="Continue" - cancelText="Start from scratch" + confirmText={integration.plus + ? "Fetch tables from database" + : "Save and continue to query"} + cancelText="Back" size="M" > (x._id = "bb_internal")).entities.length > 1 || $datasources.list.length >= 1 - $: console.log($datasources.list.length >= 1) - $: console.log( - $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 - ) + onMount(() => { if (!setupComplete) { modal.show() From 29551c4621bb70d7a6666be29679ec93446ae00d Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 24 Sep 2021 10:12:30 +0100 Subject: [PATCH 08/48] fix naming of datasources --- .../DatasourceNavigator/modals/DatasourceDetailsModal.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte index 6d494378b3..74c176c534 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte @@ -8,7 +8,7 @@ function prepareData() { let datasource = {} let existingTypeCount = $datasources.list.filter( - ds => ds.type == integration.type + ds => ds.source == integration.type ).length let baseName = IntegrationNames[integration.type] @@ -31,8 +31,8 @@ if (integration.plus) { updateDatasourceSchema(resp) } - await datasources.fetch() await datasources.select(resp["_id"]) + console.log($datasources) notifications.success(`Datasource updated successfully.`) } catch (err) { notifications.error(`Error saving datasource: ${err}`) From a833df7771cc359920d7fde31de28f15ac9d1954 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 24 Sep 2021 10:19:48 +0100 Subject: [PATCH 09/48] redirect to created datasource --- .../modals/DatasourceDetailsModal.svelte | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte index 74c176c534..130037ac23 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceDetailsModal.svelte @@ -1,10 +1,12 @@ saveDatasource()} confirmText={integration.plus ? "Fetch tables from database" @@ -65,6 +65,12 @@ cancelText="Back" size="M" > + + Connect your database to Budibase using the config below. + + + Date: Fri, 24 Sep 2021 16:13:25 +0100 Subject: [PATCH 12/48] Add a refresh action to common table searching logic --- .../backend/DataTable/DataTable.svelte | 3 ++- packages/builder/src/helpers/fetchTableData.js | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index e822692da5..308a57c394 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -21,7 +21,8 @@ $: schema = $tables.selected?.schema $: type = $tables.selected?.type $: isInternal = type !== "external" - $: fetchTable($tables.selected?._id) + $: id = $tables.selected?._id + $: fetchTable(id) const fetchTable = tableId => { data.update({ diff --git a/packages/builder/src/helpers/fetchTableData.js b/packages/builder/src/helpers/fetchTableData.js index ff7d61519f..76bd43d89f 100644 --- a/packages/builder/src/helpers/fetchTableData.js +++ b/packages/builder/src/helpers/fetchTableData.js @@ -22,6 +22,7 @@ export const fetchTableData = opts => { // Local non-observable state let query let sortType + let lastBookmark // Local observable state const store = writable({ @@ -43,6 +44,7 @@ export const fetchTableData = opts => { }) const fetchPage = async bookmark => { + lastBookmark = bookmark const { tableId, limit, sortColumn, sortOrder, paginate } = options store.update($store => ({ ...$store, loading: true })) const res = await API.post(`/api/${options.tableId}/search`, { @@ -126,7 +128,7 @@ export const fetchTableData = opts => { // Fetches the next page of data const nextPage = async () => { const state = get(derivedStore) - if (!options.paginate || !state.hasNextPage) { + if (state.loading || !options.paginate || !state.hasNextPage) { return } @@ -151,7 +153,7 @@ export const fetchTableData = opts => { // Fetches the previous page of data const prevPage = async () => { const state = get(derivedStore) - if (!options.paginate || !state.hasPrevPage) { + if (state.loading || !options.paginate || !state.hasPrevPage) { return } @@ -179,6 +181,16 @@ export const fetchTableData = opts => { await fetchData() } + // Loads the same page again + const refresh = async () => { + if (get(store).loading) { + return + } + console.log("refresh") + const page = await fetchPage(lastBookmark) + store.update($store => ({ ...$store, rows: page.rows })) + } + // Initially fetch data but don't bother waiting for the result fetchData() @@ -188,5 +200,6 @@ export const fetchTableData = opts => { nextPage, prevPage, update, + refresh, } } From dad6412271d37e2359efa1d640a5514f7d85eab5 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 27 Sep 2021 09:59:56 +0100 Subject: [PATCH 13/48] trigger internal table modal on modal cancel --- packages/bbui/src/Modal/ModalContent.svelte | 11 ++++++++++- .../modals/CreateDatasourceModal.svelte | 5 ++--- .../modals/DatasourceConfigModal.svelte | 12 ++++++++++-- .../builder/app/[application]/data/index.svelte | 8 +++++++- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte index 678a813a61..09cc4f6c52 100644 --- a/packages/bbui/src/Modal/ModalContent.svelte +++ b/packages/bbui/src/Modal/ModalContent.svelte @@ -14,6 +14,7 @@ export let showConfirmButton = true export let showCloseIcon = true export let onConfirm = undefined + export let onCancel = undefined export let disabled = false export let showDivider = true @@ -28,6 +29,14 @@ } loading = false } + + async function close() { + loading = true + if (!onCancel || (await onCancel()) !== false) { + cancel() + } + loading = false + }
{#if showCancelButton} - + {/if} {#if showConfirmButton}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 9221cbf083..e29d7d5232 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -27,15 +27,20 @@ return datasource } async function saveDatasource() { + let success = true try { // Create datasource const resp = await datasources.save(prepareData()) if (integration.plus) { - updateDatasourceSchema(resp) + fetchedSchema = updateDatasourceSchema(resp) } + + if (!fetchedSchema) { + return false + } + await datasources.select(resp._id) - $goto(`./datasource/${resp._id}`) notifications.success(`Datasource updated successfully.`) analytics.captureEvent(Events.DATASOURCE.CREATED, { name: resp.name, @@ -43,6 +48,7 @@ }) } catch (err) { notifications.error(`Error saving datasource: ${err}`) + return false } } @@ -50,8 +56,10 @@ try { await datasources.updateSchema(datasourceJson) await tables.fetch() + return true } catch (err) { notifications.error(`Error updating datasource schema: ${err}`) + return false } } diff --git a/packages/builder/src/pages/builder/app/[application]/data/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/index.svelte index ca457cd1aa..873f721a59 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/index.svelte @@ -7,7 +7,13 @@ let modal $: setupComplete = $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 || - $datasources.list.length >= 1 + $datasources.list.length > 1 + + $: console.log( + $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 + ) + $: console.log($datasources.list.length >= 1) + $: console.log($datasources.list) onMount(() => { if (!setupComplete) { From 7121b0a7c83e228e63376c3e2526c5476a7497e2 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 27 Sep 2021 10:04:01 +0100 Subject: [PATCH 14/48] fix layout of config modal --- .../modals/DatasourceConfigModal.svelte | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index e29d7d5232..9221cbf083 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -27,20 +27,15 @@ return datasource } async function saveDatasource() { - let success = true try { // Create datasource const resp = await datasources.save(prepareData()) if (integration.plus) { - fetchedSchema = updateDatasourceSchema(resp) + updateDatasourceSchema(resp) } - - if (!fetchedSchema) { - return false - } - await datasources.select(resp._id) + $goto(`./datasource/${resp._id}`) notifications.success(`Datasource updated successfully.`) analytics.captureEvent(Events.DATASOURCE.CREATED, { name: resp.name, @@ -48,7 +43,6 @@ }) } catch (err) { notifications.error(`Error saving datasource: ${err}`) - return false } } @@ -56,10 +50,8 @@ try { await datasources.updateSchema(datasourceJson) await tables.fetch() - return true } catch (err) { notifications.error(`Error updating datasource schema: ${err}`) - return false } } From 496e079d8334c1c88418e2e6ec54984ae1f5f675 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 27 Sep 2021 12:51:32 +0100 Subject: [PATCH 15/48] Remove log statement --- packages/builder/src/helpers/fetchTableData.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/helpers/fetchTableData.js b/packages/builder/src/helpers/fetchTableData.js index 76bd43d89f..8c9e9f1ea7 100644 --- a/packages/builder/src/helpers/fetchTableData.js +++ b/packages/builder/src/helpers/fetchTableData.js @@ -186,7 +186,6 @@ export const fetchTableData = opts => { if (get(store).loading) { return } - console.log("refresh") const page = await fetchPage(lastBookmark) store.update($store => ({ ...$store, rows: page.rows })) } From ec7cf96fd9ceccfa003f151c1ce1fdbdaa675232 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 27 Sep 2021 12:59:49 +0100 Subject: [PATCH 16/48] Move all lucene logic into central builder helpers file --- .../SetupPanel/AutomationBlockSetup.svelte | 2 +- .../builder/src/helpers/fetchTableData.js | 2 +- packages/builder/src/helpers/lucene.js | 187 ++++++++++++++++++ packages/client/rollup.config.js | 4 + .../client/src/components/ClientApp.svelte | 2 +- .../src/components/app/DataProvider.svelte | 2 +- packages/client/src/stores/state.js | 2 +- packages/client/src/utils/conditions.js | 2 +- packages/client/src/utils/lucene.js | 179 ----------------- 9 files changed, 197 insertions(+), 185 deletions(-) delete mode 100644 packages/client/src/utils/lucene.js diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index adc22e5daf..ff52c7d11a 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -24,7 +24,7 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte" // need the client lucene builder to convert to the structure API expects - import { buildLuceneQuery } from "../../../../../client/src/utils/lucene" + import { buildLuceneQuery } from "helpers/lucene" export let block export let webhookModal diff --git a/packages/builder/src/helpers/fetchTableData.js b/packages/builder/src/helpers/fetchTableData.js index 8c9e9f1ea7..43ecd464c9 100644 --- a/packages/builder/src/helpers/fetchTableData.js +++ b/packages/builder/src/helpers/fetchTableData.js @@ -1,6 +1,6 @@ import { writable, derived, get } from "svelte/store" import * as API from "builderStore/api" -import { buildLuceneQuery } from "../../../client/src/utils/lucene" +import { buildLuceneQuery } from "helpers/lucene" const defaultOptions = { tableId: null, diff --git a/packages/builder/src/helpers/lucene.js b/packages/builder/src/helpers/lucene.js index 18692359e4..5225467d7b 100644 --- a/packages/builder/src/helpers/lucene.js +++ b/packages/builder/src/helpers/lucene.js @@ -1,3 +1,186 @@ +/** + * Builds a lucene JSON query from the filter structure generated in the builder + * @param filter the builder filter structure + */ +export const buildLuceneQuery = filter => { + let query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + } + if (Array.isArray(filter)) { + filter.forEach(expression => { + let { operator, field, type, value } = expression + // Parse all values into correct types + if (type === "datetime" && value) { + value = new Date(value).toISOString() + } + if (type === "number") { + value = parseFloat(value) + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if (operator.startsWith("range")) { + if (!query.range[field]) { + query.range[field] = { + low: + type === "number" + ? Number.MIN_SAFE_INTEGER + : "0000-00-00T00:00:00.000Z", + high: + type === "number" + ? Number.MAX_SAFE_INTEGER + : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field].low = value + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field].high = value + } + } else if (query[operator]) { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (operator === "equal" && value === false) { + query.notEqual[field] = true + } else if (operator === "notEqual" && value === false) { + query.equal[field] = true + } else { + query[operator][field] = value + } + } else { + query[operator][field] = value + } + } + }) + } + + return query +} + +/** + * Performs a client-side lucene search on an array of data + * @param docs the data + * @param query the JSON lucene query + */ +export const luceneQuery = (docs, query) => { + if (!query) { + return docs + } + + // Iterates over a set of filters and evaluates a fail function against a doc + const match = (type, failFn) => doc => { + const filters = Object.entries(query[type] || {}) + for (let i = 0; i < filters.length; i++) { + if (failFn(filters[i][0], filters[i][1], doc)) { + return false + } + } + return true + } + + // Process a string match (fails if the value does not start with the string) + const stringMatch = match("string", (key, value, doc) => { + return !doc[key] || !doc[key].startsWith(value) + }) + + // Process a fuzzy match (treat the same as starts with when running locally) + const fuzzyMatch = match("fuzzy", (key, value, doc) => { + return !doc[key] || !doc[key].startsWith(value) + }) + + // Process a range match + const rangeMatch = match("range", (key, value, doc) => { + return !doc[key] || doc[key] < value.low || doc[key] > value.high + }) + + // Process an equal match (fails if the value is different) + const equalMatch = match("equal", (key, value, doc) => { + return value != null && value !== "" && doc[key] !== value + }) + + // Process a not-equal match (fails if the value is the same) + const notEqualMatch = match("notEqual", (key, value, doc) => { + return value != null && value !== "" && doc[key] === value + }) + + // Process an empty match (fails if the value is not empty) + const emptyMatch = match("empty", (key, value, doc) => { + return doc[key] != null && doc[key] !== "" + }) + + // Process a not-empty match (fails is the value is empty) + const notEmptyMatch = match("notEmpty", (key, value, doc) => { + return doc[key] == null || doc[key] === "" + }) + + // Match a document against all criteria + const docMatch = doc => { + return ( + stringMatch(doc) && + fuzzyMatch(doc) && + rangeMatch(doc) && + equalMatch(doc) && + notEqualMatch(doc) && + emptyMatch(doc) && + notEmptyMatch(doc) + ) + } + + // Process all docs + return docs.filter(docMatch) +} + +/** + * Performs a client-side sort from the equivalent server-side lucene sort + * parameters. + * @param docs the data + * @param sort the sort column + * @param sortOrder the sort order ("ascending" or "descending") + * @param sortType the type of sort ("string" or "number") + */ +export const luceneSort = (docs, sort, sortOrder, sortType = "string") => { + if (!sort || !sortOrder || !sortType) { + return docs + } + const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x) + return docs.slice().sort((a, b) => { + const colA = parse(a[sort]) + const colB = parse(b[sort]) + if (sortOrder === "Descending") { + return colA > colB ? -1 : 1 + } else { + return colA > colB ? 1 : -1 + } + }) +} + +/** + * Limits the specified docs to the specified number of rows from the equivalent + * server-side lucene limit parameters. + * @param docs the data + * @param limit the number of docs to limit to + */ +export const luceneLimit = (docs, limit) => { + const numLimit = parseFloat(limit) + if (isNaN(numLimit)) { + return docs + } + return docs.slice(0, numLimit) +} + +/** + * Operator options for lucene queries + */ export const OperatorOptions = { Equals: { value: "equal", @@ -41,6 +224,10 @@ export const OperatorOptions = { }, } +/** + * Returns the valid operator options for a certain data type + * @param type the data type + */ export const getValidOperatorsForType = type => { const Op = OperatorOptions if (type === "string") { diff --git a/packages/client/rollup.config.js b/packages/client/rollup.config.js index f404f93c4c..a814303069 100644 --- a/packages/client/rollup.config.js +++ b/packages/client/rollup.config.js @@ -58,6 +58,10 @@ export default { find: "sdk", replacement: path.resolve("./src/sdk"), }, + { + find: "builder", + replacement: path.resolve("../builder"), + }, ], }), svelte({ diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte index 5890c0e2a9..c9c033caa3 100644 --- a/packages/client/src/components/ClientApp.svelte +++ b/packages/client/src/components/ClientApp.svelte @@ -23,7 +23,7 @@ import SelectionIndicator from "components/preview/SelectionIndicator.svelte" import HoverIndicator from "components/preview/HoverIndicator.svelte" import CustomThemeWrapper from "./CustomThemeWrapper.svelte" - import ErrorSVG from "../../../builder/assets/error.svg" + import ErrorSVG from "builder/assets/error.svg" // Provide contexts setContext("sdk", SDK) diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index bdc9001445..991c41b77d 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -6,7 +6,7 @@ luceneQuery, luceneSort, luceneLimit, - } from "utils/lucene" + } from "builder/src/helpers/lucene" import Placeholder from "./Placeholder.svelte" export let dataSource diff --git a/packages/client/src/stores/state.js b/packages/client/src/stores/state.js index cb20149de8..ce977c4333 100644 --- a/packages/client/src/stores/state.js +++ b/packages/client/src/stores/state.js @@ -1,5 +1,5 @@ import { writable, get, derived } from "svelte/store" -import { localStorageStore } from "../../../builder/src/builderStore/store/localStorage" +import { localStorageStore } from "builder/src/builderStore/store/localStorage" import { appStore } from "./app" const createStateStore = () => { diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js index 964a63d3fd..2791fa169e 100644 --- a/packages/client/src/utils/conditions.js +++ b/packages/client/src/utils/conditions.js @@ -1,4 +1,4 @@ -import { buildLuceneQuery, luceneQuery } from "./lucene" +import { buildLuceneQuery, luceneQuery } from "builder/src/helpers/lucene" export const getActiveConditions = conditions => { if (!conditions?.length) { diff --git a/packages/client/src/utils/lucene.js b/packages/client/src/utils/lucene.js deleted file mode 100644 index 03baa751cc..0000000000 --- a/packages/client/src/utils/lucene.js +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Builds a lucene JSON query from the filter structure generated in the builder - * @param filter the builder filter structure - */ -export const buildLuceneQuery = filter => { - let query = { - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - contains: {}, - notContains: {}, - } - if (Array.isArray(filter)) { - filter.forEach(expression => { - let { operator, field, type, value } = expression - // Parse all values into correct types - if (type === "datetime" && value) { - value = new Date(value).toISOString() - } - if (type === "number") { - value = parseFloat(value) - } - if (type === "boolean") { - value = `${value}`?.toLowerCase() === "true" - } - if (operator.startsWith("range")) { - if (!query.range[field]) { - query.range[field] = { - low: - type === "number" - ? Number.MIN_SAFE_INTEGER - : "0000-00-00T00:00:00.000Z", - high: - type === "number" - ? Number.MAX_SAFE_INTEGER - : "9999-00-00T00:00:00.000Z", - } - } - if (operator === "rangeLow" && value != null && value !== "") { - query.range[field].low = value - } else if (operator === "rangeHigh" && value != null && value !== "") { - query.range[field].high = value - } - } else if (query[operator]) { - if (type === "boolean") { - // Transform boolean filters to cope with null. - // "equals false" needs to be "not equals true" - // "not equals false" needs to be "equals true" - if (operator === "equal" && value === false) { - query.notEqual[field] = true - } else if (operator === "notEqual" && value === false) { - query.equal[field] = true - } else { - query[operator][field] = value - } - } else { - query[operator][field] = value - } - } - }) - } - - return query -} - -/** - * Performs a client-side lucene search on an array of data - * @param docs the data - * @param query the JSON lucene query - */ -export const luceneQuery = (docs, query) => { - if (!query) { - return docs - } - - // Iterates over a set of filters and evaluates a fail function against a doc - const match = (type, failFn) => doc => { - const filters = Object.entries(query[type] || {}) - for (let i = 0; i < filters.length; i++) { - if (failFn(filters[i][0], filters[i][1], doc)) { - return false - } - } - return true - } - - // Process a string match (fails if the value does not start with the string) - const stringMatch = match("string", (key, value, doc) => { - return !doc[key] || !doc[key].startsWith(value) - }) - - // Process a fuzzy match (treat the same as starts with when running locally) - const fuzzyMatch = match("fuzzy", (key, value, doc) => { - return !doc[key] || !doc[key].startsWith(value) - }) - - // Process a range match - const rangeMatch = match("range", (key, value, doc) => { - return !doc[key] || doc[key] < value.low || doc[key] > value.high - }) - - // Process an equal match (fails if the value is different) - const equalMatch = match("equal", (key, value, doc) => { - return value != null && value !== "" && doc[key] !== value - }) - - // Process a not-equal match (fails if the value is the same) - const notEqualMatch = match("notEqual", (key, value, doc) => { - return value != null && value !== "" && doc[key] === value - }) - - // Process an empty match (fails if the value is not empty) - const emptyMatch = match("empty", (key, value, doc) => { - return doc[key] != null && doc[key] !== "" - }) - - // Process a not-empty match (fails is the value is empty) - const notEmptyMatch = match("notEmpty", (key, value, doc) => { - return doc[key] == null || doc[key] === "" - }) - - // Match a document against all criteria - const docMatch = doc => { - return ( - stringMatch(doc) && - fuzzyMatch(doc) && - rangeMatch(doc) && - equalMatch(doc) && - notEqualMatch(doc) && - emptyMatch(doc) && - notEmptyMatch(doc) - ) - } - - // Process all docs - return docs.filter(docMatch) -} - -/** - * Performs a client-side sort from the equivalent server-side lucene sort - * parameters. - * @param docs the data - * @param sort the sort column - * @param sortOrder the sort order ("ascending" or "descending") - * @param sortType the type of sort ("string" or "number") - */ -export const luceneSort = (docs, sort, sortOrder, sortType = "string") => { - if (!sort || !sortOrder || !sortType) { - return docs - } - const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x) - return docs.slice().sort((a, b) => { - const colA = parse(a[sort]) - const colB = parse(b[sort]) - if (sortOrder === "Descending") { - return colA > colB ? -1 : 1 - } else { - return colA > colB ? 1 : -1 - } - }) -} - -/** - * Limits the specified docs to the specified number of rows from the equivalent - * server-side lucene limit parameters. - * @param docs the data - * @param limit the number of docs to limit to - */ -export const luceneLimit = (docs, limit) => { - const numLimit = parseFloat(limit) - if (isNaN(numLimit)) { - return docs - } - return docs.slice(0, numLimit) -} From 705088362b5b87a85bb74e230fc66dc016ba812b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 27 Sep 2021 15:36:18 +0100 Subject: [PATCH 17/48] Move lucene logic into builder --- .../backend/DataTable/DataTable.svelte | 20 ++-- .../ConditionalUIDrawer.svelte | 2 +- .../FilterEditor/FilterDrawer.svelte | 2 +- packages/builder/src/constants/lucene.js | 97 ++++++++++++++++++ .../builder/src/helpers/fetchTableData.js | 6 +- packages/builder/src/helpers/lucene.js | 98 ------------------- 6 files changed, 113 insertions(+), 112 deletions(-) create mode 100644 packages/builder/src/constants/lucene.js diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 308a57c394..f8b5abc4cd 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -13,7 +13,7 @@ import { fetchTableData } from "helpers/fetchTableData" import { Pagination } from "@budibase/bbui" - const data = fetchTableData() + const search = fetchTableData() let hideAutocolumns = true $: isUsersTable = $tables.selected?._id === TableNames.USERS @@ -25,7 +25,7 @@ $: fetchTable(id) const fetchTable = tableId => { - data.update({ + search.update({ tableId, schema, limit: 10, @@ -35,7 +35,7 @@ // Fetch data whenever sorting option changes const onSort = e => { - data.update({ + search.update({ sortColumn: e.detail.column, sortOrder: e.detail.order, }) @@ -48,9 +48,9 @@ {schema} {type} tableId={$tables.selected?._id} - data={$data.rows} + data={$search.rows} bind:hideAutocolumns - loading={$data.loading} + loading={$search.loading} on:sort={onSort} allowEditing disableSorting @@ -79,11 +79,11 @@
diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte index 638fd44de6..9f0d5086f6 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte @@ -12,7 +12,7 @@ import { dndzone } from "svelte-dnd-action" import { generate } from "shortid" import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" - import { OperatorOptions, getValidOperatorsForType } from "helpers/lucene" + import { OperatorOptions, getValidOperatorsForType } from "constants/lucene" import { selectedComponent, store } from "builderStore" import { getComponentForSettingType } from "./componentSettings" import PropertyControl from "./PropertyControl.svelte" diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index d6bfadb150..eddfd9b997 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -13,7 +13,7 @@ import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" import BindingPanel from "components/common/bindings/BindingPanel.svelte" import { generate } from "shortid" - import { getValidOperatorsForType, OperatorOptions } from "helpers/lucene" + import { getValidOperatorsForType, OperatorOptions } from "constants/lucene" export let schemaFields export let filters = [] diff --git a/packages/builder/src/constants/lucene.js b/packages/builder/src/constants/lucene.js new file mode 100644 index 0000000000..00da0c29bc --- /dev/null +++ b/packages/builder/src/constants/lucene.js @@ -0,0 +1,97 @@ +/** + * Operator options for lucene queries + */ +export const OperatorOptions = { + Equals: { + value: "equal", + label: "Equals", + }, + NotEquals: { + value: "notEqual", + label: "Not equals", + }, + Empty: { + value: "empty", + label: "Is empty", + }, + NotEmpty: { + value: "notEmpty", + label: "Is not empty", + }, + StartsWith: { + value: "string", + label: "Starts with", + }, + Like: { + value: "fuzzy", + label: "Like", + }, + MoreThan: { + value: "rangeLow", + label: "More than", + }, + LessThan: { + value: "rangeHigh", + label: "Less than", + }, + Contains: { + value: "equal", + label: "Contains", + }, + NotContains: { + value: "notEqual", + label: "Does Not Contain", + }, +} + +/** + * Returns the valid operator options for a certain data type + * @param type the data type + */ +export const getValidOperatorsForType = type => { + const Op = OperatorOptions + if (type === "string") { + return [ + Op.Equals, + Op.NotEquals, + Op.StartsWith, + Op.Like, + Op.Empty, + Op.NotEmpty, + ] + } else if (type === "number") { + return [ + Op.Equals, + Op.NotEquals, + Op.MoreThan, + Op.LessThan, + Op.Empty, + Op.NotEmpty, + ] + } else if (type === "options") { + return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] + } else if (type === "array") { + return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty] + } else if (type === "boolean") { + return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] + } else if (type === "longform") { + return [ + Op.Equals, + Op.NotEquals, + Op.StartsWith, + Op.Like, + Op.Empty, + Op.NotEmpty, + ] + } else if (type === "datetime") { + return [ + Op.Equals, + Op.NotEquals, + Op.MoreThan, + Op.LessThan, + Op.Empty, + Op.NotEmpty, + ] + } + return [] +} diff --git a/packages/builder/src/helpers/fetchTableData.js b/packages/builder/src/helpers/fetchTableData.js index 43ecd464c9..5ef3bc1076 100644 --- a/packages/builder/src/helpers/fetchTableData.js +++ b/packages/builder/src/helpers/fetchTableData.js @@ -1,6 +1,8 @@ +// Do not use any aliased imports in common files, as these will be bundled +// by multiple bundlers which may not be able to resolve them import { writable, derived, get } from "svelte/store" -import * as API from "builderStore/api" -import { buildLuceneQuery } from "helpers/lucene" +import * as API from "../builderStore/api" +import { buildLuceneQuery } from "./lucene" const defaultOptions = { tableId: null, diff --git a/packages/builder/src/helpers/lucene.js b/packages/builder/src/helpers/lucene.js index 5225467d7b..03baa751cc 100644 --- a/packages/builder/src/helpers/lucene.js +++ b/packages/builder/src/helpers/lucene.js @@ -177,101 +177,3 @@ export const luceneLimit = (docs, limit) => { } return docs.slice(0, numLimit) } - -/** - * Operator options for lucene queries - */ -export const OperatorOptions = { - Equals: { - value: "equal", - label: "Equals", - }, - NotEquals: { - value: "notEqual", - label: "Not equals", - }, - Empty: { - value: "empty", - label: "Is empty", - }, - NotEmpty: { - value: "notEmpty", - label: "Is not empty", - }, - StartsWith: { - value: "string", - label: "Starts with", - }, - Like: { - value: "fuzzy", - label: "Like", - }, - MoreThan: { - value: "rangeLow", - label: "More than", - }, - LessThan: { - value: "rangeHigh", - label: "Less than", - }, - Contains: { - value: "equal", - label: "Contains", - }, - NotContains: { - value: "notEqual", - label: "Does Not Contain", - }, -} - -/** - * Returns the valid operator options for a certain data type - * @param type the data type - */ -export const getValidOperatorsForType = type => { - const Op = OperatorOptions - if (type === "string") { - return [ - Op.Equals, - Op.NotEquals, - Op.StartsWith, - Op.Like, - Op.Empty, - Op.NotEmpty, - ] - } else if (type === "number") { - return [ - Op.Equals, - Op.NotEquals, - Op.MoreThan, - Op.LessThan, - Op.Empty, - Op.NotEmpty, - ] - } else if (type === "options") { - return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] - } else if (type === "array") { - return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty] - } else if (type === "boolean") { - return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] - } else if (type === "longform") { - return [ - Op.Equals, - Op.NotEquals, - Op.StartsWith, - Op.Like, - Op.Empty, - Op.NotEmpty, - ] - } else if (type === "datetime") { - return [ - Op.Equals, - Op.NotEquals, - Op.MoreThan, - Op.LessThan, - Op.Empty, - Op.NotEmpty, - ] - } - return [] -} From f5f8e25c9f58b743b6de4e879ada342520d6f319 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 27 Sep 2021 18:25:19 +0100 Subject: [PATCH 18/48] Fix huge amount of bugs with data UI state, URL parameters and state/URL sync --- .../DatasourceNavigator.svelte | 76 ++++++++++++++----- .../[query]/_layout.svelte | 12 +++ .../[selectedDatasource]/_layout.svelte | 2 +- .../data/datasource/_layout.svelte | 13 ---- .../builder/src/stores/backend/queries.js | 9 +-- packages/builder/src/stores/backend/views.js | 1 + 6 files changed, 76 insertions(+), 37 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index 6ba8e4042f..3c6fa83c01 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -1,9 +1,9 @@ diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte index 4fa864ce7a..6d802df2e9 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte @@ -1 +1,13 @@ + + diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte index 13f8719594..f48be08fd6 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte @@ -2,7 +2,7 @@ import { params } from "@roxi/routify" import { datasources } from "stores/backend" - if ($params.selectedDatasource) { + if ($params.selectedDatasource && !$params.query) { const datasource = $datasources.list.find( m => m._id === $params.selectedDatasource ) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/_layout.svelte index d05aa882ad..4fa864ce7a 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/_layout.svelte @@ -1,14 +1 @@ - - diff --git a/packages/builder/src/stores/backend/queries.js b/packages/builder/src/stores/backend/queries.js index 2eeae29b9d..020a0c9420 100644 --- a/packages/builder/src/stores/backend/queries.js +++ b/packages/builder/src/stores/backend/queries.js @@ -1,5 +1,5 @@ import { writable, get } from "svelte/store" -import { datasources, integrations, tables } from "./" +import { datasources, integrations, tables, views } from "./" import api from "builderStore/api" export function createQueriesStore() { @@ -55,10 +55,9 @@ export function createQueriesStore() { }, select: query => { update(state => ({ ...state, selected: query._id })) - tables.update(state => ({ - ...state, - selected: null, - })) + views.unselect() + tables.unselect() + datasources.unselect() }, unselect: () => { update(state => ({ ...state, selected: null })) diff --git a/packages/builder/src/stores/backend/views.js b/packages/builder/src/stores/backend/views.js index 0b15d18fa5..14c7bf92a4 100644 --- a/packages/builder/src/stores/backend/views.js +++ b/packages/builder/src/stores/backend/views.js @@ -16,6 +16,7 @@ export function createViewsStore() { ...state, selected: view, })) + tables.unselect() queries.unselect() datasources.unselect() }, From 1a1c0f2892d77384920b451011615a2507c1d221 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 28 Sep 2021 12:25:57 +0100 Subject: [PATCH 19/48] check tables can be fetched before saving config --- .../modals/DatasourceConfigModal.svelte | 19 +++-------- .../app/[application]/data/index.svelte | 6 ---- .../builder/src/stores/backend/datasources.js | 7 ++-- .../server/src/api/controllers/datasource.js | 32 +++++++++++++------ 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 9221cbf083..56fa26ee0a 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -3,7 +3,7 @@ import { ModalContent, notifications, Body, Layout } from "@budibase/bbui" import analytics, { Events } from "analytics" import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte" - import { datasources, tables } from "stores/backend" + import { datasources } from "stores/backend" import { IntegrationNames } from "constants" export let integration @@ -27,13 +27,11 @@ return datasource } async function saveDatasource() { + const datasource = prepareData() try { // Create datasource - const resp = await datasources.save(prepareData()) + const resp = await datasources.save(datasource, datasource.plus) - if (integration.plus) { - updateDatasourceSchema(resp) - } await datasources.select(resp._id) $goto(`./datasource/${resp._id}`) notifications.success(`Datasource updated successfully.`) @@ -41,17 +39,10 @@ name: resp.name, source: resp.source, }) + return true } catch (err) { notifications.error(`Error saving datasource: ${err}`) - } - } - - async function updateDatasourceSchema(datasourceJson) { - try { - await datasources.updateSchema(datasourceJson) - await tables.fetch() - } catch (err) { - notifications.error(`Error updating datasource schema: ${err}`) + return false } } diff --git a/packages/builder/src/pages/builder/app/[application]/data/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/index.svelte index 873f721a59..c27ee7b342 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/index.svelte @@ -9,12 +9,6 @@ $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 || $datasources.list.length > 1 - $: console.log( - $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 - ) - $: console.log($datasources.list.length >= 1) - $: console.log($datasources.list) - onMount(() => { if (!setupComplete) { modal.show() diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 5c6ed3f2cb..f7e689d8ad 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -58,7 +58,7 @@ export function createDatasourcesStore() { }) return json }, - save: async datasource => { + save: async (datasource, fetchSchema = false) => { let response if (datasource._id) { response = await api.put( @@ -66,7 +66,10 @@ export function createDatasourcesStore() { datasource ) } else { - response = await api.post("/api/datasources", datasource) + response = await api.post("/api/datasources", { + datasource: datasource, + fetchSchema, + }) } const json = await response.json() diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index 4a2fd7d86a..2ff7c7f9b8 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -41,15 +41,10 @@ exports.fetch = async function (ctx) { exports.buildSchemaFromDb = async function (ctx) { const db = new CouchDB(ctx.appId) - const datasourceId = ctx.params.datasourceId - const datasource = await db.get(datasourceId) + const datasource = await db.get(ctx.params.datasourceId) - const Connector = integrations[datasource.source] - - // Connect to the DB and build the schema - const connector = new Connector(datasource.config) - await connector.buildSchema(datasource._id, datasource.entities) - datasource.entities = connector.tables + const tables = await buildSchemaHelper(datasource) + datasource.entities = tables const response = await db.put(datasource) datasource._rev = response.rev @@ -81,12 +76,18 @@ exports.update = async function (ctx) { exports.save = async function (ctx) { const db = new CouchDB(ctx.appId) - const plus = ctx.request.body.plus + const plus = ctx.request.body.datasource.plus + const fetchSchema = ctx.request.body.fetchSchema const datasource = { _id: generateDatasourceID({ plus }), type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE, - ...ctx.request.body, + ...ctx.request.body.datasource, + } + + if (fetchSchema) { + let tables = await buildSchemaHelper(datasource) + datasource.entities = tables } const response = await db.put(datasource) @@ -133,3 +134,14 @@ exports.query = async function (ctx) { ctx.throw(400, err) } } + +const buildSchemaHelper = async datasource => { + const Connector = integrations[datasource.source] + + // Connect to the DB and build the schema + const connector = new Connector(datasource.config) + await connector.buildSchema(datasource._id, datasource.entities) + datasource.entities = connector.tables + + return connector.tables +} From a117f3542c7f4a32ec2a89e4297a147ee73b0178 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 28 Sep 2021 12:26:38 +0100 Subject: [PATCH 20/48] update cypress tests to account for new modal --- packages/builder/cypress.json | 4 ++-- .../cypress/integration/createTable.spec.js | 6 ++--- packages/builder/cypress/support/commands.js | 22 ++++++++++++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/builder/cypress.json b/packages/builder/cypress.json index 0908f2c839..669ee5af34 100644 --- a/packages/builder/cypress.json +++ b/packages/builder/cypress.json @@ -1,9 +1,9 @@ { - "baseUrl": "http://localhost:10001/builder/", + "baseUrl": "http://localhost:10000/builder/", "video": true, "projectId": "bmbemn", "env": { - "PORT": "10001", + "PORT": "10000", "JWT_SECRET": "test" } } diff --git a/packages/builder/cypress/integration/createTable.spec.js b/packages/builder/cypress/integration/createTable.spec.js index eda72ba36d..a0d741910c 100644 --- a/packages/builder/cypress/integration/createTable.spec.js +++ b/packages/builder/cypress/integration/createTable.spec.js @@ -6,7 +6,7 @@ context("Create a Table", () => { it("should create a new Table", () => { cy.createTable("dog") - + cy.wait(1000) // Check if Table exists cy.get(".table-title h1").should("have.text", "dog") }) @@ -36,7 +36,7 @@ context("Create a Table", () => { it("edits a row", () => { cy.contains("button", "Edit").click({ force: true }) cy.wait(1000) - cy.get(".spectrum-Modal input").type("Updated") + cy.get(".spectrum-Modal input").type("RoverUpdated") cy.contains("Save").click() cy.contains("RoverUpdated").should("have.text", "RoverUpdated") }) @@ -62,7 +62,7 @@ context("Create a Table", () => { it("deletes a table", () => { cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use") - .first() + .eq(1) .click({ force: true }) cy.get(".spectrum-Menu > :nth-child(2)").click() cy.contains("Delete Table").click() diff --git a/packages/builder/cypress/support/commands.js b/packages/builder/cypress/support/commands.js index ea6ca81e66..d681fda8ad 100644 --- a/packages/builder/cypress/support/commands.js +++ b/packages/builder/cypress/support/commands.js @@ -35,8 +35,11 @@ Cypress.Commands.add("createApp", name => { .within(() => { cy.get("input").eq(0).type(name).should("have.value", name).blur() cy.get(".spectrum-ButtonGroup").contains("Create app").click() + cy.wait(7000) }) .then(() => { + // Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future + cy.createInitialDatasource("initialTable") cy.expandBudibaseConnection() cy.get(".nav-item.selected > .content").should("be.visible") }) @@ -69,11 +72,28 @@ Cypress.Commands.add("createTestTableWithData", () => { cy.addColumn("dog", "age", "Number") }) -Cypress.Commands.add("createTable", tableName => { +Cypress.Commands.add("createInitialDatasource", tableName => { // Enter table name + cy.get(".spectrum-Modal").within(() => { + cy.contains("Budibase DB").trigger("mouseover").click().click() + cy.wait(1000) + cy.contains("Continue").click() + }) + + cy.get(".spectrum-Modal").within(() => { + cy.wait(1000) + cy.get("input").first().type(tableName).blur() + cy.get(".spectrum-ButtonGroup").contains("Create").click() + }) + cy.contains(tableName).should("be.visible") +}) + +Cypress.Commands.add("createTable", tableName => { cy.contains("Budibase DB").click() cy.contains("Create new table").click() + cy.get(".spectrum-Modal").within(() => { + cy.wait(1000) cy.get("input").first().type(tableName).blur() cy.get(".spectrum-ButtonGroup").contains("Create").click() }) From a24f2e4142fa09fb739d9b5a778151f252eb7086 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 28 Sep 2021 12:29:54 +0100 Subject: [PATCH 21/48] update modal button text --- .../DatasourceNavigator/modals/CreateDatasourceModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index bbf3ec22ee..be0e83f80c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -68,7 +68,7 @@ disabled={!Object.keys(integration).length} title="Data" confirmText="Continue" - cancelText="Start from scratch" + cancelText="Create a new table with Budibase DB" size="M" onCancel={() => internalTableModal.show()} onConfirm={() => { From 7527c19a8c8bb121b4da1f937d9c9284fce8c64d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 28 Sep 2021 15:29:15 +0100 Subject: [PATCH 22/48] Add basic search implementation to data UI tables --- .../backend/DataTable/DataTable.svelte | 95 ++++++++++++++----- .../components/backend/DataTable/Table.svelte | 2 +- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index f8b5abc4cd..d145b3136c 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -11,10 +11,13 @@ import { TableNames } from "constants" import CreateEditRow from "./modals/CreateEditRow.svelte" import { fetchTableData } from "helpers/fetchTableData" - import { Pagination } from "@budibase/bbui" + import { Layout, Pagination, Select, Input } from "@budibase/bbui" + import { OperatorOptions } from "constants/lucene" const search = fetchTableData() let hideAutocolumns = true + let searchColumn + let searchValue $: isUsersTable = $tables.selected?._id === TableNames.USERS $: title = $tables.selected?.name @@ -22,12 +25,16 @@ $: type = $tables.selected?.type $: isInternal = type !== "external" $: id = $tables.selected?._id - $: fetchTable(id) + $: columnOptions = Object.keys($search.schema || {}) + $: filter = buildFilter(searchColumn, searchValue) + $: fetchTable(id, filter) - const fetchTable = tableId => { + // Fetches new data whenever the table changes + const fetchTable = (tableId, filter) => { search.update({ tableId, schema, + filter, limit: 10, paginate: true, }) @@ -40,6 +47,23 @@ sortOrder: e.detail.order, }) } + + // Builds a filter expression to search with + const buildFilter = (column, value) => { + if (!column || !value) { + return null + } + return [ + { + type: "string", + field: column, + operator: OperatorOptions.StartsWith.value, + value, + }, + ] + } + + $: console.log(filter)
@@ -55,27 +79,39 @@ allowEditing disableSorting > - {#if isInternal} - - {/if} - {#if schema && Object.keys(schema).length > 0} - {#if !isUsersTable} - +
+ {#if isInternal} + + {/if} + {#if schema && Object.keys(schema).length > 0} + {#if !isUsersTable} + + {/if} + {#if isInternal} + + {/if} + + {#if isUsersTable} + + {/if} + + + + {/if} +
+ +