From d03fdb6df94f6d17ad0c1a79ed38f1196e70f535 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 3 Oct 2023 17:35:00 +0100 Subject: [PATCH 01/37] Add initial support for query datasources in grids --- packages/client/manifest.json | 5 +- .../src/components/app/GridBlock.svelte | 7 +- .../src/components/grid/cells/DataCell.svelte | 2 +- .../src/components/grid/layout/NewRow.svelte | 6 +- .../src/components/grid/stores/config.js | 11 +- .../src/components/grid/stores/datasource.js | 25 +++- .../src/components/grid/stores/index.js | 2 + .../src/components/grid/stores/query.js | 108 ++++++++++++++++++ .../src/components/grid/stores/rows.js | 10 +- .../src/components/grid/stores/table.js | 7 +- .../src/components/grid/stores/viewV2.js | 16 +-- .../src/fetch/{fetchData.js => index.js} | 11 ++ packages/frontend-core/src/index.js | 2 +- 13 files changed, 178 insertions(+), 34 deletions(-) create mode 100644 packages/frontend-core/src/components/grid/stores/query.js rename packages/frontend-core/src/fetch/{fetchData.js => index.js} (72%) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 4e56ca758d..b1391e36fd 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5545,12 +5545,11 @@ "width": 600, "height": 400 }, - "info": "Grid Blocks are only compatible with internal or SQL tables", "settings": [ { - "type": "table", + "type": "dataSource", "label": "Data", - "key": "table", + "key": "datasource", "required": true }, { diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte index 9bdea52124..1cdbcf00ff 100644 --- a/packages/client/src/components/app/GridBlock.svelte +++ b/packages/client/src/components/app/GridBlock.svelte @@ -4,7 +4,7 @@ import { getContext } from "svelte" import { Grid } from "@budibase/frontend-core" - export let table + export let datasource export let allowAddRows = true export let allowEditRows = true export let allowDeleteRows = true @@ -15,6 +15,9 @@ export let fixedRowHeight = null export let columns = null + // Legacy settings + export let table + const component = getContext("component") const { styleable, API, builderStore, notificationStore } = getContext("sdk") @@ -38,7 +41,7 @@ class:in-builder={$builderStore.inBuilder} > { [props, hasNonAutoColumn], ([$props, $hasNonAutoColumn]) => { let config = { ...$props } + const type = $props.datasource?.type // Disable some features if we're editing a view - if ($props.datasource?.type === "viewV2") { + if (type === "viewV2") { config.canEditColumns = false } @@ -48,6 +49,14 @@ export const deriveStores = context => { config.canAddRows = false } + // Disable features for non DS+ + if (!["table", "viewV2"].includes(type)) { + config.canAddRows = false + config.canEditRows = false + config.canDeleteRows = false + config.canExpandRows = false + } + return config } ) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 3f4347953e..e56b37a5f3 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -1,4 +1,5 @@ import { derived, get, writable } from "svelte/store" +import { getDatasourceDefinition } from "../../../fetch" export const createStores = () => { const definition = writable(null) @@ -19,6 +20,14 @@ export const deriveStores = context => { } let newSchema = { ...$definition?.schema } + // Ensure schema is configured as objects. + // Certain datasources like queries use primitives. + Object.keys(newSchema).forEach(key => { + if (typeof newSchema[key] !== "object") { + newSchema[key] = { type: newSchema[key] } + } + }) + // Apply schema overrides Object.keys($schemaOverrides || {}).forEach(field => { if (newSchema[field]) { @@ -48,7 +57,16 @@ export const deriveStores = context => { } export const createActions = context => { - const { datasource, definition, config, dispatch, table, viewV2 } = context + const { + API, + datasource, + definition, + config, + dispatch, + table, + viewV2, + query, + } = context // Gets the appropriate API for the configured datasource type const getAPI = () => { @@ -58,6 +76,8 @@ export const createActions = context => { return table case "viewV2": return viewV2 + case "query": + return query default: return null } @@ -65,7 +85,8 @@ export const createActions = context => { // Refreshes the datasource definition const refreshDefinition = async () => { - return await getAPI()?.actions.refreshDefinition() + const def = await getDatasourceDefinition({ API, datasource }) + definition.set(def) } // Saves the datasource definition diff --git a/packages/frontend-core/src/components/grid/stores/index.js b/packages/frontend-core/src/components/grid/stores/index.js index 7b73ea8be6..5a46227b49 100644 --- a/packages/frontend-core/src/components/grid/stores/index.js +++ b/packages/frontend-core/src/components/grid/stores/index.js @@ -17,6 +17,7 @@ import * as Filter from "./filter" import * as Notifications from "./notifications" import * as Table from "./table" import * as ViewV2 from "./viewV2" +import * as Query from "./query" import * as Datasource from "./datasource" const DependencyOrderedStores = [ @@ -26,6 +27,7 @@ const DependencyOrderedStores = [ Scroll, Table, ViewV2, + Query, Datasource, Columns, Rows, diff --git a/packages/frontend-core/src/components/grid/stores/query.js b/packages/frontend-core/src/components/grid/stores/query.js new file mode 100644 index 0000000000..6ac0547d27 --- /dev/null +++ b/packages/frontend-core/src/components/grid/stores/query.js @@ -0,0 +1,108 @@ +import { get } from "svelte/store" + +export const createActions = context => { + const { API, columns, stickyColumn } = context + + const saveDefinition = async newDefinition => { + await API.saveQuery(newDefinition) + } + + const saveRow = async () => { + throw "Rows cannot be updated through queries" + } + + const deleteRows = async () => { + throw "Rows cannot be deleted through queries" + } + + const getRow = () => { + throw "Queries don't support fetching individual rows" + } + + const isDatasourceValid = datasource => { + return datasource?.type === "query" && datasource?._id + } + + const canUseColumn = name => { + const $columns = get(columns) + const $sticky = get(stickyColumn) + return $columns.some(col => col.name === name) || $sticky?.name === name + } + + return { + query: { + actions: { + saveDefinition, + addRow: saveRow, + updateRow: saveRow, + deleteRows, + getRow, + isDatasourceValid, + canUseColumn, + }, + }, + } +} + +export const initialise = context => { + const { + datasource, + sort, + filter, + query, + initialFilter, + initialSortColumn, + initialSortOrder, + fetch, + } = context + + // Keep a list of subscriptions so that we can clear them when the datasource + // config changes + let unsubscribers = [] + + // Observe datasource changes and apply logic for view V2 datasources + datasource.subscribe($datasource => { + // Clear previous subscriptions + unsubscribers?.forEach(unsubscribe => unsubscribe()) + unsubscribers = [] + if (!query.actions.isDatasourceValid($datasource)) { + return + } + + // Wipe state + filter.set(get(initialFilter)) + sort.set({ + column: get(initialSortColumn), + order: get(initialSortOrder) || "ascending", + }) + + // Update fetch when filter changes + unsubscribers.push( + filter.subscribe($filter => { + // Ensure we're updating the correct fetch + const $fetch = get(fetch) + if ($fetch?.options?.datasource?._id !== $datasource._id) { + return + } + $fetch.update({ + filter: $filter, + }) + }) + ) + + // Update fetch when sorting changes + unsubscribers.push( + sort.subscribe($sort => { + // Ensure we're updating the correct fetch + const $fetch = get(fetch) + if ($fetch?.options?.datasource?._id !== $datasource._id) { + return + } + $fetch.update({ + sortOrder: $sort.order || "ascending", + sortColumn: $sort.column, + }) + }) + ) + }) +} diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 392bf392e8..2e2832f209 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -1,7 +1,8 @@ import { writable, derived, get } from "svelte/store" -import { fetchData } from "../../../fetch/fetchData" +import { fetchData } from "../../../fetch" import { NewRowID, RowPageSize } from "../lib/constants" import { tick } from "svelte" +import { Helpers } from "@budibase/bbui" export const createStores = () => { const rows = writable([]) @@ -413,6 +414,13 @@ export const createActions = context => { let newRow for (let i = 0; i < newRows.length; i++) { newRow = newRows[i] + + // Ensure we have a unique _id. + // This means generating one for non DS+. + if (!newRow._id) { + newRow._id = Helpers.hashString(JSON.stringify(newRow)) + } + if (!rowCacheMap[newRow._id]) { rowCacheMap[newRow._id] = true rowsToAppend.push(newRow) diff --git a/packages/frontend-core/src/components/grid/stores/table.js b/packages/frontend-core/src/components/grid/stores/table.js index ed13609f45..9a4058ac8f 100644 --- a/packages/frontend-core/src/components/grid/stores/table.js +++ b/packages/frontend-core/src/components/grid/stores/table.js @@ -3,11 +3,7 @@ import { get } from "svelte/store" const SuppressErrors = true export const createActions = context => { - const { definition, API, datasource, columns, stickyColumn } = context - - const refreshDefinition = async () => { - definition.set(await API.fetchTableDefinition(get(datasource).tableId)) - } + const { API, datasource, columns, stickyColumn } = context const saveDefinition = async newDefinition => { await API.saveTable(newDefinition) @@ -52,7 +48,6 @@ export const createActions = context => { return { table: { actions: { - refreshDefinition, saveDefinition, addRow: saveRow, updateRow: saveRow, diff --git a/packages/frontend-core/src/components/grid/stores/viewV2.js b/packages/frontend-core/src/components/grid/stores/viewV2.js index b9a4bc099b..000727b262 100644 --- a/packages/frontend-core/src/components/grid/stores/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/viewV2.js @@ -3,20 +3,7 @@ import { get } from "svelte/store" const SuppressErrors = true export const createActions = context => { - const { definition, API, datasource, columns, stickyColumn } = context - - const refreshDefinition = async () => { - const $datasource = get(datasource) - if (!$datasource) { - definition.set(null) - return - } - const table = await API.fetchTableDefinition($datasource.tableId) - const view = Object.values(table?.views || {}).find( - view => view.id === $datasource.id - ) - definition.set(view) - } + const { API, datasource, columns, stickyColumn } = context const saveDefinition = async newDefinition => { await API.viewV2.update(newDefinition) @@ -61,7 +48,6 @@ export const createActions = context => { return { viewV2: { actions: { - refreshDefinition, saveDefinition, addRow: saveRow, updateRow: saveRow, diff --git a/packages/frontend-core/src/fetch/fetchData.js b/packages/frontend-core/src/fetch/index.js similarity index 72% rename from packages/frontend-core/src/fetch/fetchData.js rename to packages/frontend-core/src/fetch/index.js index 063dd02cbf..0edd07762b 100644 --- a/packages/frontend-core/src/fetch/fetchData.js +++ b/packages/frontend-core/src/fetch/index.js @@ -24,7 +24,18 @@ const DataFetchMap = { jsonarray: JSONArrayFetch, } +// Constructs a new fetch model for a certain datasource export const fetchData = ({ API, datasource, options }) => { const Fetch = DataFetchMap[datasource?.type] || TableFetch return new Fetch({ API, datasource, ...options }) } + +// Fetches the definition of any type of datasource +export const getDatasourceDefinition = async ({ API, datasource }) => { + const handler = DataFetchMap[datasource?.type] + if (!handler) { + return null + } + const instance = new handler({ API }) + return await instance.getDefinition(datasource) +} diff --git a/packages/frontend-core/src/index.js b/packages/frontend-core/src/index.js index b0afc0c25d..f51be616f8 100644 --- a/packages/frontend-core/src/index.js +++ b/packages/frontend-core/src/index.js @@ -1,5 +1,5 @@ export { createAPIClient } from "./api" -export { fetchData } from "./fetch/fetchData" +export { fetchData } from "./fetch" export { Utils } from "./utils" export * as Constants from "./constants" export * from "./stores" From c4a516ccb37e838038011ba39d198e65a1ca2c79 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 4 Oct 2023 09:25:35 +0100 Subject: [PATCH 02/37] Ensure grid schema structure is predictable and prevent copying IDs for query datasources --- .../components/grid/cells/HeaderCell.svelte | 7 +-- .../grid/overlays/MenuOverlay.svelte | 4 +- .../src/components/grid/stores/columns.js | 33 +++++++------ .../src/components/grid/stores/datasource.js | 48 +++++++++++-------- .../src/components/grid/stores/rows.js | 2 +- .../src/components/grid/stores/sort.js | 15 +++--- 6 files changed, 61 insertions(+), 48 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index a58b4c8fe4..5ac70c93c8 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -22,6 +22,7 @@ columns, definition, datasource, + schema, } = getContext("grid") const bannedDisplayColumnTypes = [ @@ -126,16 +127,16 @@ // Generate new name let newName = `${column.name} copy` let attempts = 2 - while ($definition.schema[newName]) { + while ($schema[newName]) { newName = `${column.name} copy ${attempts++}` } // Save schema with new column - const existingColumnDefinition = $definition.schema[column.name] + const existingColumnDefinition = $schema[column.name] await datasource.actions.saveDefinition({ ...$definition, schema: { - ...$definition.schema, + ...$schema, [newName]: { ...existingColumnDefinition, name: newName, diff --git a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte index cbf2c6ee4e..7fb2bb138d 100644 --- a/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte +++ b/packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte @@ -75,7 +75,9 @@ copyToClipboard($focusedRow?._id)} on:click={menu.actions.close} > diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 629d5dd893..c36afea7a1 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -69,7 +69,7 @@ export const deriveStores = context => { } export const createActions = context => { - const { columns, stickyColumn, datasource, definition } = context + const { columns, stickyColumn, datasource, definition, schema } = context // Updates the datasources primary display column const changePrimaryDisplay = async column => { @@ -101,7 +101,7 @@ export const createActions = context => { const $columns = get(columns) const $definition = get(definition) const $stickyColumn = get(stickyColumn) - const newSchema = cloneDeep($definition.schema) + let newSchema = cloneDeep(get(schema)) || {} // Build new updated datasource schema Object.keys(newSchema).forEach(column => { @@ -142,11 +142,11 @@ export const createActions = context => { } export const initialise = context => { - const { definition, columns, stickyColumn, schema } = context + const { definition, columns, stickyColumn, enrichedSchema } = context // Merge new schema fields with existing schema in order to preserve widths - schema.subscribe($schema => { - if (!$schema) { + enrichedSchema.subscribe($enrichedSchema => { + if (!$enrichedSchema) { columns.set([]) stickyColumn.set(null) return @@ -155,13 +155,16 @@ export const initialise = context => { // Find primary display let primaryDisplay - if ($definition.primaryDisplay && $schema[$definition.primaryDisplay]) { + if ( + $definition.primaryDisplay && + $enrichedSchema[$definition.primaryDisplay] + ) { primaryDisplay = $definition.primaryDisplay } // Get field list let fields = [] - Object.keys($schema).forEach(field => { + Object.keys($enrichedSchema).forEach(field => { if (field !== primaryDisplay) { fields.push(field) } @@ -172,11 +175,11 @@ export const initialise = context => { fields .map(field => ({ name: field, - label: $schema[field].displayName || field, - schema: $schema[field], - width: $schema[field].width || DefaultColumnWidth, - visible: $schema[field].visible ?? true, - order: $schema[field].order, + label: $enrichedSchema[field].displayName || field, + schema: $enrichedSchema[field], + width: $enrichedSchema[field].width || DefaultColumnWidth, + visible: $enrichedSchema[field].visible ?? true, + order: $enrichedSchema[field].order, })) .sort((a, b) => { // Sort by order first @@ -207,9 +210,9 @@ export const initialise = context => { } stickyColumn.set({ name: primaryDisplay, - label: $schema[primaryDisplay].displayName || primaryDisplay, - schema: $schema[primaryDisplay], - width: $schema[primaryDisplay].width || DefaultColumnWidth, + label: $enrichedSchema[primaryDisplay].displayName || primaryDisplay, + schema: $enrichedSchema[primaryDisplay], + width: $enrichedSchema[primaryDisplay].width || DefaultColumnWidth, visible: true, order: 0, left: GutterWidth, diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index e56b37a5f3..0d70424550 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -12,27 +12,36 @@ export const createStores = () => { export const deriveStores = context => { const { definition, schemaOverrides, columnWhitelist } = context - const schema = derived( - [definition, schemaOverrides, columnWhitelist], - ([$definition, $schemaOverrides, $columnWhitelist]) => { - if (!$definition?.schema) { + const schema = derived(definition, $definition => { + let schema = $definition?.schema + if (!schema) { + return null + } + + // Ensure schema is configured as objects. + // Certain datasources like queries use primitives. + Object.keys(schema || {}).forEach(key => { + if (typeof schema[key] !== "object") { + schema[key] = { type: schema[key] } + } + }) + + return schema + }) + + const enrichedSchema = derived( + [schema, schemaOverrides, columnWhitelist], + ([$schema, $schemaOverrides, $columnWhitelist]) => { + if (!$schema) { return null } - let newSchema = { ...$definition?.schema } - - // Ensure schema is configured as objects. - // Certain datasources like queries use primitives. - Object.keys(newSchema).forEach(key => { - if (typeof newSchema[key] !== "object") { - newSchema[key] = { type: newSchema[key] } - } - }) + let enrichedSchema = { ...$schema } // Apply schema overrides Object.keys($schemaOverrides || {}).forEach(field => { - if (newSchema[field]) { - newSchema[field] = { - ...newSchema[field], + if (enrichedSchema[field]) { + enrichedSchema[field] = { + ...enrichedSchema[field], ...$schemaOverrides[field], } } @@ -40,19 +49,20 @@ export const deriveStores = context => { // Apply whitelist if specified if ($columnWhitelist?.length) { - Object.keys(newSchema).forEach(key => { + Object.keys(enrichedSchema).forEach(key => { if (!$columnWhitelist.includes(key)) { - delete newSchema[key] + delete enrichedSchema[key] } }) } - return newSchema + return enrichedSchema } ) return { schema, + enrichedSchema, } } diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 2e2832f209..1eafb20756 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -418,7 +418,7 @@ export const createActions = context => { // Ensure we have a unique _id. // This means generating one for non DS+. if (!newRow._id) { - newRow._id = Helpers.hashString(JSON.stringify(newRow)) + newRow._id = `fake-${Helpers.hashString(JSON.stringify(newRow))}` } if (!rowCacheMap[newRow._id]) { diff --git a/packages/frontend-core/src/components/grid/stores/sort.js b/packages/frontend-core/src/components/grid/stores/sort.js index 734a876eed..336570d012 100644 --- a/packages/frontend-core/src/components/grid/stores/sort.js +++ b/packages/frontend-core/src/components/grid/stores/sort.js @@ -17,7 +17,7 @@ export const createStores = context => { } export const initialise = context => { - const { sort, initialSortColumn, initialSortOrder, definition } = context + const { sort, initialSortColumn, initialSortOrder, schema } = context // Reset sort when initial sort props change initialSortColumn.subscribe(newSortColumn => { @@ -28,15 +28,12 @@ export const initialise = context => { }) // Derive if the current sort column exists in the schema - const sortColumnExists = derived( - [sort, definition], - ([$sort, $definition]) => { - if (!$sort?.column || !$definition) { - return true - } - return $definition.schema?.[$sort.column] != null + const sortColumnExists = derived([sort, schema], ([$sort, $schema]) => { + if (!$sort?.column || !$schema) { + return true } - ) + return $schema[$sort.column] != null + }) // Clear sort state if our sort column does not exist sortColumnExists.subscribe(exists => { From 77f87af87f3f169b0b90b9fa39b4c102991bb956 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 4 Oct 2023 09:36:42 +0100 Subject: [PATCH 03/37] Improve grid handling of invalid datasources and fix potential error when encoutering invalid datasources --- .../src/components/grid/layout/Grid.svelte | 16 ++++++++-------- .../src/components/grid/stores/rows.js | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 44e0a972f1..e2ecd0f968 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -141,7 +141,14 @@ {/if} - {#if $loaded} + {#if $error} +
+
There was a problem loading your grid
+
+ {$error} +
+
+ {:else if $loaded}
@@ -171,13 +178,6 @@
- {:else if $error} -
-
There was a problem loading your grid
-
- {$error} -
-
{/if} {#if $loading && !$error}
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 1eafb20756..e6251a5afa 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -94,12 +94,14 @@ export const createActions = context => { datasource.subscribe(async $datasource => { // Unsub from previous fetch if one exists unsubscribe?.() + unsubscribe = null fetch.set(null) instanceLoaded.set(false) loading.set(true) // Abandon if we don't have a valid datasource if (!datasource.actions.isDatasourceValid($datasource)) { + error.set("Datasource is invalid") return } From 88c4d0cd203c0b1f8788b2898810c381c5c1af1f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 5 Oct 2023 08:23:01 +0100 Subject: [PATCH 04/37] Update grids to work with all datasources --- .../src/components/grid/stores/config.js | 2 + .../src/components/grid/stores/datasource.js | 12 +++--- .../{query.js => datasources/nonPlus.js} | 38 ++++++++++++------- .../grid/stores/{ => datasources}/table.js | 0 .../grid/stores/{ => datasources}/viewV2.js | 0 .../src/components/grid/stores/index.js | 8 ++-- 6 files changed, 38 insertions(+), 22 deletions(-) rename packages/frontend-core/src/components/grid/stores/{query.js => datasources/nonPlus.js} (64%) rename packages/frontend-core/src/components/grid/stores/{ => datasources}/table.js (100%) rename packages/frontend-core/src/components/grid/stores/{ => datasources}/viewV2.js (100%) diff --git a/packages/frontend-core/src/components/grid/stores/config.js b/packages/frontend-core/src/components/grid/stores/config.js index 0efdc2104e..6da6ebf11e 100644 --- a/packages/frontend-core/src/components/grid/stores/config.js +++ b/packages/frontend-core/src/components/grid/stores/config.js @@ -55,6 +55,8 @@ export const deriveStores = context => { config.canEditRows = false config.canDeleteRows = false config.canExpandRows = false + config.canSaveSchema = false + config.canEditColumns = false } return config diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 0d70424550..6a10cb8b9b 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -75,21 +75,23 @@ export const createActions = context => { dispatch, table, viewV2, - query, + nonPlus, } = context // Gets the appropriate API for the configured datasource type const getAPI = () => { const $datasource = get(datasource) - switch ($datasource?.type) { + const type = $datasource?.type + if (!type) { + return null + } + switch (type) { case "table": return table case "viewV2": return viewV2 - case "query": - return query default: - return null + return nonPlus } } diff --git a/packages/frontend-core/src/components/grid/stores/query.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js similarity index 64% rename from packages/frontend-core/src/components/grid/stores/query.js rename to packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js index 6ac0547d27..ea024643aa 100644 --- a/packages/frontend-core/src/components/grid/stores/query.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js @@ -1,26 +1,32 @@ import { get } from "svelte/store" export const createActions = context => { - const { API, columns, stickyColumn } = context + const { columns, stickyColumn, table, viewV2 } = context - const saveDefinition = async newDefinition => { - await API.saveQuery(newDefinition) + const saveDefinition = async () => { + throw "This datasource does not support updating the definition" } const saveRow = async () => { - throw "Rows cannot be updated through queries" + throw "This datasource does not support saving rows" } const deleteRows = async () => { - throw "Rows cannot be deleted through queries" + throw "This datasource does not support deleting rows" } const getRow = () => { - throw "Queries don't support fetching individual rows" + throw "This datasource does not support fetching individual rows" } const isDatasourceValid = datasource => { - return datasource?.type === "query" && datasource?._id + // There are many different types and shapes of datasource, so we only + // check that we aren't null + return ( + !table.actions.isDatasourceValid(datasource) && + !viewV2.actions.isDatasourceValid(datasource) && + datasource?.type != null + ) } const canUseColumn = name => { @@ -30,7 +36,7 @@ export const createActions = context => { } return { - query: { + nonPlus: { actions: { saveDefinition, addRow: saveRow, @@ -44,18 +50,22 @@ export const createActions = context => { } } +// Small util to compare datasource definitions +const isSameDatasource = (a, b) => { + return JSON.stringify(a) === JSON.stringify(b) +} + export const initialise = context => { const { datasource, sort, filter, - query, + nonPlus, initialFilter, initialSortColumn, initialSortOrder, fetch, } = context - // Keep a list of subscriptions so that we can clear them when the datasource // config changes let unsubscribers = [] @@ -65,7 +75,7 @@ export const initialise = context => { // Clear previous subscriptions unsubscribers?.forEach(unsubscribe => unsubscribe()) unsubscribers = [] - if (!query.actions.isDatasourceValid($datasource)) { + if (!nonPlus.actions.isDatasourceValid($datasource)) { return } @@ -81,7 +91,8 @@ export const initialise = context => { filter.subscribe($filter => { // Ensure we're updating the correct fetch const $fetch = get(fetch) - if ($fetch?.options?.datasource?._id !== $datasource._id) { + if (!isSameDatasource($fetch?.options?.datasource, $datasource)) { + console.log("skip, different ds") return } $fetch.update({ @@ -95,7 +106,8 @@ export const initialise = context => { sort.subscribe($sort => { // Ensure we're updating the correct fetch const $fetch = get(fetch) - if ($fetch?.options?.datasource?._id !== $datasource._id) { + if (!isSameDatasource($fetch?.options?.datasource, $datasource)) { + console.log("skip, different ds") return } $fetch.update({ diff --git a/packages/frontend-core/src/components/grid/stores/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js similarity index 100% rename from packages/frontend-core/src/components/grid/stores/table.js rename to packages/frontend-core/src/components/grid/stores/datasources/table.js diff --git a/packages/frontend-core/src/components/grid/stores/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js similarity index 100% rename from packages/frontend-core/src/components/grid/stores/viewV2.js rename to packages/frontend-core/src/components/grid/stores/datasources/viewV2.js diff --git a/packages/frontend-core/src/components/grid/stores/index.js b/packages/frontend-core/src/components/grid/stores/index.js index 5a46227b49..10fe932aab 100644 --- a/packages/frontend-core/src/components/grid/stores/index.js +++ b/packages/frontend-core/src/components/grid/stores/index.js @@ -15,10 +15,10 @@ import * as Config from "./config" import * as Sort from "./sort" import * as Filter from "./filter" import * as Notifications from "./notifications" -import * as Table from "./table" -import * as ViewV2 from "./viewV2" -import * as Query from "./query" import * as Datasource from "./datasource" +import * as Table from "./datasources/table" +import * as ViewV2 from "./datasources/viewV2" +import * as NonPlus from "./datasources/nonPlus" const DependencyOrderedStores = [ Sort, @@ -27,7 +27,7 @@ const DependencyOrderedStores = [ Scroll, Table, ViewV2, - Query, + NonPlus, Datasource, Columns, Rows, From b75c78dae54f74df60341615580449851297fe87 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 5 Oct 2023 08:24:30 +0100 Subject: [PATCH 05/37] Remove log --- .../src/components/grid/stores/datasources/nonPlus.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js index ea024643aa..e04d05ec59 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js @@ -92,7 +92,6 @@ export const initialise = context => { // Ensure we're updating the correct fetch const $fetch = get(fetch) if (!isSameDatasource($fetch?.options?.datasource, $datasource)) { - console.log("skip, different ds") return } $fetch.update({ @@ -107,7 +106,6 @@ export const initialise = context => { // Ensure we're updating the correct fetch const $fetch = get(fetch) if (!isSameDatasource($fetch?.options?.datasource, $datasource)) { - console.log("skip, different ds") return } $fetch.update({ From 43c30d877beb3b1e0b21b3da617d7ad205534206 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 5 Oct 2023 14:42:26 +0100 Subject: [PATCH 06/37] Add new custom datasource type --- .../settings/controls/DataSourceSelect.svelte | 81 ++++++++++++---- .../src/components/grid/lib/utils.js | 2 +- .../src/components/grid/stores/rows.js | 2 +- .../frontend-core/src/fetch/CustomFetch.js | 97 +++++++++++++++++++ packages/frontend-core/src/fetch/index.js | 2 + 5 files changed, 162 insertions(+), 22 deletions(-) create mode 100644 packages/frontend-core/src/fetch/CustomFetch.js diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte index d70929469a..e4b3f972ba 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte @@ -9,6 +9,7 @@ Heading, Drawer, DrawerContent, + Icon, } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import { store, currentAsset } from "builderStore" @@ -22,6 +23,7 @@ import BindingBuilder from "components/integration/QueryBindingBuilder.svelte" import IntegrationQueryEditor from "components/integration/index.svelte" import { makePropSafe as safe } from "@budibase/string-templates" + import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" export let value = {} export let otherSources @@ -31,9 +33,12 @@ const dispatch = createEventDispatcher() const arrayTypes = ["attachment", "array"] + let anchorRight, dropdownRight let drawer let tmpQueryParams + let tmpCustomData + let customDataValid = true $: text = value?.label ?? "Choose an option" $: tables = $tablesStore.list.map(m => ({ @@ -125,6 +130,10 @@ value: `{{ literal ${runtimeBinding} }}`, } }) + $: custom = { + type: "custom", + label: "Custom", + } const handleSelected = selected => { dispatch("change", selected) @@ -151,6 +160,11 @@ drawer.show() } + const openCustomDrawer = () => { + tmpCustomData = value.data || "" + drawer.show() + } + const getQueryValue = queries => { return queries.find(q => q._id === value._id) || value } @@ -162,6 +176,14 @@ }) drawer.hide() } + + const saveCustomData = () => { + handleSelected({ + ...value, + data: tmpCustomData, + }) + drawer.hide() + }
@@ -172,7 +194,9 @@ on:click={dropdownRight.show} /> {#if value?.type === "query"} - +
+ +
@@ -198,6 +222,31 @@ {/if} + {#if value?.type === "custom"} +
+ +
+ + +
+ Provide a JavaScript or JSON array to use as data +
+ (tmpCustomData = event.detail)} + {bindings} + allowJS + allowHelpers + /> + + {/if}
@@ -285,17 +334,18 @@ {/each} {/if} - {#if otherSources?.length} - -
- Other -
-
    + +
    + Other +
    +
      +
    • handleSelected(custom)}>{custom.label}
      • + {#if otherSources?.length} {#each otherSources as source}
    • handleSelected(source)}>{source.label}
      • {/each} -
    - {/if} + {/if} +
@@ -340,16 +390,7 @@ background-color: var(--spectrum-global-color-gray-200); } - i { - margin-left: 5px; - display: flex; - align-items: center; - transition: all 0.2s; - } - - i:hover { - transform: scale(1.1); - font-weight: 600; - cursor: pointer; + .icon { + margin-left: 8px; } diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js index 9383f69f66..d3898a3b18 100644 --- a/packages/frontend-core/src/components/grid/lib/utils.js +++ b/packages/frontend-core/src/components/grid/lib/utils.js @@ -1,6 +1,6 @@ export const getColor = (idx, opacity = 0.3) => { if (idx == null || idx === -1) { - return null + idx = 0 } return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, ${opacity})` } diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index e6251a5afa..5b6f4fd7f9 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -420,7 +420,7 @@ export const createActions = context => { // Ensure we have a unique _id. // This means generating one for non DS+. if (!newRow._id) { - newRow._id = `fake-${Helpers.hashString(JSON.stringify(newRow))}` + newRow._id = `fake-${Math.random()}` } if (!rowCacheMap[newRow._id]) { diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js new file mode 100644 index 0000000000..baa7ce3d4d --- /dev/null +++ b/packages/frontend-core/src/fetch/CustomFetch.js @@ -0,0 +1,97 @@ +import DataFetch from "./DataFetch.js" + +export default class CustomFetch extends DataFetch { + getType(value) { + if (value == null) { + return "string" + } + const type = typeof value + if (type === "object") { + if (Array.isArray(value)) { + return "array" + } + return "json" + } else { + return type + } + } + + // Parses the custom data into an array format + parseCustomData(data) { + if (!data) { + return [] + } + + // Happy path - already an array + if (Array.isArray(data)) { + return data + } + + // Handle string cases + if (typeof data === "string") { + // Try JSON parsing + try { + data = JSON.parse(data) + if (Array.isArray(data)) { + return data + } + } catch (error) { + // Ignore + } + + // Try a simple CSV + return data.split(",").map(x => x.trim()) + } + + // Other cases we just assume it's a single object and wrap it + return [data] + } + + // Enriches the custom data to ensure the structure and format is usable + enrichCustomData(data) { + if (!data?.length) { + return [] + } + + // Filter out any invalid values + data = data.filter(x => x != null && !Array.isArray(x)) + + // Ensure all values are packed into objects + return data.map(x => (typeof x === "object" ? x : { value: x })) + } + + getCustomData(datasource) { + return this.enrichCustomData(this.parseCustomData(datasource?.data)) + } + + async getDefinition(datasource) { + // Try and work out the schema from the array provided + let schema = {} + const data = this.getCustomData(datasource) + + // Go through every object and extract all valid keys + for (let datum of data) { + for (let key of Object.keys(datum)) { + if (key === "_id") { + continue + } + if (!schema[key]) { + schema[key] = { type: this.getType(datum[key]) } + } + } + } + + return { + schema, + } + } + + async getData() { + const { datasource } = this.options + return { + rows: this.getCustomData(datasource), + hasNextPage: false, + cursor: null, + } + } +} diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.js index 0edd07762b..d133942bb7 100644 --- a/packages/frontend-core/src/fetch/index.js +++ b/packages/frontend-core/src/fetch/index.js @@ -8,6 +8,7 @@ import FieldFetch from "./FieldFetch.js" import JSONArrayFetch from "./JSONArrayFetch.js" import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch.js" +import CustomFetch from "./CustomFetch.js" const DataFetchMap = { table: TableFetch, @@ -17,6 +18,7 @@ const DataFetchMap = { link: RelationshipFetch, user: UserFetch, groupUser: GroupUserFetch, + custom: CustomFetch, // Client specific datasource types provider: NestedProviderFetch, From 9667c954ef79288a42838c0a72c8bea6490d9a63 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 5 Oct 2023 17:55:57 +0100 Subject: [PATCH 07/37] Enable uploading a CSV file as a datasource and fix edge cases --- .../settings/controls/DataSourceSelect.svelte | 44 ++++++++++++++++--- .../src/components/grid/stores/datasource.js | 6 +++ .../grid/stores/datasources/nonPlus.js | 9 ++++ .../grid/stores/datasources/table.js | 8 ++++ .../grid/stores/datasources/viewV2.js | 8 ++++ .../src/components/grid/stores/rows.js | 7 ++- .../frontend-core/src/fetch/CustomFetch.js | 6 +-- 7 files changed, 77 insertions(+), 11 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte index e4b3f972ba..de9a37dfae 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte @@ -10,6 +10,10 @@ Drawer, DrawerContent, Icon, + Modal, + ModalContent, + CoreDropzone, + notifications, } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import { store, currentAsset } from "builderStore" @@ -24,6 +28,7 @@ import IntegrationQueryEditor from "components/integration/index.svelte" import { makePropSafe as safe } from "@budibase/string-templates" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" + import { API } from "api" export let value = {} export let otherSources @@ -39,6 +44,7 @@ let tmpQueryParams let tmpCustomData let customDataValid = true + let modal $: text = value?.label ?? "Choose an option" $: tables = $tablesStore.list.map(m => ({ @@ -184,6 +190,26 @@ }) drawer.hide() } + + const promptForCSV = () => { + drawer.hide() + modal.show() + } + + const handleCSV = async e => { + try { + const csv = await e.detail[0]?.text() + if (csv?.length) { + const js = await API.csvToJson(csv) + tmpCustomData = JSON.stringify(js) + } + } catch (error) { + console.log(error) + notifications.error("Failed to parse CSV") + } + modal.hide() + drawer.show() + }
@@ -227,12 +253,12 @@
- +
+ + +
Provide a JavaScript or JSON array to use as data
@@ -349,6 +375,12 @@
+ + + + + +