From cd5d370e7b0767766d18bc8744e53ea00bcbf989 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 6 Dec 2021 11:41:17 +0000 Subject: [PATCH] Allow using JSON field arrays as a data provider source and add data bindings for nested JSON fields --- .../{storeUtils.js => componentUtils.js} | 0 .../builder/src/builderStore/dataBinding.js | 56 ++++++++++++--- packages/builder/src/builderStore/index.js | 2 +- .../builder/src/builderStore/jsonUtils.js | 36 ++++++++++ .../src/builderStore/store/frontend.js | 2 +- .../AppPreview/CurrentItemPreview.svelte | 2 +- .../ComponentDropdownMenu.svelte | 2 +- .../ComponentNavigationTree/dragDropStore.js | 2 +- .../ConditionalUIDrawer.svelte | 2 +- .../DataProviderSelect.svelte | 2 +- .../PropertyControls/DataSourceSelect.svelte | 71 +++++++++++++++++-- .../PropertyControls/FormFieldSelect.svelte | 2 +- .../PropertyControls/ResetFieldsButton.svelte | 2 +- .../ValidationEditor/ValidationDrawer.svelte | 2 +- .../design/[assetType]/_layout.svelte | 2 +- packages/client/src/api/datasources.js | 13 ++++ .../src/components/app/DataProvider.svelte | 3 + 17 files changed, 175 insertions(+), 26 deletions(-) rename packages/builder/src/builderStore/{storeUtils.js => componentUtils.js} (100%) create mode 100644 packages/builder/src/builderStore/jsonUtils.js diff --git a/packages/builder/src/builderStore/storeUtils.js b/packages/builder/src/builderStore/componentUtils.js similarity index 100% rename from packages/builder/src/builderStore/storeUtils.js rename to packages/builder/src/builderStore/componentUtils.js diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index b36613fbc5..f5b8fcbb5f 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -5,7 +5,7 @@ import { findComponent, findComponentPath, getComponentSettings, -} from "./storeUtils" +} from "./componentUtils" import { store } from "builderStore" import { queries as queriesStores, tables as tablesStore } from "stores/backend" import { @@ -15,6 +15,7 @@ import { encodeJSBinding, } from "@budibase/string-templates" import { TableNames } from "../constants" +import { convertJSONSchemaToTableSchema } from "./jsonUtils" // Regex to match all instances of template strings const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g @@ -186,6 +187,7 @@ const getProviderContextBindings = (asset, dataProviders) => { } let schema + let table let readablePrefix let runtimeSuffix = context.suffix @@ -209,7 +211,16 @@ const getProviderContextBindings = (asset, dataProviders) => { } const info = getSchemaForDatasource(asset, datasource) schema = info.schema - readablePrefix = info.table?.name + table = info.table + + // For JSON arrays, use the array name as the readable prefix. + // Otherwise use the table name + if (datasource.type === "jsonarray") { + const split = datasource.label.split(".") + readablePrefix = split[split.length - 1] + } else { + readablePrefix = info.table?.name + } } if (!schema) { return @@ -229,7 +240,8 @@ const getProviderContextBindings = (asset, dataProviders) => { const fieldSchema = schema[key] // Make safe runtime binding - const runtimeBinding = `${safeComponentId}.${makePropSafe(key)}` + const safeKey = key.split(".").map(makePropSafe).join(".") + const runtimeBinding = `${safeComponentId}.${safeKey}` // Optionally use a prefix with readable bindings let readableBinding = component._instanceName @@ -247,6 +259,8 @@ const getProviderContextBindings = (asset, dataProviders) => { // datasource options, based on bindable properties fieldSchema, providerId, + // Table ID is used by JSON fields to know what table the field is in + tableId: table?._id, }) }) }) @@ -347,16 +361,26 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => { if (datasource) { const { type } = datasource + const tables = get(tablesStore).list - // Determine the source table from the datasource type + // Determine the entity which backs this datasource. + // "provider" datasources are those targeting another data provider if (type === "provider") { const component = findComponent(asset.props, datasource.providerId) const source = getDatasourceForProvider(asset, component) return getSchemaForDatasource(asset, source, isForm) - } else if (type === "query") { + } + + // "query" datasources are those targeting non-plus datasources or + // custom queries + else if (type === "query") { const queries = get(queriesStores).list table = queries.find(query => query._id === datasource._id) - } else if (type === "field") { + } + + // "field" datasources are array-like fields of rows, such as attachments + // or multi-select fields + else if (type === "field") { table = { name: datasource.fieldName } const { fieldType } = datasource if (fieldType === "attachment") { @@ -375,12 +399,26 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => { }, } } - } else { - const tables = get(tablesStore).list + } + + // "jsonarray" datasources are arrays inside JSON fields + else if (type === "jsonarray") { + table = tables.find(table => table._id === datasource.tableId) + const keysToSchema = datasource.label.split(".").slice(2) + let jsonSchema = table?.schema + for (let i = 0; i < keysToSchema.length; i++) { + jsonSchema = jsonSchema[keysToSchema[i]].schema + } + schema = convertJSONSchemaToTableSchema(jsonSchema, true) + } + + // Otherwise we assume we're targeting an internal table or a plus + // datasource, and we can treat it as a table with a schema + else { table = tables.find(table => table._id === datasource.tableId) } - // Determine the schema from the table if not already determined + // Determine the schema from the backing entity if not already determined if (table && !schema) { if (type === "view") { schema = cloneDeep(table.views?.[datasource.name]?.schema) diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index f32dedd47e..23704556ad 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -4,7 +4,7 @@ import { getHostingStore } from "./store/hosting" import { getThemeStore } from "./store/theme" import { derived, writable } from "svelte/store" import { FrontendTypes, LAYOUT_NAMES } from "../constants" -import { findComponent } from "./storeUtils" +import { findComponent } from "./componentUtils" export const store = getFrontendStore() export const automationStore = getAutomationStore() diff --git a/packages/builder/src/builderStore/jsonUtils.js b/packages/builder/src/builderStore/jsonUtils.js new file mode 100644 index 0000000000..bc865893e3 --- /dev/null +++ b/packages/builder/src/builderStore/jsonUtils.js @@ -0,0 +1,36 @@ +export const convertJSONSchemaToTableSchema = jsonSchema => { + if (!jsonSchema) { + return null + } + if (jsonSchema.schema) { + jsonSchema = jsonSchema.schema + } + const keys = extractJSONSchemaKeys(jsonSchema) + let schema = {} + keys.forEach(({ key, type }) => { + schema[key] = { type, name: key } + }) + return schema +} + +const extractJSONSchemaKeys = (jsonSchema, squashObjects = false) => { + if (!jsonSchema || !Object.keys(jsonSchema).length) { + return [] + } + let keys = [] + Object.keys(jsonSchema).forEach(key => { + const type = jsonSchema[key].type + if (type === "json" && squashObjects) { + const childKeys = extractJSONSchemaKeys(jsonSchema[key].schema) + keys = keys.concat( + childKeys.map(childKey => ({ + key: `${key}.${childKey.key}`, + type: childKey.type, + })) + ) + } else if (type !== "array") { + keys.push({ key, type }) + } + }) + return keys +} diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index c94c759792..b6227f4bd6 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -26,7 +26,7 @@ import { findAllMatchingComponents, findComponent, getComponentSettings, -} from "../storeUtils" +} from "../componentUtils" import { uuid } from "../uuid" import { removeBindings } from "../dataBinding" diff --git a/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte b/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte index 8e3e2dc21d..2bfed936dd 100644 --- a/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte +++ b/packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte @@ -14,7 +14,7 @@ notifications, } from "@budibase/bbui" import ErrorSVG from "assets/error.svg?raw" - import { findComponent, findComponentPath } from "builderStore/storeUtils" + import { findComponent, findComponentPath } from "builderStore/componentUtils" let iframe let layout diff --git a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte index 56c5eef2ad..8ab8c23134 100644 --- a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte +++ b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte @@ -2,7 +2,7 @@ import { get } from "svelte/store" import { store, currentAsset } from "builderStore" import ConfirmDialog from "components/common/ConfirmDialog.svelte" - import { findComponentParent } from "builderStore/storeUtils" + import { findComponentParent } from "builderStore/componentUtils" import { ActionMenu, MenuItem, Icon, notifications } from "@budibase/bbui" export let component diff --git a/packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/dragDropStore.js b/packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/dragDropStore.js index fed3361fec..aa5cd230e7 100644 --- a/packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/dragDropStore.js +++ b/packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/dragDropStore.js @@ -1,6 +1,6 @@ import { writable, get } from "svelte/store" import { store as frontendStore } from "builderStore" -import { findComponentPath } from "builderStore/storeUtils" +import { findComponentPath } from "builderStore/componentUtils" export const DropEffect = { MOVE: "move", diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte index 7645c69d0d..e303729d0b 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte @@ -16,7 +16,7 @@ import { selectedComponent } from "builderStore" import { getComponentForSettingType } from "./componentSettings" import PropertyControl from "./PropertyControl.svelte" - import { getComponentSettings } from "builderStore/storeUtils" + import { getComponentSettings } from "builderStore/componentUtils" export let conditions = [] export let bindings = [] diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataProviderSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataProviderSelect.svelte index 979443a403..d7118fd3ec 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataProviderSelect.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataProviderSelect.svelte @@ -2,7 +2,7 @@ import { Select } from "@budibase/bbui" import { makePropSafe } from "@budibase/string-templates" import { currentAsset, store } from "builderStore" - import { findComponentPath } from "builderStore/storeUtils" + import { findComponentPath } from "builderStore/componentUtils" import { createEventDispatcher, onMount } from "svelte" export let value diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte index 8f5f7ef807..4c173b619e 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte @@ -20,7 +20,10 @@ import { notifications } from "@budibase/bbui" import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte" import IntegrationQueryEditor from "components/integration/index.svelte" - import { makePropSafe as safe } from "@budibase/string-templates" + import { + makePropSafe, + makePropSafe as safe, + } from "@budibase/string-templates" export let value = {} export let otherSources @@ -48,9 +51,7 @@ return [...acc, ...viewsArr] }, []) $: queries = $queriesStore.list - .filter( - query => showAllQueries || query.queryVerb === "read" || query.readable - ) + .filter(q => showAllQueries || q.queryVerb === "read" || q.readable) .map(query => ({ label: query.name, name: query.name, @@ -104,13 +105,60 @@ value: `{{ literal ${runtimeBinding} }}`, } }) + $: jsonArrays = findJSONArrays(bindings) - function handleSelected(selected) { + const findJSONArrays = bindings => { + let arrays = [] + const jsonBindings = bindings.filter(x => x.fieldSchema?.type === "json") + jsonBindings.forEach(binding => { + const { + providerId, + readableBinding, + runtimeBinding, + fieldSchema, + tableId, + } = binding + const { name, type } = fieldSchema + const schemaArrays = findArraysInSchema(fieldSchema).map(path => { + const safePath = path.split(".").map(makePropSafe).join(".") + return { + providerId, + label: `${readableBinding}.${path}`, + fieldName: name, + fieldType: type, + tableId, + type: "jsonarray", + value: `{{ literal ${runtimeBinding}.${safePath} }}`, + } + }) + arrays = arrays.concat(schemaArrays) + }) + + return arrays + } + + const findArraysInSchema = (schema, path) => { + if (!schema?.schema || !Object.keys(schema.schema).length) { + return [] + } + if (schema.type === "array") { + return [path] + } + let arrays = [] + Object.keys(schema.schema).forEach(key => { + const newPath = `${path ? `${path}.` : ""}${key}` + const childArrays = findArraysInSchema(schema.schema[key], newPath) + arrays = arrays.concat(childArrays) + }) + return arrays + } + + const handleSelected = selected => { dispatch("change", selected) dropdownRight.hide() } - function fetchQueryDefinition(query) { + const fetchQueryDefinition = query => { const source = $datasources.list.find( ds => ds._id === query.datasourceId ).source @@ -227,6 +275,17 @@ {/each} {/if} + {#if jsonArrays?.length} + +
+ Key/Value Arrays +
+ + {/if} {#if dataProviders?.length}
diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte index 5527941bd5..ba54de5478 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte @@ -5,7 +5,7 @@ getSchemaForDatasource, } from "builderStore/dataBinding" import { currentAsset } from "builderStore" - import { findClosestMatchingComponent } from "builderStore/storeUtils" + import { findClosestMatchingComponent } from "builderStore/componentUtils" export let componentInstance export let value diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ResetFieldsButton.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ResetFieldsButton.svelte index d5b82176ce..fa2a0d6088 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ResetFieldsButton.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ResetFieldsButton.svelte @@ -1,7 +1,7 @@