Allow using JSON field arrays as a data provider source and add data bindings for nested JSON fields

packages/builder/src/builderStore/dataBinding.js

@@ -5,7 +5,7 @@ import {
   findComponent,
   findComponentPath,
   getComponentSettings,
-} from "./storeUtils"
+} from "./componentUtils"
 import { store } from "builderStore"
 import { queries as queriesStores, tables as tablesStore } from "stores/backend"
 import {
@@ -15,6 +15,7 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import { TableNames } from "../constants"
+import { convertJSONSchemaToTableSchema } from "./jsonUtils"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
@@ -186,6 +187,7 @@ const getProviderContextBindings = (asset, dataProviders) => {
       }
 
       let schema
+      let table
       let readablePrefix
       let runtimeSuffix = context.suffix
 
@@ -209,8 +211,17 @@ const getProviderContextBindings = (asset, dataProviders) => {
         }
         const info = getSchemaForDatasource(asset, datasource)
         schema = info.schema
+        table = info.table
+
+        // For JSON arrays, use the array name as the readable prefix.
+        // Otherwise use the table name
+        if (datasource.type === "jsonarray") {
+          const split = datasource.label.split(".")
+          readablePrefix = split[split.length - 1]
+        } else {
           readablePrefix = info.table?.name
         }
+      }
       if (!schema) {
         return
       }
@@ -229,7 +240,8 @@ const getProviderContextBindings = (asset, dataProviders) => {
         const fieldSchema = schema[key]
 
         // Make safe runtime binding
-        const runtimeBinding = `${safeComponentId}.${makePropSafe(key)}`
+        const safeKey = key.split(".").map(makePropSafe).join(".")
+        const runtimeBinding = `${safeComponentId}.${safeKey}`
 
         // Optionally use a prefix with readable bindings
         let readableBinding = component._instanceName
@@ -247,6 +259,8 @@ const getProviderContextBindings = (asset, dataProviders) => {
           // datasource options, based on bindable properties
           fieldSchema,
           providerId,
+          // Table ID is used by JSON fields to know what table the field is in
+          tableId: table?._id,
         })
       })
     })
@@ -347,16 +361,26 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
 
   if (datasource) {
     const { type } = datasource
+    const tables = get(tablesStore).list
 
-    // Determine the source table from the datasource type
+    // Determine the entity which backs this datasource.
+    // "provider" datasources are those targeting another data provider
     if (type === "provider") {
       const component = findComponent(asset.props, datasource.providerId)
       const source = getDatasourceForProvider(asset, component)
       return getSchemaForDatasource(asset, source, isForm)
-    } else if (type === "query") {
+    }
+
+    // "query" datasources are those targeting non-plus datasources or
+    // custom queries
+    else if (type === "query") {
       const queries = get(queriesStores).list
       table = queries.find(query => query._id === datasource._id)
-    } else if (type === "field") {
+    }
+
+    // "field" datasources are array-like fields of rows, such as attachments
+    // or multi-select fields
+    else if (type === "field") {
       table = { name: datasource.fieldName }
       const { fieldType } = datasource
       if (fieldType === "attachment") {
@@ -375,12 +399,26 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
           },
         }
       }
-    } else {
+    }
-      const tables = get(tablesStore).list
+
+    // "jsonarray" datasources are arrays inside JSON fields
+    else if (type === "jsonarray") {
+      table = tables.find(table => table._id === datasource.tableId)
+      const keysToSchema = datasource.label.split(".").slice(2)
+      let jsonSchema = table?.schema
+      for (let i = 0; i < keysToSchema.length; i++) {
+        jsonSchema = jsonSchema[keysToSchema[i]].schema
+      }
+      schema = convertJSONSchemaToTableSchema(jsonSchema, true)
+    }
+
+    // Otherwise we assume we're targeting an internal table or a plus
+    // datasource, and we can treat it as a table with a schema
+    else {
       table = tables.find(table => table._id === datasource.tableId)
     }
 
-    // Determine the schema from the table if not already determined
+    // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
         schema = cloneDeep(table.views?.[datasource.name]?.schema)

packages/builder/src/builderStore/index.js

@@ -4,7 +4,7 @@ import { getHostingStore } from "./store/hosting"
 import { getThemeStore } from "./store/theme"
 import { derived, writable } from "svelte/store"
 import { FrontendTypes, LAYOUT_NAMES } from "../constants"
-import { findComponent } from "./storeUtils"
+import { findComponent } from "./componentUtils"
 
 export const store = getFrontendStore()
 export const automationStore = getAutomationStore()

packages/builder/src/builderStore/jsonUtils.js

@@ -0,0 +1,36 @@
+export const convertJSONSchemaToTableSchema = jsonSchema => {
+  if (!jsonSchema) {
+    return null
+  }
+  if (jsonSchema.schema) {
+    jsonSchema = jsonSchema.schema
+  }
+  const keys = extractJSONSchemaKeys(jsonSchema)
+  let schema = {}
+  keys.forEach(({ key, type }) => {
+    schema[key] = { type, name: key }
+  })
+  return schema
+}
+
+const extractJSONSchemaKeys = (jsonSchema, squashObjects = false) => {
+  if (!jsonSchema || !Object.keys(jsonSchema).length) {
+    return []
+  }
+  let keys = []
+  Object.keys(jsonSchema).forEach(key => {
+    const type = jsonSchema[key].type
+    if (type === "json" && squashObjects) {
+      const childKeys = extractJSONSchemaKeys(jsonSchema[key].schema)
+      keys = keys.concat(
+        childKeys.map(childKey => ({
+          key: `${key}.${childKey.key}`,
+          type: childKey.type,
+        }))
+      )
+    } else if (type !== "array") {
+      keys.push({ key, type })
+    }
+  })
+  return keys
+}

packages/builder/src/builderStore/store/frontend.js

@@ -26,7 +26,7 @@ import {
   findAllMatchingComponents,
   findComponent,
   getComponentSettings,
-} from "../storeUtils"
+} from "../componentUtils"
 import { uuid } from "../uuid"
 import { removeBindings } from "../dataBinding"
 

packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte

@@ -14,7 +14,7 @@
     notifications,
   } from "@budibase/bbui"
   import ErrorSVG from "assets/error.svg?raw"
-  import { findComponent, findComponentPath } from "builderStore/storeUtils"
+  import { findComponent, findComponentPath } from "builderStore/componentUtils"
 
   let iframe
   let layout

packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte

@@ -2,7 +2,7 @@
   import { get } from "svelte/store"
   import { store, currentAsset } from "builderStore"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { findComponentParent } from "builderStore/storeUtils"
+  import { findComponentParent } from "builderStore/componentUtils"
   import { ActionMenu, MenuItem, Icon, notifications } from "@budibase/bbui"
 
   export let component

packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/dragDropStore.js

@@ -1,6 +1,6 @@
 import { writable, get } from "svelte/store"
 import { store as frontendStore } from "builderStore"
-import { findComponentPath } from "builderStore/storeUtils"
+import { findComponentPath } from "builderStore/componentUtils"
 
 export const DropEffect = {
   MOVE: "move",

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte

@@ -16,7 +16,7 @@
   import { selectedComponent } from "builderStore"
   import { getComponentForSettingType } from "./componentSettings"
   import PropertyControl from "./PropertyControl.svelte"
-  import { getComponentSettings } from "builderStore/storeUtils"
+  import { getComponentSettings } from "builderStore/componentUtils"
 
   export let conditions = []
   export let bindings = []

packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataProviderSelect.svelte

@@ -2,7 +2,7 @@
   import { Select } from "@budibase/bbui"
   import { makePropSafe } from "@budibase/string-templates"
   import { currentAsset, store } from "builderStore"
-  import { findComponentPath } from "builderStore/storeUtils"
+  import { findComponentPath } from "builderStore/componentUtils"
   import { createEventDispatcher, onMount } from "svelte"
 
   export let value

packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte

@@ -20,7 +20,10 @@
   import { notifications } from "@budibase/bbui"
   import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
   import IntegrationQueryEditor from "components/integration/index.svelte"
-  import { makePropSafe as safe } from "@budibase/string-templates"
+  import {
+    makePropSafe,
+    makePropSafe as safe,
+  } from "@budibase/string-templates"
 
   export let value = {}
   export let otherSources
@@ -48,9 +51,7 @@
     return [...acc, ...viewsArr]
   }, [])
   $: queries = $queriesStore.list
-    .filter(
+    .filter(q => showAllQueries || q.queryVerb === "read" || q.readable)
-      query => showAllQueries || query.queryVerb === "read" || query.readable
-    )
     .map(query => ({
       label: query.name,
       name: query.name,
@@ -104,13 +105,60 @@
         value: `{{ literal ${runtimeBinding} }}`,
       }
     })
+  $: jsonArrays = findJSONArrays(bindings)
 
-  function handleSelected(selected) {
+  const findJSONArrays = bindings => {
+    let arrays = []
+    const jsonBindings = bindings.filter(x => x.fieldSchema?.type === "json")
+    jsonBindings.forEach(binding => {
+      const {
+        providerId,
+        readableBinding,
+        runtimeBinding,
+        fieldSchema,
+        tableId,
+      } = binding
+      const { name, type } = fieldSchema
+      const schemaArrays = findArraysInSchema(fieldSchema).map(path => {
+        const safePath = path.split(".").map(makePropSafe).join(".")
+        return {
+          providerId,
+          label: `${readableBinding}.${path}`,
+          fieldName: name,
+          fieldType: type,
+          tableId,
+          type: "jsonarray",
+          value: `{{ literal ${runtimeBinding}.${safePath} }}`,
+        }
+      })
+      arrays = arrays.concat(schemaArrays)
+    })
+
+    return arrays
+  }
+
+  const findArraysInSchema = (schema, path) => {
+    if (!schema?.schema || !Object.keys(schema.schema).length) {
+      return []
+    }
+    if (schema.type === "array") {
+      return [path]
+    }
+    let arrays = []
+    Object.keys(schema.schema).forEach(key => {
+      const newPath = `${path ? `${path}.` : ""}${key}`
+      const childArrays = findArraysInSchema(schema.schema[key], newPath)
+      arrays = arrays.concat(childArrays)
+    })
+    return arrays
+  }
+
+  const handleSelected = selected => {
     dispatch("change", selected)
     dropdownRight.hide()
   }
 
-  function fetchQueryDefinition(query) {
+  const fetchQueryDefinition = query => {
     const source = $datasources.list.find(
       ds => ds._id === query.datasourceId
     ).source
@@ -227,6 +275,17 @@
         {/each}
       </ul>
     {/if}
+    {#if jsonArrays?.length}
+      <Divider size="S" />
+      <div class="title">
+        <Heading size="XS">Key/Value Arrays</Heading>
+      </div>
+      <ul>
+        {#each jsonArrays as field}
+          <li on:click={() => handleSelected(field)}>{field.label}</li>
+        {/each}
+      </ul>
+    {/if}
     {#if dataProviders?.length}
       <Divider size="S" />
       <div class="title">

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -5,7 +5,7 @@
     getSchemaForDatasource,
   } from "builderStore/dataBinding"
   import { currentAsset } from "builderStore"
-  import { findClosestMatchingComponent } from "builderStore/storeUtils"
+  import { findClosestMatchingComponent } from "builderStore/componentUtils"
 
   export let componentInstance
   export let value

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ResetFieldsButton.svelte

@@ -1,7 +1,7 @@
 <script>
   import { ActionButton } from "@budibase/bbui"
   import { currentAsset, store } from "builderStore"
-  import { findClosestMatchingComponent } from "builderStore/storeUtils"
+  import { findClosestMatchingComponent } from "builderStore/componentUtils"
   import { makeDatasourceFormComponents } from "builderStore/store/screenTemplates/utils/commonComponents"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ValidationEditor/ValidationDrawer.svelte

@@ -11,7 +11,7 @@
     DatePicker,
   } from "@budibase/bbui"
   import { currentAsset, selectedComponent } from "builderStore"
-  import { findClosestMatchingComponent } from "builderStore/storeUtils"
+  import { findClosestMatchingComponent } from "builderStore/componentUtils"
   import { getSchemaForDatasource } from "builderStore/dataBinding"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import { generate } from "shortid"

packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte

@@ -13,7 +13,7 @@
   import FrontendNavigatePane from "components/design/NavigationPanel/FrontendNavigatePane.svelte"
   import { goto, leftover, params } from "@roxi/routify"
   import { FrontendTypes } from "constants"
-  import { findComponent, findComponentPath } from "builderStore/storeUtils"
+  import { findComponent, findComponentPath } from "builderStore/componentUtils"
   import { get } from "svelte/store"
   import AppThemeSelect from "components/design/AppPreview/AppThemeSelect.svelte"
   import ThemeEditor from "components/design/AppPreview/ThemeEditor.svelte"

packages/client/src/api/datasources.js

@@ -4,6 +4,7 @@ import { fetchViewData } from "./views"
 import { fetchRelationshipData } from "./relationships"
 import { FieldTypes } from "../constants"
 import { executeQuery, fetchQueryDefinition } from "./queries"
+import { convertJSONSchemaToTableSchema } from "builder/src/builderStore/jsonUtils"
 
 /**
  * Fetches all rows for a particular Budibase data source.
@@ -75,6 +76,18 @@ export const fetchDatasourceSchema = async dataSource => {
     }
   }
 
+  // JSON arrays need their table definitions fetched.
+  // We can then extract their schema as a subset of the table schema.
+  if (type === "jsonarray") {
+    const table = await fetchTableDefinition(dataSource.tableId)
+    const keysToSchema = dataSource.label.split(".").slice(2)
+    let schema = table?.schema
+    for (let i = 0; i < keysToSchema.length; i++) {
+      schema = schema[keysToSchema[i]].schema
+    }
+    return convertJSONSchemaToTableSchema(schema)
+  }
+
   // Tables, views and links can be fetched by table ID
   if (
     (type === "table" || type === "view" || type === "link") &&

packages/client/src/components/app/DataProvider.svelte

@@ -203,6 +203,9 @@
       } else {
         allRows = data
       }
+    } else if (dataSource?.type === "jsonarray") {
+      // JSON array sources will be available from context
+      allRows = dataSource?.value || []
     } else {
       // For other data sources like queries or views, fetch all rows from the
       // server