Update datasource schema generation to take new flags, and update automation blocks to use core databinding utilities

packages/builder/src/builderStore/dataBinding.js

@@ -275,10 +275,7 @@ const getProviderContextBindings = (asset, dataProviders) => {
  */
 const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForDatasource(null, {
-    type: "table",
-    tableId: TableNames.USERS,
-  })
+  const { schema } = getSchemaForTable(TableNames.USERS)
   const keys = Object.keys(schema).sort()
   const safeUser = makePropSafe("user")
   keys.forEach(key => {
@@ -385,9 +382,33 @@ export const getButtonContextBindings = (actions, actionId) => {
 }
 
 /**
- * Gets a schema for a datasource object.
+ * Gets the schema for a certain table ID.
+ * The options which can be passed in are:
+ *   formSchema: whether the schema is for a form
+ *   searchableSchema: whether to generate a searchable schema, which may have
+ *     fewer fields than a readable schema
+ * @param tableId the table ID to get the schema for
+ * @param options options for generating the schema
+ * @return {{schema: Object, table: Object}}
  */
-export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
+export const getSchemaForTable = (tableId, options) => {
+  return getSchemaForDatasource(null, { type: "table", tableId }, options)
+}
+
+/**
+ * Gets a schema for a datasource object.
+ * The options which can be passed in are:
+ *   formSchema: whether the schema is for a form
+ *   searchableSchema: whether to generate a searchable schema, which may have
+ *     fewer fields than a readable schema
+ * @param asset the current root client app asset (layout or screen). This is
+ *   optional and only needed for "provider" datasource types.
+ * @param datasource the datasource definition
+ * @param options options for generating the schema
+ * @return {{schema: Object, table: Object}}
+ */
+export const getSchemaForDatasource = (asset, datasource, options) => {
+  options = options || {}
   let schema, table
 
   if (datasource) {
@@ -399,7 +420,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     if (type === "provider") {
       const component = findComponent(asset.props, datasource.providerId)
       const source = getDatasourceForProvider(asset, component)
-      return getSchemaForDatasource(asset, source, isForm)
+      return getSchemaForDatasource(asset, source, options)
     }
 
     // "query" datasources are those targeting non-plus datasources or
@@ -448,8 +469,16 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
+        // For views, the schema is pulled from the `views` property of the
+        // table
         schema = cloneDeep(table.views?.[datasource.name]?.schema)
-      } else if (type === "query" && isForm) {
+      } else if (
+        type === "query" &&
+        (options.formSchema || options.searchableSchema)
+      ) {
+        // For queries, if we are generating a schema for a form or a searchable
+        // schema then we want to use the query parameters rather than the
+        // query schema
         schema = {}
         const params = table.parameters || []
         params.forEach(param => {
@@ -458,6 +487,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
           }
         })
       } else {
+        // Otherwise we just want the schema of the table
         schema = cloneDeep(table.schema)
       }
     }
@@ -485,9 +515,31 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
       schema = { ...schema, ...jsonAdditions }
     }
 
-    // Add _id and _rev fields for certain types
-    if (schema && !isForm && ["table", "link"].includes(datasource.type)) {
+    // Determine if we should add ID and rev to the schema
+    const isInternal = table && !table.sql
+    const isTable = ["table", "link"].includes(datasource.type)
+
+    // ID is part of the readable schema for all tables
+    // Rev is part of the readable schema for internal tables only
+    let addId = isTable
+    let addRev = isTable && isInternal
+
+    // Don't add ID or rev for form schemas
+    if (options.formSchema) {
+      addId = false
+      addRev = false
+    }
+
+    // ID is only searchable for internal tables
+    else if (options.searchableSchema) {
+      addId = isTable && isInternal
+    }
+
+    // Add schema properties if required
+    if (addId) {
       schema["_id"] = { type: "string" }
+    }
+    if (addRev) {
       schema["_rev"] = { type: "string" }
     }
 

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -141,7 +141,9 @@ const fieldTypeToComponentMap = {
 }
 
 export function makeDatasourceFormComponents(datasource) {
-  const { schema } = getSchemaForDatasource(null, datasource, true)
+  const { schema } = getSchemaForDatasource(null, datasource, {
+    formSchema: true,
+  })
   let components = []
   let fields = Object.keys(schema || {})
   fields.forEach(field => {

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -28,8 +28,8 @@
   import { debounce } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
-  // need the client lucene builder to convert to the structure API expects
   import { LuceneUtils } from "@budibase/frontend-core"
+  import { getSchemaForTable } from "builderStore/dataBinding"
 
   export let block
   export let testData
@@ -51,7 +51,8 @@
   $: table = tableId
     ? $tables.list.find(table => table._id === inputData.tableId)
     : { schema: {} }
-  $: schemaFields = table ? Object.values(table.schema) : []
+  $: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
+  $: schemaFields = Object.values(schema || {})
 
   const onChange = debounce(async function (e, key) {
     try {
@@ -173,7 +174,7 @@
             slot="body"
             bind:filters={tempFilters}
             {bindings}
-            {table}
+            {schemaFields}
             panel={AutomationBindingPanel}
           />
         </Drawer>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -4,7 +4,7 @@
   import { tables } from "stores/backend"
   import {
     getContextProviderComponents,
-    getSchemaForDatasource,
+    getSchemaForTable,
   } from "builderStore/dataBinding"
   import SaveFields from "./SaveFields.svelte"
 
@@ -60,7 +60,7 @@
   }
 
   const getSchemaFields = (asset, tableId) => {
-    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
+    const { schema } = getSchemaForTable(tableId)
     delete schema._id
     delete schema._rev
     return Object.values(schema || {})

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/SaveRow.svelte

@@ -4,7 +4,7 @@
   import { tables } from "stores/backend"
   import {
     getContextProviderComponents,
-    getSchemaForDatasource,
+    getSchemaForTable,
   } from "builderStore/dataBinding"
   import SaveFields from "./SaveFields.svelte"
 
@@ -60,7 +60,7 @@
   }
 
   const getSchemaFields = (asset, tableId) => {
-    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
+    const { schema } = getSchemaForTable(tableId)
     return Object.values(schema || {})
   }
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -16,23 +16,17 @@
   import { LuceneUtils, Constants } from "@budibase/frontend-core"
   import { getFields } from "helpers/searchFields"
 
-  // can pass in either the table, or the fields (if working in design,
-  // fields is easier to pass in from the dataBinding:getSchemaForDatasource fn
-  export let table
   export let schemaFields
   export let filters = []
   export let bindings = []
   export let panel = ClientBindingPanel
   export let allowBindings = true
 
-  let fields, enrichedSchemaFields
-  let fieldOptions, valueTypeOptions
-  $: fields = schemaFields ? schemaFields : getSchemaFields(table)
-  $: enrichedSchemaFields = getFields(fields || [], { allowLinks: table?.sql })
+  $: enrichedSchemaFields = getFields(schemaFields || [])
   $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
-  function addFilter() {
+  const addFilter = () => {
     filters = [
       ...filters,
       {
@@ -45,30 +39,17 @@
     ]
   }
 
-  function getSchemaFields(table) {
-    const base = table ? Object.values(table.schema) : []
-    // if internal table, can use the _id field as well
-    const isInternal = table && !table.sql
-    if (isInternal && !base.find(field => field.name === "_id")) {
-      base.push({ name: "_id", type: "string" })
-    }
-    if (isInternal && !base.find(field => field.name === "_rev")) {
-      base.push({ name: "_rev", type: "string" })
-    }
-    return base
-  }
-
-  function removeFilter(id) {
+  const removeFilter = id => {
     filters = filters.filter(field => field.id !== id)
   }
 
-  function duplicateFilter(id) {
+  const duplicateFilter = id => {
     const existingFilter = filters.find(filter => filter.id === id)
     const duplicate = { ...existingFilter, id: generate() }
     filters = [...filters, duplicate]
   }
 
-  function onFieldChange(expression, field) {
+  const onFieldChange = (expression, field) => {
     // Update the field type
     expression.type = enrichedSchemaFields.find(x => x.name === field)?.type
 
@@ -91,7 +72,7 @@
     }
   }
 
-  function onOperatorChange(expression, operator) {
+  const onOperatorChange = (expression, operator) => {
     const noValueOptions = [
       Constants.OperatorOptions.Empty.value,
       Constants.OperatorOptions.NotEmpty.value,
@@ -102,14 +83,14 @@
     }
   }
 
-  function getFieldOptions(field) {
+  const getFieldOptions = field => {
     const schema = enrichedSchemaFields.find(x => x.name === field)
     return schema?.constraints?.inclusion || []
   }
 </script>
 
 <DrawerContent>
-  <div class="container">
+  <div className="container">
     <Layout noPadding>
       <Body size="S">
         {#if !filters?.length}
@@ -203,6 +184,7 @@
     max-width: 1000px;
     margin: 0 auto;
   }
+
   .fields {
     display: grid;
     column-gap: var(--spacing-l);

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -18,7 +18,9 @@
   let tempValue = value || []
 
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, dataSource, {
+    searchableSchema: true,
+  })?.schema
   $: schemaFields = Object.values(schema || {})
 
   const saveFilter = async () => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -17,7 +17,9 @@
     component => component._component === "@budibase/standard-components/form"
   )
   $: datasource = getDatasourceForProvider($currentAsset, form)
-  $: schema = getSchemaForDatasource($currentAsset, datasource, true).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, {
+    formSchema: true,
+  }).schema
   $: options = getOptions(schema, type)
 
   const getOptions = (schema, type) => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte

@@ -15,7 +15,9 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, {
+    searchableSchema: true,
+  }).schema
   $: options = getOptions(datasource, schema || {})
   $: boundValue = getSelectedOption(value, options)
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ValidationEditor/ValidationDrawer.svelte

@@ -116,6 +116,7 @@
   $: schemaRules = parseRulesFromSchema(field, dataSourceSchema || {})
   $: fieldType = type?.split("/")[1] || "string"
   $: constraintOptions = getConstraintsForType(fieldType)
+
   const getConstraintsForType = type => {
     return ConstraintMap[type]
   }