Merge pull request #4073 from Budibase/feature/sql-relationship-filtering

SQL relationship filtering

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -14,6 +14,7 @@
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { generate } from "shortid"
   import { getValidOperatorsForType, OperatorOptions } from "constants/lucene"
+  import { getFields } from "helpers/searchFields"
 
   export let schemaFields
   export let filters = []
@@ -21,11 +22,8 @@
   export let panel = ClientBindingPanel
   export let allowBindings = true
 
-  const BannedTypes = ["link", "attachment", "formula", "json", "jsonarray"]
+  $: enrichedSchemaFields = getFields(schemaFields || [])
-
+  $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
-  $: fieldOptions = (schemaFields ?? [])
-    .filter(field => !BannedTypes.includes(field.type))
-    .map(field => field.name)
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
   const addFilter = () => {
@@ -53,7 +51,7 @@
 
   const onFieldChange = (expression, field) => {
     // Update the field type
-    expression.type = schemaFields.find(x => x.name === field)?.type
+    expression.type = enrichedSchemaFields.find(x => x.name === field)?.type
 
     // Ensure a valid operator is set
     const validOperators = getValidOperatorsForType(expression.type).map(
@@ -85,7 +83,7 @@
   }
 
   const getFieldOptions = field => {
-    const schema = schemaFields.find(x => x.name === field)
+    const schema = enrichedSchemaFields.find(x => x.name === field)
     return schema?.constraints?.inclusion || []
   }
 </script>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte

@@ -0,0 +1,47 @@
+<script>
+  import { Multiselect } from "@budibase/bbui"
+  import {
+    getDatasourceForProvider,
+    getSchemaForDatasource,
+  } from "builderStore/dataBinding"
+  import { currentAsset } from "builderStore"
+  import { tables } from "stores/backend"
+  import { createEventDispatcher } from "svelte"
+  import { getFields } from "helpers/searchFields"
+
+  export let componentInstance = {}
+  export let value = ""
+  export let placeholder
+
+  const dispatch = createEventDispatcher()
+  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
+  $: options = getOptions(datasource, schema || {})
+  $: boundValue = getSelectedOption(value, options)
+
+  function getOptions(ds, dsSchema) {
+    let base = Object.values(dsSchema)
+    if (!ds?.tableId) {
+      return base
+    }
+    const currentTable = $tables.list.find(table => table._id === ds.tableId)
+    return getFields(base, { allowLinks: currentTable.sql }).map(
+      field => field.name
+    )
+  }
+
+  function getSelectedOption(selectedOptions, allOptions) {
+    // Fix the hardcoded default string value
+    if (!Array.isArray(selectedOptions)) {
+      selectedOptions = []
+    }
+    return selectedOptions.filter(val => allOptions.indexOf(val) !== -1)
+  }
+
+  const setValue = value => {
+    boundValue = getSelectedOption(value.detail, options)
+    dispatch("change", boundValue)
+  }
+</script>
+
+<Multiselect {placeholder} value={boundValue} on:change={setValue} {options} />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -7,6 +7,7 @@ import ColorPicker from "./ColorPicker.svelte"
 import { IconSelect } from "./IconSelect"
 import FieldSelect from "./FieldSelect.svelte"
 import MultiFieldSelect from "./MultiFieldSelect.svelte"
+import SearchFieldSelect from "./SearchFieldSelect.svelte"
 import SchemaSelect from "./SchemaSelect.svelte"
 import SectionSelect from "./SectionSelect.svelte"
 import NavigationEditor from "./NavigationEditor/NavigationEditor.svelte"
@@ -30,6 +31,7 @@ const componentMap = {
   icon: IconSelect,
   field: FieldSelect,
   multifield: MultiFieldSelect,
+  searchfield: SearchFieldSelect,
   options: OptionsEditor,
   schema: SchemaSelect,
   section: SectionSelect,

packages/builder/src/constants/backend/index.js

@@ -229,3 +229,11 @@ export const PaginationLocations = [
   { label: "Query parameters", value: "query" },
   { label: "Request body", value: "body" },
 ]
+
+export const BannedSearchTypes = [
+  "link",
+  "attachment",
+  "formula",
+  "json",
+  "jsonarray",
+]

packages/builder/src/helpers/searchFields.js

@@ -0,0 +1,31 @@
+import { tables } from "../stores/backend"
+import { BannedSearchTypes } from "../constants/backend"
+import { get } from "svelte/store"
+
+export function getTableFields(linkField) {
+  const table = get(tables).list.find(table => table._id === linkField.tableId)
+  if (!table || !table.sql) {
+    return []
+  }
+  const linkFields = getFields(Object.values(table.schema), {
+    allowLinks: false,
+  })
+  return linkFields.map(field => ({
+    ...field,
+    name: `${table.name}.${field.name}`,
+  }))
+}
+
+export function getFields(fields, { allowLinks } = { allowLinks: true }) {
+  let filteredFields = fields.filter(
+    field => !BannedSearchTypes.includes(field.type)
+  )
+  if (allowLinks) {
+    const linkFields = fields.filter(field => field.type === "link")
+    for (let linkField of linkFields) {
+      // only allow one depth of SQL relationship filtering
+      filteredFields = filteredFields.concat(getTableFields(linkField))
+    }
+  }
+  return filteredFields
+}

packages/client/manifest.json

@@ -2811,7 +2811,7 @@
         "key": "dataSource"
       },
       {
-        "type": "multifield",
+        "type": "searchfield",
         "label": "Search Columns",
         "key": "searchColumns",
         "placeholder": "Choose search columns"
@@ -2958,7 +2958,7 @@
         "key": "dataSource"
       },
       {
-        "type": "multifield",
+        "type": "searchfield",
         "label": "Search Columns",
         "key": "searchColumns",
         "placeholder": "Choose search columns"

packages/client/src/components/app/DataProvider.svelte

@@ -67,6 +67,7 @@
   $: dataContext = {
     rows: $fetch.rows,
     info: $fetch.info,
+    datasource: dataSource || {},
     schema: $fetch.schema,
     rowsLength: $fetch.rows.length,
 

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -71,12 +71,13 @@
   const enrichFilter = (filter, columns, formId) => {
     let enrichedFilter = [...(filter || [])]
     columns?.forEach(column => {
+      const safePath = column.name.split(".").map(safe).join(".")
       enrichedFilter.push({
         field: column.name,
         operator: column.type === "string" ? "string" : "equal",
         type: column.type === "string" ? "string" : "number",
         valueType: "Binding",
-        value: `{{ [${formId}].[${column.name}] }}`,
+        value: `{{ ${safe(formId)}.${safePath} }}`,
       })
     })
     return enrichedFilter
@@ -112,7 +113,9 @@
   // Load the datasource schema so we can determine column types
   const fetchSchema = async dataSource => {
     if (dataSource) {
-      schema = await fetchDatasourceSchema(dataSource)
+      schema = await fetchDatasourceSchema(dataSource, {
+        enrichRelationships: true,
+      })
     }
     schemaLoaded = true
   }

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -59,12 +59,13 @@
   const enrichFilter = (filter, columns, formId) => {
     let enrichedFilter = [...(filter || [])]
     columns?.forEach(column => {
+      const safePath = column.name.split(".").map(safe).join(".")
       enrichedFilter.push({
         field: column.name,
         operator: column.type === "string" ? "string" : "equal",
         type: column.type === "string" ? "string" : "number",
         valueType: "Binding",
-        value: `{{ ${safe(formId)}.${safe(column.name)} }}`,
+        value: `{{ ${safe(formId)}.${safePath} }}`,
       })
     })
     return enrichedFilter
@@ -90,7 +91,9 @@
   // Load the datasource schema so we can determine column types
   const fetchSchema = async dataSource => {
     if (dataSource) {
-      schema = await fetchDatasourceSchema(dataSource)
+      schema = await fetchDatasourceSchema(dataSource, {
+        enrichRelationships: true,
+      })
     }
     schemaLoaded = true
   }

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -11,11 +11,14 @@
   export let size = "M"
 
   const component = getContext("component")
-  const { builderStore, ActionTypes, getAction } = getContext("sdk")
+  const { builderStore, ActionTypes, getAction, fetchDatasourceSchema } =
+    getContext("sdk")
 
   let modal
   let tmpFilters = []
   let filters = []
+  let schemaLoaded = false,
+    schema
 
   $: dataProviderId = dataProvider?.id
   $: addExtension = getAction(
@@ -26,7 +29,7 @@
     dataProviderId,
     ActionTypes.RemoveDataProviderQueryExtension
   )
-  $: schema = dataProvider?.schema
+  $: fetchSchema(dataProvider || {})
   $: schemaFields = getSchemaFields(schema, allowedFields)
 
   // Add query extension to data provider
@@ -39,7 +42,20 @@
     }
   }
 
-  const getSchemaFields = (schema, allowedFields) => {
+  async function fetchSchema(dataProvider) {
+    const datasource = dataProvider?.datasource
+    if (datasource) {
+      schema = await fetchDatasourceSchema(datasource, {
+        enrichRelationships: true,
+      })
+    }
+    schemaLoaded = true
+  }
+
+  function getSchemaFields(schema, allowedFields) {
+    if (!schemaLoaded) {
+      return {}
+    }
     let clonedSchema = {}
     if (!allowedFields?.length) {
       clonedSchema = schema
@@ -68,18 +84,20 @@
   })
 </script>
 
-<Button
+{#if schemaLoaded}
-  onClick={openEditor}
+  <Button
-  icon="Properties"
+    onClick={openEditor}
-  text="Filter"
+    icon="Properties"
-  {size}
+    text="Filter"
-  type="secondary"
+    {size}
-  quiet
+    type="secondary"
-  active={filters?.length > 0}
+    quiet
-/>
+    active={filters?.length > 0}
+  />
 
-<Modal bind:this={modal}>
+  <Modal bind:this={modal}>
-  <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
+    <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
-    <FilterModal bind:filters={tmpFilters} {schemaFields} />
+      <FilterModal bind:filters={tmpFilters} {schemaFields} />
-  </ModalContent>
+    </ModalContent>
-</Modal>
+  </Modal>
+{/if}

packages/client/src/utils/fetch/DataFetch.js

@@ -67,7 +67,6 @@ export default class DataFetch {
     this.getPage = this.getPage.bind(this)
     this.getInitialData = this.getInitialData.bind(this)
     this.determineFeatureFlags = this.determineFeatureFlags.bind(this)
-    this.enrichSchema = this.enrichSchema.bind(this)
     this.refresh = this.refresh.bind(this)
     this.update = this.update.bind(this)
     this.hasNextPage = this.hasNextPage.bind(this)
@@ -129,7 +128,7 @@ export default class DataFetch {
 
     // Fetch and enrich schema
     let schema = this.constructor.getSchema(datasource, definition)
-    schema = this.enrichSchema(schema)
+    schema = DataFetch.enrichSchema(schema)
     if (!schema) {
       return
     }
@@ -248,7 +247,7 @@ export default class DataFetch {
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
-  enrichSchema(schema) {
+  static enrichSchema(schema) {
     if (schema == null) {
       return null
     }

packages/client/src/utils/schema.js

@@ -6,13 +6,19 @@ import RelationshipFetch from "./fetch/RelationshipFetch.js"
 import NestedProviderFetch from "./fetch/NestedProviderFetch.js"
 import FieldFetch from "./fetch/FieldFetch.js"
 import JSONArrayFetch from "./fetch/JSONArrayFetch.js"
+import DataFetch from "./fetch/DataFetch.js"
 
 /**
  * Fetches the schema of any kind of datasource.
  * All datasource fetch classes implement their own functionality to get the
  * schema of a datasource of their respective types.
+ * @param datasource the datasource to fetch the schema for
+ * @param options options for enriching the schema
  */
-export const fetchDatasourceSchema = async datasource => {
+export const fetchDatasourceSchema = async (
+  datasource,
+  options = { enrichRelationships: false }
+) => {
   const handler = {
     table: TableFetch,
     view: ViewFetch,
@@ -28,7 +34,7 @@ export const fetchDatasourceSchema = async datasource => {
 
   // Get the datasource definition and then schema
   const definition = await handler.getDefinition(datasource)
-  const schema = handler.getSchema(datasource, definition)
+  let schema = handler.getSchema(datasource, definition)
   if (!schema) {
     return null
   }
@@ -49,5 +55,28 @@ export const fetchDatasourceSchema = async datasource => {
       })
     }
   })
-  return { ...schema, ...jsonAdditions }
+  schema = { ...schema, ...jsonAdditions }
+
+  // Check for any relationship fields if required
+  if (options?.enrichRelationships && definition.sql) {
+    let relationshipAdditions = {}
+    for (let fieldKey of Object.keys(schema)) {
+      const fieldSchema = schema[fieldKey]
+      if (fieldSchema?.type === "link") {
+        const linkSchema = await fetchDatasourceSchema({
+          type: "table",
+          tableId: fieldSchema?.tableId,
+        })
+        Object.keys(linkSchema || {}).forEach(linkKey => {
+          relationshipAdditions[`${fieldKey}.${linkKey}`] = {
+            type: linkSchema[linkKey].type,
+          }
+        })
+      }
+    }
+    schema = { ...schema, ...relationshipAdditions }
+  }
+
+  // Ensure schema structure is correct
+  return DataFetch.enrichSchema(schema)
 }

packages/server/src/api/controllers/table/index.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable } = require("../../../integrations/utils")
+const { isExternalTable, isSQL } = require("../../../integrations/utils")
 const {
   getTableParams,
   getDatasourceParams,
@@ -32,8 +32,8 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const internal = internalTables.rows.map(row => ({
+  const internal = internalTables.rows.map(tableDoc => ({
-    ...row.doc,
+    ...tableDoc.doc,
     type: "internal",
     sourceId: BudibaseInternalDB._id,
   }))
@@ -44,12 +44,18 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(row => {
+  const external = externalTables.rows.flatMap(tableDoc => {
-    return Object.values(row.doc.entities || {}).map(entity => ({
+    let entities = tableDoc.doc.entities
-      ...entity,
+    if (entities) {
-      type: "external",
+      return Object.values(entities).map(entity => ({
-      sourceId: row.doc._id,
+        ...entity,
-    }))
+        type: "external",
+        sourceId: tableDoc.doc._id,
+        sql: isSQL(tableDoc.doc),
+      }))
+    } else {
+      return []
+    }
   })
 
   ctx.body = [...internal, ...external]

packages/server/src/api/controllers/table/utils.js

@@ -12,6 +12,7 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
+  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
@@ -242,7 +243,9 @@ exports.getTable = async (appId, tableId) => {
   const db = new CouchDB(appId)
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    return exports.getExternalTable(appId, datasourceId, tableName)
+    const datasource = await db.get(datasourceId)
+    const table = await exports.getExternalTable(appId, datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
   } else {
     return db.get(tableId)
   }

packages/server/src/integrations/base/sql.ts

@@ -72,16 +72,22 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
-    tableName: string,
     query: KnexQuery,
-    filters: SearchFilters | undefined
+    filters: SearchFilters | undefined,
+    opts: { relationship?: boolean; tableName?: string }
   ): KnexQuery {
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        fn(`${tableName}.${key}`, value)
+        const isRelationshipField = key.includes(".")
+        if (!opts.relationship && !isRelationshipField) {
+          fn(`${opts.tableName}.${key}`, value)
+        }
+        if (opts.relationship && isRelationshipField) {
+          fn(key, value)
+        }
       }
     }
     if (!filters) {
@@ -272,7 +278,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(tableName, query, filters)
+    query = this.addFilters(query, filters, { tableName })
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
@@ -285,20 +291,21 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    return this.addRelationships(
+    query = this.addRelationships(
       knex,
       preQuery,
       selectStatement,
       tableName,
       relationships
     )
+    return this.addFilters(query, filters, { relationship: true })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     const parsedBody = parseBody(body)
-    query = this.addFilters(endpoint.entityId, query, filters)
+    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -310,7 +317,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
-    query = this.addFilters(endpoint.entityId, query, filters)
+    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

packages/server/src/integrations/tests/sql.spec.js

@@ -119,6 +119,22 @@ describe("SQL query builder", () => {
     })
   })
 
+  it("should allow filtering on a related field", () => {
+    const query = sql._query(generateReadJson({
+      filters: {
+        equal: {
+          age: 10,
+          "task.name": "task 1",
+        },
+      },
+    }))
+    // order of bindings changes because relationship filters occur outside inner query
+    expect(query).toEqual({
+      bindings: [10, limit, "task 1"],
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
+    })
+  })
+
   it("should test an create statement", () => {
     const query = sql._query(generateCreateJson(TABLE_NAME, {
       name: "Michael",

packages/server/src/integrations/utils.ts

@@ -1,6 +1,5 @@
-import { SqlQuery } from "../definitions/datasource"
+import { SourceNames, SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
-import { SourceNames } from "../definitions/datasource"
 import { DocumentTypes, SEPARATOR } from "../db/utils"
 import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
@@ -127,7 +126,12 @@ export function isSQL(datasource: Datasource): boolean {
   if (!datasource || !datasource.source) {
     return false
   }
-  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
+  const SQL = [
+    SourceNames.POSTGRES,
+    SourceNames.SQL_SERVER,
+    SourceNames.MYSQL,
+    SourceNames.ORACLE,
+  ]
   return SQL.indexOf(datasource.source) !== -1
 }
 

packages/string-templates/test/basic.spec.js

@@ -146,3 +146,14 @@ describe("check manifest", () => {
     )
   })
 })
+
+describe("check full stops that are safe", () => {
+  it("should allow using an escaped full stop", async () => {
+    const data = {
+      "c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI": { "persons.firstname": "1" }
+    }
+    const template = "{{ [c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI].[persons.firstname] }}"
+    const output = await processString(template, data)
+    expect(output).toEqual("1")
+  })
+})