Fix SQL table `_id` filtering (#9030)

* Re-add support for filtering on _id using external SQL tables and fix filter key prefixes not working with _id field * Remove like operator from internal tables and only allow basic operators on SQL table _id column * Update data section filtering to respect new rules * Update automation section filtering to respect new rules * Update dynamic filter component to respect new rules
2022-12-15 09:22:28 +00:00 · 2022-12-15 09:22:28 +00:00 · c3b6afbb6f
parent 5913df0359
commit c3b6afbb6f
10 changed files with 70 additions and 43 deletions
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@ -232,6 +232,7 @@
            {filters}
            {bindings}
            {schemaFields}
+            datasource={{ type: "table", tableId }}
            panel={AutomationBindingPanel}
            fillWidth
            on:change={e => (tempFilters = e.detail)}
--- a/packages/builder/src/components/backend/DataTable/DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte
@ -190,6 +190,7 @@
            {filters}
            on:change={onFilter}
            disabled={!hasCols}
+            tableId={id}
          />
        {/key}
      </div>
--- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
@ -6,6 +6,7 @@
  export let schema
  export let filters
  export let disabled = false
+  export let tableId

  const dispatch = createEventDispatcher()

@ -37,6 +38,7 @@
        allowBindings={false}
        {filters}
        {schemaFields}
+        datasource={{ type: "table", tableId }}
        on:change={e => (tempValue = e.detail)}
      />
    </div>
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
@ -25,7 +25,7 @@
  export let panel = ClientBindingPanel
  export let allowBindings = true
  export let fillWidth = false
-  export let tableId
+  export let datasource

  const dispatch = createEventDispatcher()
  const { OperatorOptions } = Constants
@ -41,11 +41,7 @@

  $: parseFilters(filters)
  $: dispatch("change", enrichFilters(rawFilters, matchAny))
-  $: enrichedSchemaFields = getFields(
-    schemaFields || [],
-    { allowLinks: true },
-    tableId
-  )
+  $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
  $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
  $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]

@ -119,7 +115,11 @@

  const santizeOperator = filter => {
    // Ensure a valid operator is selected
-    const operators = getValidOperatorsForType(filter.type).map(x => x.value)
+    const operators = getValidOperatorsForType(
+      filter.type,
+      filter.field,
+      datasource
+    ).map(x => x.value)
    if (!operators.includes(filter.operator)) {
      filter.operator = operators[0] ?? OperatorOptions.Equals.value
    }
@ -201,7 +201,11 @@
              />
              <Select
                disabled={!filter.field}
-                options={getValidOperatorsForType(filter.type)}
+                options={getValidOperatorsForType(
+                  filter.type,
+                  filter.field,
+                  datasource
+                )}
                bind:value={filter.operator}
                on:change={() => onOperatorChange(filter)}
                placeholder={null}
--- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
+++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
@ -17,8 +17,8 @@
  let drawer

  $: tempValue = value
-  $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
+  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
+  $: schema = getSchemaForDatasource($currentAsset, datasource)?.schema
  $: schemaFields = Object.values(schema || {})

  async function saveFilter() {
@ -36,7 +36,7 @@
    filters={value}
    {bindings}
    {schemaFields}
-    tableId={dataSource.tableId}
+    {datasource}
    on:change={e => (tempValue = e.detail)}
  />
 </Drawer>
--- a/packages/builder/src/helpers/searchFields.js
+++ b/packages/builder/src/helpers/searchFields.js
@ -16,11 +16,7 @@ export function getTableFields(linkField) {
  }))
 }

-export function getFields(
-  fields,
-  { allowLinks } = { allowLinks: true },
-  tableId
-) {
+export function getFields(fields, { allowLinks } = { allowLinks: true }) {
  let filteredFields = fields.filter(
    field => !BannedSearchTypes.includes(field.type)
  )
@ -34,9 +30,5 @@ export function getFields(
  const staticFormulaFields = fields.filter(
    field => field.type === "formula" && field.formulaType === "static"
  )
-  const table = get(tables).list.find(table => table._id === tableId)
-  if (table?.type === "external" && table?.sql) {
-    filteredFields = filteredFields.filter(field => field.name !== "_id")
-  }
  return filteredFields.concat(staticFormulaFields)
 }
--- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
+++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte
@ -21,6 +21,7 @@
    schema

  $: dataProviderId = dataProvider?.id
+  $: datasource = dataProvider?.datasource
  $: addExtension = getAction(
    dataProviderId,
    ActionTypes.AddDataProviderQueryExtension
@ -29,7 +30,7 @@
    dataProviderId,
    ActionTypes.RemoveDataProviderQueryExtension
  )
-  $: fetchSchema(dataProvider || {})
+  $: fetchSchema(datasource)
  $: schemaFields = getSchemaFields(schema, allowedFields)

  // Add query extension to data provider
@ -42,8 +43,7 @@
    }
  }

-  async function fetchSchema(dataProvider) {
-    const datasource = dataProvider?.datasource
+  async function fetchSchema(datasource) {
    if (datasource) {
      schema = await fetchDatasourceSchema(datasource, {
        enrichRelationships: true,
@ -102,7 +102,7 @@

  <Modal bind:this={modal}>
    <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
-      <FilterModal bind:filters={tmpFilters} {schemaFields} />
+      <FilterModal bind:filters={tmpFilters} {schemaFields} {datasource} />
    </ModalContent>
  </Modal>
 {/if}
--- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte
+++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte
@ -15,6 +15,7 @@

  export let schemaFields
  export let filters = []
+  export let datasource

  const context = getContext("context")
  const BannedTypes = ["link", "attachment", "json"]
@ -59,7 +60,9 @@

    // Ensure a valid operator is set
    const validOperators = LuceneUtils.getValidOperatorsForType(
-      expression.type
+      expression.type,
+      expression.field,
+      datasource
    ).map(x => x.value)
    if (!validOperators.includes(expression.operator)) {
      expression.operator =
@ -118,7 +121,11 @@
          />
          <Select
            disabled={!filter.field}
-            options={LuceneUtils.getValidOperatorsForType(filter.type)}
+            options={LuceneUtils.getValidOperatorsForType(
+              filter.type,
+              filter.field,
+              datasource
+            )}
            bind:value={filter.operator}
            on:change={e => onOperatorChange(filter, e.detail)}
            placeholder={null}
--- a/packages/frontend-core/src/utils/lucene.js
+++ b/packages/frontend-core/src/utils/lucene.js
@ -7,7 +7,7 @@ const HBS_REGEX = /{{([^{].*?)}}/g
 * Returns the valid operator options for a certain data type
 * @param type the data type
 */
-export const getValidOperatorsForType = type => {
+export const getValidOperatorsForType = (type, field, datasource) => {
  const Op = OperatorOptions
  const stringOps = [
    Op.Equals,
@ -27,24 +27,37 @@ export const getValidOperatorsForType = type => {
    Op.NotEmpty,
    Op.In,
  ]
+  let ops = []
  if (type === "string") {
-    return stringOps
+    ops = stringOps
  } else if (type === "number") {
-    return numOps
+    ops = numOps
  } else if (type === "options") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
  } else if (type === "array") {
-    return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
+    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
  } else if (type === "boolean") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
  } else if (type === "longform") {
-    return stringOps
+    ops = stringOps
  } else if (type === "datetime") {
-    return numOps
+    ops = numOps
  } else if (type === "formula") {
-    return stringOps.concat([Op.MoreThan, Op.LessThan])
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
  }
-  return []
+
+  // Filter out "like" for internal tables
+  const externalTable = datasource?.tableId?.includes("datasource_plus")
+  if (datasource?.type === "table" && !externalTable) {
+    ops = ops.filter(x => x !== Op.Like)
+  }
+
+  // Only allow equal/not equal for _id in SQL tables
+  if (field === "_id" && externalTable) {
+    ops = [Op.Equals, Op.NotEquals]
+  }
+
+  return ops
 }

 /**
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@ -25,6 +25,7 @@ import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
+import { removeKeyNumbering } from "./utils"

 export interface ManyRelationship {
  tableId?: string
@ -55,15 +56,21 @@ function buildFilters(
  let idCopy: undefined | string | any[] = cloneDeep(id)
  if (filters) {
    // need to map over the filters and make sure the _id field isn't present
-    for (let filter of Object.values(filters)) {
-      if (filter._id && primary) {
-        const parts = breakRowIdField(filter._id)
-        for (let field of primary) {
-          filter[field] = parts.shift()
+    let prefix = 1
+    for (let operator of Object.values(filters)) {
+      for (let field of Object.keys(operator || {})) {
+        if (removeKeyNumbering(field) === "_id") {
+          if (primary) {
+            const parts = breakRowIdField(operator[field])
+            for (let field of primary) {
+              operator[`${prefix}:${field}`] = parts.shift()
+            }
+            prefix++
+          }
+          // make sure this field doesn't exist on any filter
+          delete operator[field]
        }
      }
-      // make sure this field doesn't exist on any filter
-      delete filter._id
    }
  }
  // there is no id, just use the user provided filters