Fix SQL table `_id` filtering (#9030)

* Re-add support for filtering on _id using external SQL tables and fix filter key prefixes not working with _id field

* Remove like operator from internal tables and only allow basic operators on SQL table _id column

* Update data section filtering to respect new rules

* Update automation section filtering to respect new rules

* Update dynamic filter component to respect new rules

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -232,6 +232,7 @@
             {filters}
             {bindings}
             {schemaFields}
+            datasource={{ type: "table", tableId }}
             panel={AutomationBindingPanel}
             fillWidth
             on:change={e => (tempFilters = e.detail)}

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -190,6 +190,7 @@
             {filters}
             on:change={onFilter}
             disabled={!hasCols}
+            tableId={id}
           />
         {/key}
       </div>

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -6,6 +6,7 @@
   export let schema
   export let filters
   export let disabled = false
+  export let tableId
 
   const dispatch = createEventDispatcher()
 
@@ -37,6 +38,7 @@
         allowBindings={false}
         {filters}
         {schemaFields}
+        datasource={{ type: "table", tableId }}
         on:change={e => (tempValue = e.detail)}
       />
     </div>

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -25,7 +25,7 @@
   export let panel = ClientBindingPanel
   export let allowBindings = true
   export let fillWidth = false
-  export let tableId
+  export let datasource
 
   const dispatch = createEventDispatcher()
   const { OperatorOptions } = Constants
@@ -41,11 +41,7 @@
 
   $: parseFilters(filters)
   $: dispatch("change", enrichFilters(rawFilters, matchAny))
-  $: enrichedSchemaFields = getFields(
+  $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
-    schemaFields || [],
-    { allowLinks: true },
-    tableId
-  )
   $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
@@ -119,7 +115,11 @@
 
   const santizeOperator = filter => {
     // Ensure a valid operator is selected
-    const operators = getValidOperatorsForType(filter.type).map(x => x.value)
+    const operators = getValidOperatorsForType(
+      filter.type,
+      filter.field,
+      datasource
+    ).map(x => x.value)
     if (!operators.includes(filter.operator)) {
       filter.operator = operators[0] ?? OperatorOptions.Equals.value
     }
@@ -201,7 +201,11 @@
               />
               <Select
                 disabled={!filter.field}
-                options={getValidOperatorsForType(filter.type)}
+                options={getValidOperatorsForType(
+                  filter.type,
+                  filter.field,
+                  datasource
+                )}
                 bind:value={filter.operator}
                 on:change={() => onOperatorChange(filter)}
                 placeholder={null}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte

@@ -17,8 +17,8 @@
   let drawer
 
   $: tempValue = value
-  $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
+  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource)?.schema
   $: schemaFields = Object.values(schema || {})
 
   async function saveFilter() {
@@ -36,7 +36,7 @@
     filters={value}
     {bindings}
     {schemaFields}
-    tableId={dataSource.tableId}
+    {datasource}
     on:change={e => (tempValue = e.detail)}
   />
 </Drawer>

packages/builder/src/helpers/searchFields.js

@@ -16,11 +16,7 @@ export function getTableFields(linkField) {
   }))
 }
 
-export function getFields(
+export function getFields(fields, { allowLinks } = { allowLinks: true }) {
-  fields,
-  { allowLinks } = { allowLinks: true },
-  tableId
-) {
   let filteredFields = fields.filter(
     field => !BannedSearchTypes.includes(field.type)
   )
@@ -34,9 +30,5 @@ export function getFields(
   const staticFormulaFields = fields.filter(
     field => field.type === "formula" && field.formulaType === "static"
   )
-  const table = get(tables).list.find(table => table._id === tableId)
-  if (table?.type === "external" && table?.sql) {
-    filteredFields = filteredFields.filter(field => field.name !== "_id")
-  }
   return filteredFields.concat(staticFormulaFields)
 }

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -21,6 +21,7 @@
     schema
 
   $: dataProviderId = dataProvider?.id
+  $: datasource = dataProvider?.datasource
   $: addExtension = getAction(
     dataProviderId,
     ActionTypes.AddDataProviderQueryExtension
@@ -29,7 +30,7 @@
     dataProviderId,
     ActionTypes.RemoveDataProviderQueryExtension
   )
-  $: fetchSchema(dataProvider || {})
+  $: fetchSchema(datasource)
   $: schemaFields = getSchemaFields(schema, allowedFields)
 
   // Add query extension to data provider
@@ -42,8 +43,7 @@
     }
   }
 
-  async function fetchSchema(dataProvider) {
+  async function fetchSchema(datasource) {
-    const datasource = dataProvider?.datasource
     if (datasource) {
       schema = await fetchDatasourceSchema(datasource, {
         enrichRelationships: true,
@@ -102,7 +102,7 @@
 
   <Modal bind:this={modal}>
     <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
-      <FilterModal bind:filters={tmpFilters} {schemaFields} />
+      <FilterModal bind:filters={tmpFilters} {schemaFields} {datasource} />
     </ModalContent>
   </Modal>
 {/if}

packages/client/src/components/app/dynamic-filter/FilterModal.svelte

@@ -15,6 +15,7 @@
 
   export let schemaFields
   export let filters = []
+  export let datasource
 
   const context = getContext("context")
   const BannedTypes = ["link", "attachment", "json"]
@@ -59,7 +60,9 @@
 
     // Ensure a valid operator is set
     const validOperators = LuceneUtils.getValidOperatorsForType(
-      expression.type
+      expression.type,
+      expression.field,
+      datasource
     ).map(x => x.value)
     if (!validOperators.includes(expression.operator)) {
       expression.operator =
@@ -118,7 +121,11 @@
           />
           <Select
             disabled={!filter.field}
-            options={LuceneUtils.getValidOperatorsForType(filter.type)}
+            options={LuceneUtils.getValidOperatorsForType(
+              filter.type,
+              filter.field,
+              datasource
+            )}
             bind:value={filter.operator}
             on:change={e => onOperatorChange(filter, e.detail)}
             placeholder={null}

packages/frontend-core/src/utils/lucene.js

@@ -7,7 +7,7 @@ const HBS_REGEX = /{{([^{].*?)}}/g
  * Returns the valid operator options for a certain data type
  * @param type the data type
  */
-export const getValidOperatorsForType = type => {
+export const getValidOperatorsForType = (type, field, datasource) => {
   const Op = OperatorOptions
   const stringOps = [
     Op.Equals,
@@ -27,24 +27,37 @@ export const getValidOperatorsForType = type => {
     Op.NotEmpty,
     Op.In,
   ]
+  let ops = []
   if (type === "string") {
-    return stringOps
+    ops = stringOps
   } else if (type === "number") {
-    return numOps
+    ops = numOps
   } else if (type === "options") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
   } else if (type === "array") {
-    return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
+    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
   } else if (type === "boolean") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
   } else if (type === "longform") {
-    return stringOps
+    ops = stringOps
   } else if (type === "datetime") {
-    return numOps
+    ops = numOps
   } else if (type === "formula") {
-    return stringOps.concat([Op.MoreThan, Op.LessThan])
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
   }
-  return []
+
+  // Filter out "like" for internal tables
+  const externalTable = datasource?.tableId?.includes("datasource_plus")
+  if (datasource?.type === "table" && !externalTable) {
+    ops = ops.filter(x => x !== Op.Like)
+  }
+
+  // Only allow equal/not equal for _id in SQL tables
+  if (field === "_id" && externalTable) {
+    ops = [Op.Equals, Op.NotEquals]
+  }
+
+  return ops
 }
 
 /**

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -25,6 +25,7 @@ import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
+import { removeKeyNumbering } from "./utils"
 
 export interface ManyRelationship {
   tableId?: string
@@ -55,15 +56,21 @@ function buildFilters(
   let idCopy: undefined | string | any[] = cloneDeep(id)
   if (filters) {
     // need to map over the filters and make sure the _id field isn't present
-    for (let filter of Object.values(filters)) {
+    let prefix = 1
-      if (filter._id && primary) {
+    for (let operator of Object.values(filters)) {
-        const parts = breakRowIdField(filter._id)
+      for (let field of Object.keys(operator || {})) {
-        for (let field of primary) {
+        if (removeKeyNumbering(field) === "_id") {
-          filter[field] = parts.shift()
+          if (primary) {
+            const parts = breakRowIdField(operator[field])
+            for (let field of primary) {
+              operator[`${prefix}:${field}`] = parts.shift()
+            }
+            prefix++
+          }
+          // make sure this field doesn't exist on any filter
+          delete operator[field]
         }
       }
-      // make sure this field doesn't exist on any filter
-      delete filter._id
     }
   }
   // there is no id, just use the user provided filters