From 4188754bbec8059994cde30618dc32a4b9a3087f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Dec 2022 09:22:28 +0000 Subject: [PATCH] Fix SQL table `_id` filtering (#9030) * Re-add support for filtering on _id using external SQL tables and fix filter key prefixes not working with _id field * Remove like operator from internal tables and only allow basic operators on SQL table _id column * Update data section filtering to respect new rules * Update automation section filtering to respect new rules * Update dynamic filter component to respect new rules --- .../SetupPanel/AutomationBlockSetup.svelte | 1 + .../backend/DataTable/DataTable.svelte | 1 + .../buttons/TableFilterButton.svelte | 2 ++ .../controls/FilterEditor/FilterDrawer.svelte | 20 ++++++----- .../controls/FilterEditor/FilterEditor.svelte | 6 ++-- packages/builder/src/helpers/searchFields.js | 10 +----- .../app/dynamic-filter/DynamicFilter.svelte | 8 ++--- .../app/dynamic-filter/FilterModal.svelte | 11 +++++-- packages/frontend-core/src/utils/lucene.js | 33 +++++++++++++------ .../api/controllers/row/ExternalRequest.ts | 21 ++++++++---- 10 files changed, 70 insertions(+), 43 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index d09faa34c9..a73db5648b 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -232,6 +232,7 @@ {filters} {bindings} {schemaFields} + datasource={{ type: "table", tableId }} panel={AutomationBindingPanel} fillWidth on:change={e => (tempFilters = e.detail)} diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 4f5c3375bd..bdf2f46b2c 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -190,6 +190,7 @@ {filters} on:change={onFilter} disabled={!hasCols} + tableId={id} /> {/key} diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index be9c6259c6..5db4eb5288 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -6,6 +6,7 @@ export let schema export let filters export let disabled = false + export let tableId const dispatch = createEventDispatcher() @@ -37,6 +38,7 @@ allowBindings={false} {filters} {schemaFields} + datasource={{ type: "table", tableId }} on:change={e => (tempValue = e.detail)} /> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index d495e37216..629e2024e7 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -25,7 +25,7 @@ export let panel = ClientBindingPanel export let allowBindings = true export let fillWidth = false - export let tableId + export let datasource const dispatch = createEventDispatcher() const { OperatorOptions } = Constants @@ -41,11 +41,7 @@ $: parseFilters(filters) $: dispatch("change", enrichFilters(rawFilters, matchAny)) - $: enrichedSchemaFields = getFields( - schemaFields || [], - { allowLinks: true }, - tableId - ) + $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true }) $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] @@ -119,7 +115,11 @@ const santizeOperator = filter => { // Ensure a valid operator is selected - const operators = getValidOperatorsForType(filter.type).map(x => x.value) + const operators = getValidOperatorsForType( + filter.type, + filter.field, + datasource + ).map(x => x.value) if (!operators.includes(filter.operator)) { filter.operator = operators[0] ?? OperatorOptions.Equals.value } @@ -201,7 +201,11 @@ /> onOperatorChange(filter, e.detail)} placeholder={null} diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index 774ddbd834..427d8f8b97 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -7,7 +7,7 @@ const HBS_REGEX = /{{([^{].*?)}}/g * Returns the valid operator options for a certain data type * @param type the data type */ -export const getValidOperatorsForType = type => { +export const getValidOperatorsForType = (type, field, datasource) => { const Op = OperatorOptions const stringOps = [ Op.Equals, @@ -27,24 +27,37 @@ export const getValidOperatorsForType = type => { Op.NotEmpty, Op.In, ] + let ops = [] if (type === "string") { - return stringOps + ops = stringOps } else if (type === "number") { - return numOps + ops = numOps } else if (type === "options") { - return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In] + ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In] } else if (type === "array") { - return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny] + ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny] } else if (type === "boolean") { - return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] + ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] } else if (type === "longform") { - return stringOps + ops = stringOps } else if (type === "datetime") { - return numOps + ops = numOps } else if (type === "formula") { - return stringOps.concat([Op.MoreThan, Op.LessThan]) + ops = stringOps.concat([Op.MoreThan, Op.LessThan]) } - return [] + + // Filter out "like" for internal tables + const externalTable = datasource?.tableId?.includes("datasource_plus") + if (datasource?.type === "table" && !externalTable) { + ops = ops.filter(x => x !== Op.Like) + } + + // Only allow equal/not equal for _id in SQL tables + if (field === "_id" && externalTable) { + ops = [Op.Equals, Op.NotEquals] + } + + return ops } /** diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index a343553fc8..da3ff7bc53 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -25,6 +25,7 @@ import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { processFormulas, processDates } from "../../../utilities/rowProcessor" import { context } from "@budibase/backend-core" +import { removeKeyNumbering } from "./utils" export interface ManyRelationship { tableId?: string @@ -55,15 +56,21 @@ function buildFilters( let idCopy: undefined | string | any[] = cloneDeep(id) if (filters) { // need to map over the filters and make sure the _id field isn't present - for (let filter of Object.values(filters)) { - if (filter._id && primary) { - const parts = breakRowIdField(filter._id) - for (let field of primary) { - filter[field] = parts.shift() + let prefix = 1 + for (let operator of Object.values(filters)) { + for (let field of Object.keys(operator || {})) { + if (removeKeyNumbering(field) === "_id") { + if (primary) { + const parts = breakRowIdField(operator[field]) + for (let field of primary) { + operator[`${prefix}:${field}`] = parts.shift() + } + prefix++ + } + // make sure this field doesn't exist on any filter + delete operator[field] } } - // make sure this field doesn't exist on any filter - delete filter._id } } // there is no id, just use the user provided filters