From 5eeb2ea6006d9fded675b10da3856a1b452d51e4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 2 Aug 2022 18:34:58 +0100 Subject: [PATCH] Updating the filter system to allow adding multiple filter properties of the same name at once, as well as enabling the use of the allOr property from within the UI - resolves an old issue #2585. --- .../controls/FilterEditor/FilterDrawer.svelte | 29 +++++++- .../controls/FilterEditor/FilterEditor.svelte | 66 +++++++++++++++++-- packages/frontend-core/src/utils/lucene.js | 4 ++ .../src/api/controllers/row/internalSearch.js | 3 + .../server/src/api/controllers/row/utils.js | 7 +- packages/server/src/integrations/base/sql.ts | 8 ++- .../server/src/integrations/base/utils.ts | 40 ++++++----- .../server/src/integrations/queries/sql.ts | 1 - 8 files changed, 130 insertions(+), 28 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index aa191ce0ea..1d9c3dea09 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -9,19 +9,26 @@ Input, Layout, Select, + Toggle, + Label, } from "@budibase/bbui" import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { generate } from "shortid" import { LuceneUtils, Constants } from "@budibase/frontend-core" import { getFields } from "helpers/searchFields" + import { createEventDispatcher } from "svelte" + + const dispatch = createEventDispatcher() export let schemaFields export let filters = [] export let bindings = [] export let panel = ClientBindingPanel export let allowBindings = true + export let allOr = false + $: dispatch("change", filters) $: enrichedSchemaFields = getFields(schemaFields || []) $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] @@ -69,7 +76,7 @@ } // if changed to an array, change default value to empty array - const idx = filters.findIndex(x => x.field === field) + const idx = filters.findIndex(x => x.id === expression.id) if (expression.type === "array") { filters[idx].value = [] } else { @@ -179,10 +186,16 @@ {/each} {/if} -
+
+
+ (allOr = event.detail)} + /> +
@@ -202,4 +215,16 @@ align-items: center; grid-template-columns: 1fr 120px 120px 1fr auto auto; } + + .toggle { + display: flex; + align-items: center; + padding-right: var(--spacing-s); + } + + .bottom { + display: flex; + justify-content: space-between; + align-items: center; + } diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index 2cb35a9cf5..ea54afc0ee 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -8,21 +8,73 @@ import FilterDrawer from "./FilterDrawer.svelte" import { currentAsset } from "builderStore" + const QUERY_START_REGEX = /\d[0-9]*:/g const dispatch = createEventDispatcher() export let value = [] export let componentInstance export let bindings = [] - let drawer - let tempValue = value || [] + let drawer, + toSaveFilters = null, + allOr, + initialAllOr + $: initialFilters = correctFilters(value || []) $: dataSource = getDatasourceForProvider($currentAsset, componentInstance) $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema $: schemaFields = Object.values(schema || {}) - const saveFilter = async () => { - dispatch("change", tempValue) + function addNumbering(filters) { + let count = 1 + for (let value of filters) { + if (value.field && value.field?.match(QUERY_START_REGEX) == null) { + value.field = `${count++}:${value.field}` + } + } + return filters + } + + function correctFilters(filters) { + const corrected = [] + for (let filter of filters) { + let field = filter.field + if (filter.operator === "allOr") { + initialAllOr = allOr = true + continue + } + if ( + typeof filter.field === "string" && + filter.field.match(QUERY_START_REGEX) != null + ) { + const parts = field.split(":") + const number = parts[0] + // it's the new format, remove number + if (!isNaN(parseInt(number))) { + parts.shift() + field = parts.join(":") + } + } + corrected.push({ + ...filter, + field, + }) + } + return corrected + } + + async function saveFilter() { + if (!toSaveFilters && allOr !== initialAllOr) { + toSaveFilters = initialFilters + } + const filters = toSaveFilters?.filter(filter => filter.operator !== "allOr") + if (allOr && filters) { + filters.push({ operator: "allOr" }) + } + // only save if anything was updated + if (filters) { + dispatch("change", addNumbering(filters)) + } notifications.success("Filters saved.") drawer.hide() } @@ -33,8 +85,12 @@ { + toSaveFilters = event.detail + }} /> diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index b6699628d1..780cb2da39 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -103,6 +103,10 @@ export const buildLuceneQuery = filter => { const isHbs = typeof value === "string" && value.match(HBS_REGEX)?.length > 0 // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } if (type === "datetime") { // Ensure date value is a valid date and parse into correct format if (!value) { diff --git a/packages/server/src/api/controllers/row/internalSearch.js b/packages/server/src/api/controllers/row/internalSearch.js index 8a04fc2bd0..ab084d9e0b 100644 --- a/packages/server/src/api/controllers/row/internalSearch.js +++ b/packages/server/src/api/controllers/row/internalSearch.js @@ -1,4 +1,5 @@ const { SearchIndexes } = require("../../../db/utils") +const { removeKeyNumbering } = require("./utils") const fetch = require("node-fetch") const { getCouchInfo } = require("@budibase/backend-core/db") const { getAppId } = require("@budibase/backend-core/context") @@ -197,6 +198,8 @@ class QueryBuilder { function build(structure, queryFn) { for (let [key, value] of Object.entries(structure)) { + // check for new format - remove numbering if needed + key = removeKeyNumbering(key) key = builder.preprocess(key.replace(/ /g, "_"), { escape: true, }) diff --git a/packages/server/src/api/controllers/row/utils.js b/packages/server/src/api/controllers/row/utils.js index 5da7ca331e..da14020757 100644 --- a/packages/server/src/api/controllers/row/utils.js +++ b/packages/server/src/api/controllers/row/utils.js @@ -3,8 +3,11 @@ const { cloneDeep } = require("lodash/fp") const { InternalTables } = require("../../../db/utils") const userController = require("../user") const { FieldTypes } = require("../../../constants") -const { makeExternalQuery } = require("../../../integrations/base/utils") const { getAppDB } = require("@budibase/backend-core/context") +const { + makeExternalQuery, + removeKeyNumbering, +} = require("../../../integrations/base/utils") validateJs.extend(validateJs.validators.datetime, { parse: function (value) { @@ -16,6 +19,8 @@ validateJs.extend(validateJs.validators.datetime, { }, }) +exports.removeKeyNumbering = removeKeyNumbering + exports.getDatasourceAndQuery = async json => { const datasourceId = json.endpoint.datasourceId const db = getAppDB() diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 750564c6ff..a46ce7aea2 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -10,6 +10,7 @@ import { import { isIsoDateString, SqlClients } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import environment from "../../environment" +import { removeKeyNumbering } from "./utils" const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -133,12 +134,13 @@ class InternalBuilder { fn: (key: string, value: any) => void ) { for (let [key, value] of Object.entries(structure)) { - const isRelationshipField = key.includes(".") + const updatedKey = removeKeyNumbering(key) + const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${opts.tableName}.${key}`, value) + fn(`${opts.tableName}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { - fn(key, value) + fn(updatedKey, value) } } } diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts index 086912b920..0913d59b2a 100644 --- a/packages/server/src/integrations/base/utils.ts +++ b/packages/server/src/integrations/base/utils.ts @@ -1,22 +1,30 @@ import { QueryJson } from "../../definitions/datasource" import { Datasource } from "../../definitions/common" +const { integrations } = require("../index") -module DatasourceUtils { - const { integrations } = require("../index") +const QUERY_START_REGEX = /\d[0-9]*:/g - export async function makeExternalQuery( - datasource: Datasource, - json: QueryJson - ) { - const Integration = integrations[datasource.source] - // query is the opinionated function - if (Integration.prototype.query) { - const integration = new Integration(datasource.config) - return integration.query(json) - } else { - throw "Datasource does not support query." - } +export async function makeExternalQuery( + datasource: Datasource, + json: QueryJson +) { + const Integration = integrations[datasource.source] + // query is the opinionated function + if (Integration.prototype.query) { + const integration = new Integration(datasource.config) + return integration.query(json) + } else { + throw "Datasource does not support query." + } +} + +export function removeKeyNumbering(key: any): string { + if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { + const parts = key.split(":") + // remove the number + parts.shift() + return parts.join(":") + } else { + return key } - - module.exports.makeExternalQuery = makeExternalQuery } diff --git a/packages/server/src/integrations/queries/sql.ts b/packages/server/src/integrations/queries/sql.ts index 271a414d44..7fbcfea0a3 100644 --- a/packages/server/src/integrations/queries/sql.ts +++ b/packages/server/src/integrations/queries/sql.ts @@ -1,5 +1,4 @@ import { findHBSBlocks, processStringSync } from "@budibase/string-templates" -import { Integration } from "../../definitions/datasource" import { DatasourcePlus } from "../base/datasourcePlus" const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")