From ec7cf96fd9ceccfa003f151c1ce1fdbdaa675232 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 27 Sep 2021 12:59:49 +0100 Subject: [PATCH] Move all lucene logic into central builder helpers file --- .../SetupPanel/AutomationBlockSetup.svelte | 2 +- .../builder/src/helpers/fetchTableData.js | 2 +- packages/builder/src/helpers/lucene.js | 187 ++++++++++++++++++ packages/client/rollup.config.js | 4 + .../client/src/components/ClientApp.svelte | 2 +- .../src/components/app/DataProvider.svelte | 2 +- packages/client/src/stores/state.js | 2 +- packages/client/src/utils/conditions.js | 2 +- packages/client/src/utils/lucene.js | 179 ----------------- 9 files changed, 197 insertions(+), 185 deletions(-) delete mode 100644 packages/client/src/utils/lucene.js diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index adc22e5daf..ff52c7d11a 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -24,7 +24,7 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte" // need the client lucene builder to convert to the structure API expects - import { buildLuceneQuery } from "../../../../../client/src/utils/lucene" + import { buildLuceneQuery } from "helpers/lucene" export let block export let webhookModal diff --git a/packages/builder/src/helpers/fetchTableData.js b/packages/builder/src/helpers/fetchTableData.js index 8c9e9f1ea7..43ecd464c9 100644 --- a/packages/builder/src/helpers/fetchTableData.js +++ b/packages/builder/src/helpers/fetchTableData.js @@ -1,6 +1,6 @@ import { writable, derived, get } from "svelte/store" import * as API from "builderStore/api" -import { buildLuceneQuery } from "../../../client/src/utils/lucene" +import { buildLuceneQuery } from "helpers/lucene" const defaultOptions = { tableId: null, diff --git a/packages/builder/src/helpers/lucene.js b/packages/builder/src/helpers/lucene.js index 18692359e4..5225467d7b 100644 --- a/packages/builder/src/helpers/lucene.js +++ b/packages/builder/src/helpers/lucene.js @@ -1,3 +1,186 @@ +/** + * Builds a lucene JSON query from the filter structure generated in the builder + * @param filter the builder filter structure + */ +export const buildLuceneQuery = filter => { + let query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + } + if (Array.isArray(filter)) { + filter.forEach(expression => { + let { operator, field, type, value } = expression + // Parse all values into correct types + if (type === "datetime" && value) { + value = new Date(value).toISOString() + } + if (type === "number") { + value = parseFloat(value) + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if (operator.startsWith("range")) { + if (!query.range[field]) { + query.range[field] = { + low: + type === "number" + ? Number.MIN_SAFE_INTEGER + : "0000-00-00T00:00:00.000Z", + high: + type === "number" + ? Number.MAX_SAFE_INTEGER + : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field].low = value + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field].high = value + } + } else if (query[operator]) { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (operator === "equal" && value === false) { + query.notEqual[field] = true + } else if (operator === "notEqual" && value === false) { + query.equal[field] = true + } else { + query[operator][field] = value + } + } else { + query[operator][field] = value + } + } + }) + } + + return query +} + +/** + * Performs a client-side lucene search on an array of data + * @param docs the data + * @param query the JSON lucene query + */ +export const luceneQuery = (docs, query) => { + if (!query) { + return docs + } + + // Iterates over a set of filters and evaluates a fail function against a doc + const match = (type, failFn) => doc => { + const filters = Object.entries(query[type] || {}) + for (let i = 0; i < filters.length; i++) { + if (failFn(filters[i][0], filters[i][1], doc)) { + return false + } + } + return true + } + + // Process a string match (fails if the value does not start with the string) + const stringMatch = match("string", (key, value, doc) => { + return !doc[key] || !doc[key].startsWith(value) + }) + + // Process a fuzzy match (treat the same as starts with when running locally) + const fuzzyMatch = match("fuzzy", (key, value, doc) => { + return !doc[key] || !doc[key].startsWith(value) + }) + + // Process a range match + const rangeMatch = match("range", (key, value, doc) => { + return !doc[key] || doc[key] < value.low || doc[key] > value.high + }) + + // Process an equal match (fails if the value is different) + const equalMatch = match("equal", (key, value, doc) => { + return value != null && value !== "" && doc[key] !== value + }) + + // Process a not-equal match (fails if the value is the same) + const notEqualMatch = match("notEqual", (key, value, doc) => { + return value != null && value !== "" && doc[key] === value + }) + + // Process an empty match (fails if the value is not empty) + const emptyMatch = match("empty", (key, value, doc) => { + return doc[key] != null && doc[key] !== "" + }) + + // Process a not-empty match (fails is the value is empty) + const notEmptyMatch = match("notEmpty", (key, value, doc) => { + return doc[key] == null || doc[key] === "" + }) + + // Match a document against all criteria + const docMatch = doc => { + return ( + stringMatch(doc) && + fuzzyMatch(doc) && + rangeMatch(doc) && + equalMatch(doc) && + notEqualMatch(doc) && + emptyMatch(doc) && + notEmptyMatch(doc) + ) + } + + // Process all docs + return docs.filter(docMatch) +} + +/** + * Performs a client-side sort from the equivalent server-side lucene sort + * parameters. + * @param docs the data + * @param sort the sort column + * @param sortOrder the sort order ("ascending" or "descending") + * @param sortType the type of sort ("string" or "number") + */ +export const luceneSort = (docs, sort, sortOrder, sortType = "string") => { + if (!sort || !sortOrder || !sortType) { + return docs + } + const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x) + return docs.slice().sort((a, b) => { + const colA = parse(a[sort]) + const colB = parse(b[sort]) + if (sortOrder === "Descending") { + return colA > colB ? -1 : 1 + } else { + return colA > colB ? 1 : -1 + } + }) +} + +/** + * Limits the specified docs to the specified number of rows from the equivalent + * server-side lucene limit parameters. + * @param docs the data + * @param limit the number of docs to limit to + */ +export const luceneLimit = (docs, limit) => { + const numLimit = parseFloat(limit) + if (isNaN(numLimit)) { + return docs + } + return docs.slice(0, numLimit) +} + +/** + * Operator options for lucene queries + */ export const OperatorOptions = { Equals: { value: "equal", @@ -41,6 +224,10 @@ export const OperatorOptions = { }, } +/** + * Returns the valid operator options for a certain data type + * @param type the data type + */ export const getValidOperatorsForType = type => { const Op = OperatorOptions if (type === "string") { diff --git a/packages/client/rollup.config.js b/packages/client/rollup.config.js index f404f93c4c..a814303069 100644 --- a/packages/client/rollup.config.js +++ b/packages/client/rollup.config.js @@ -58,6 +58,10 @@ export default { find: "sdk", replacement: path.resolve("./src/sdk"), }, + { + find: "builder", + replacement: path.resolve("../builder"), + }, ], }), svelte({ diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte index 5890c0e2a9..c9c033caa3 100644 --- a/packages/client/src/components/ClientApp.svelte +++ b/packages/client/src/components/ClientApp.svelte @@ -23,7 +23,7 @@ import SelectionIndicator from "components/preview/SelectionIndicator.svelte" import HoverIndicator from "components/preview/HoverIndicator.svelte" import CustomThemeWrapper from "./CustomThemeWrapper.svelte" - import ErrorSVG from "../../../builder/assets/error.svg" + import ErrorSVG from "builder/assets/error.svg" // Provide contexts setContext("sdk", SDK) diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index bdc9001445..991c41b77d 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -6,7 +6,7 @@ luceneQuery, luceneSort, luceneLimit, - } from "utils/lucene" + } from "builder/src/helpers/lucene" import Placeholder from "./Placeholder.svelte" export let dataSource diff --git a/packages/client/src/stores/state.js b/packages/client/src/stores/state.js index cb20149de8..ce977c4333 100644 --- a/packages/client/src/stores/state.js +++ b/packages/client/src/stores/state.js @@ -1,5 +1,5 @@ import { writable, get, derived } from "svelte/store" -import { localStorageStore } from "../../../builder/src/builderStore/store/localStorage" +import { localStorageStore } from "builder/src/builderStore/store/localStorage" import { appStore } from "./app" const createStateStore = () => { diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js index 964a63d3fd..2791fa169e 100644 --- a/packages/client/src/utils/conditions.js +++ b/packages/client/src/utils/conditions.js @@ -1,4 +1,4 @@ -import { buildLuceneQuery, luceneQuery } from "./lucene" +import { buildLuceneQuery, luceneQuery } from "builder/src/helpers/lucene" export const getActiveConditions = conditions => { if (!conditions?.length) { diff --git a/packages/client/src/utils/lucene.js b/packages/client/src/utils/lucene.js deleted file mode 100644 index 03baa751cc..0000000000 --- a/packages/client/src/utils/lucene.js +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Builds a lucene JSON query from the filter structure generated in the builder - * @param filter the builder filter structure - */ -export const buildLuceneQuery = filter => { - let query = { - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - contains: {}, - notContains: {}, - } - if (Array.isArray(filter)) { - filter.forEach(expression => { - let { operator, field, type, value } = expression - // Parse all values into correct types - if (type === "datetime" && value) { - value = new Date(value).toISOString() - } - if (type === "number") { - value = parseFloat(value) - } - if (type === "boolean") { - value = `${value}`?.toLowerCase() === "true" - } - if (operator.startsWith("range")) { - if (!query.range[field]) { - query.range[field] = { - low: - type === "number" - ? Number.MIN_SAFE_INTEGER - : "0000-00-00T00:00:00.000Z", - high: - type === "number" - ? Number.MAX_SAFE_INTEGER - : "9999-00-00T00:00:00.000Z", - } - } - if (operator === "rangeLow" && value != null && value !== "") { - query.range[field].low = value - } else if (operator === "rangeHigh" && value != null && value !== "") { - query.range[field].high = value - } - } else if (query[operator]) { - if (type === "boolean") { - // Transform boolean filters to cope with null. - // "equals false" needs to be "not equals true" - // "not equals false" needs to be "equals true" - if (operator === "equal" && value === false) { - query.notEqual[field] = true - } else if (operator === "notEqual" && value === false) { - query.equal[field] = true - } else { - query[operator][field] = value - } - } else { - query[operator][field] = value - } - } - }) - } - - return query -} - -/** - * Performs a client-side lucene search on an array of data - * @param docs the data - * @param query the JSON lucene query - */ -export const luceneQuery = (docs, query) => { - if (!query) { - return docs - } - - // Iterates over a set of filters and evaluates a fail function against a doc - const match = (type, failFn) => doc => { - const filters = Object.entries(query[type] || {}) - for (let i = 0; i < filters.length; i++) { - if (failFn(filters[i][0], filters[i][1], doc)) { - return false - } - } - return true - } - - // Process a string match (fails if the value does not start with the string) - const stringMatch = match("string", (key, value, doc) => { - return !doc[key] || !doc[key].startsWith(value) - }) - - // Process a fuzzy match (treat the same as starts with when running locally) - const fuzzyMatch = match("fuzzy", (key, value, doc) => { - return !doc[key] || !doc[key].startsWith(value) - }) - - // Process a range match - const rangeMatch = match("range", (key, value, doc) => { - return !doc[key] || doc[key] < value.low || doc[key] > value.high - }) - - // Process an equal match (fails if the value is different) - const equalMatch = match("equal", (key, value, doc) => { - return value != null && value !== "" && doc[key] !== value - }) - - // Process a not-equal match (fails if the value is the same) - const notEqualMatch = match("notEqual", (key, value, doc) => { - return value != null && value !== "" && doc[key] === value - }) - - // Process an empty match (fails if the value is not empty) - const emptyMatch = match("empty", (key, value, doc) => { - return doc[key] != null && doc[key] !== "" - }) - - // Process a not-empty match (fails is the value is empty) - const notEmptyMatch = match("notEmpty", (key, value, doc) => { - return doc[key] == null || doc[key] === "" - }) - - // Match a document against all criteria - const docMatch = doc => { - return ( - stringMatch(doc) && - fuzzyMatch(doc) && - rangeMatch(doc) && - equalMatch(doc) && - notEqualMatch(doc) && - emptyMatch(doc) && - notEmptyMatch(doc) - ) - } - - // Process all docs - return docs.filter(docMatch) -} - -/** - * Performs a client-side sort from the equivalent server-side lucene sort - * parameters. - * @param docs the data - * @param sort the sort column - * @param sortOrder the sort order ("ascending" or "descending") - * @param sortType the type of sort ("string" or "number") - */ -export const luceneSort = (docs, sort, sortOrder, sortType = "string") => { - if (!sort || !sortOrder || !sortType) { - return docs - } - const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x) - return docs.slice().sort((a, b) => { - const colA = parse(a[sort]) - const colB = parse(b[sort]) - if (sortOrder === "Descending") { - return colA > colB ? -1 : 1 - } else { - return colA > colB ? 1 : -1 - } - }) -} - -/** - * Limits the specified docs to the specified number of rows from the equivalent - * server-side lucene limit parameters. - * @param docs the data - * @param limit the number of docs to limit to - */ -export const luceneLimit = (docs, limit) => { - const numLimit = parseFloat(limit) - if (isNaN(numLimit)) { - return docs - } - return docs.slice(0, numLimit) -}