From 848b42dad54606584eb2fb6401ef356d08efe095 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 24 Jun 2022 14:39:16 +0100 Subject: [PATCH 01/12] Add In/NotIn constants --- packages/frontend-core/src/constants.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 9ca47de23f..ddc327411c 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -42,6 +42,14 @@ export const OperatorOptions = { value: "notEqual", label: "Does Not Contain", }, + In: { + value: "oneOf", + label: "Is in", + }, + NotIn: { + value: "notOneOf", + label: "Is not in", + }, } // Cookie names From 36a59632c4f967b407f258a2a9d9c00faa140e77 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 24 Jun 2022 14:41:08 +0100 Subject: [PATCH 02/12] Add (not)oneOf filter methods --- packages/frontend-core/src/utils/lucene.js | 19 +++++++++++++-- packages/server/yarn.lock | 28 +++++++++++----------- packages/worker/yarn.lock | 24 ++++++++++++------- 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index 8f59a2bd9d..270de05011 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -14,6 +14,8 @@ export const getValidOperatorsForType = type => { Op.Like, Op.Empty, Op.NotEmpty, + Op.In, + Op.NotIn, ] const numOps = [ Op.Equals, @@ -91,6 +93,8 @@ export const buildLuceneQuery = filter => { notEmpty: {}, contains: {}, notContains: {}, + oneOf: {}, + notOneOf: {}, } if (Array.isArray(filter)) { filter.forEach(expression => { @@ -139,7 +143,6 @@ export const buildLuceneQuery = filter => { } }) } - return query } @@ -211,6 +214,16 @@ export const runLuceneQuery = (docs, query) => { return docValue == null || docValue === "" }) + // Process an includes match (fails if the value is not included) + const oneOf = match("oneOf", (docValue, testValue) => { + return !testValue?.includes(docValue) + }) + + // Process a not included match (fails if the value is included) + const notOneOf = match("notOneOf", (docValue, testValue) => { + return testValue?.includes(docValue) + }) + // Match a document against all criteria const docMatch = doc => { return ( @@ -220,7 +233,9 @@ export const runLuceneQuery = (docs, query) => { equalMatch(doc) && notEqualMatch(doc) && emptyMatch(doc) && - notEmptyMatch(doc) + notEmptyMatch(doc) && + oneOf(doc) && + notOneOf(doc) ) } diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index e8277b88b5..bdbe1b8c58 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.207-alpha.6": - version "1.0.207-alpha.6" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.6.tgz#47fed5cc78686e23951a050479c777673f725c17" - integrity sha512-mB3i9TyNbdlE8TmsAWGXhphwLRlrBd2bDfvOYTz3CP7xzql1FXGoWfOqA87vNaGBDrtOyQQnmbYeTc3Tn2OHcg== +"@budibase/backend-core@1.0.207-alpha.10": + version "1.0.207-alpha.10" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.10.tgz#31facb74ac2279a3ecf7d6affb933cb0ccb81860" + integrity sha512-pEyL+DY9sK6cmGv9FHLKQgrTBaTR2uMQspb/B2HGKfhiEe/wI9DgEsrvfeDt6rsD02W5xcB21FfPyyfhFvfNYw== dependencies: - "@budibase/types" "^1.0.207-alpha.6" + "@budibase/types" "^1.0.207-alpha.10" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -1176,12 +1176,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.207-alpha.6": - version "1.0.207-alpha.6" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.6.tgz#04a81281beeb230c0c1a1f48237a94e1150a7851" - integrity sha512-IDQdKHaojfGlL8xLSQ1gYrLyipgUYPJ6Mjrrp8TcWnpwTOA2Wtzen31E5HG6YxZU8g8rN6k9S0Nsp88JKOGSrg== +"@budibase/pro@1.0.207-alpha.10": + version "1.0.207-alpha.10" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.10.tgz#5b1130676503eff7d84b62469e13111d3a896f09" + integrity sha512-42raVveRgHH6Qf6H1kS4lBhftTWuNJ+uhVAmDoPlHtYxuffkBVRcf6wOgYbHTN0Xx3suLrZ9yW+TupuIZQMM9Q== dependencies: - "@budibase/backend-core" "1.0.207-alpha.6" + "@budibase/backend-core" "1.0.207-alpha.10" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": @@ -1202,10 +1202,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@^1.0.207-alpha.6": - version "1.0.208" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.208.tgz#c45cb494fb5b85229e15a34c6ac1805bae5be867" - integrity sha512-zKIHg6TGK+soVxMNZNrGypP3DCrd3jhlUQEFeQ+rZR6/tCue1G74bjzydY5FjnLEsXeLH1a0hkS5HulTmvQ2bA== +"@budibase/types@^1.0.207-alpha.10": + version "1.0.210" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.210.tgz#b28f9d7b37d2cbc6b75ae8de201958ccc540cf06" + integrity sha512-16foNHGAlJxvh4IpPEPw9eZyIvA2n9nssrNu54Ag73E85A+mIiZ6NGHrw+mojrdC1yaSrj/xeQ461iYXdxT6/g== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 87208f6452..f3ed55d161 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,11 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.207-alpha.3": - version "1.0.207-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.3.tgz#98bced0575ec4e2b158239a8c73b39ca2d816719" - integrity sha512-DU4X6jJ+DfhzOv4TTa1w4Dk5ZEdlK/z1joCTruT+SGM5qI75bXrGeol5OX2OaEbNKtXFKJ1zeVTmBCYcu7OFUg== +"@budibase/backend-core@1.0.207-alpha.10": + version "1.0.207-alpha.10" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.10.tgz#31facb74ac2279a3ecf7d6affb933cb0ccb81860" + integrity sha512-pEyL+DY9sK6cmGv9FHLKQgrTBaTR2uMQspb/B2HGKfhiEe/wI9DgEsrvfeDt6rsD02W5xcB21FfPyyfhFvfNYw== dependencies: + "@budibase/types" "^1.0.207-alpha.10" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -322,14 +323,19 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@1.0.207-alpha.3": - version "1.0.207-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.3.tgz#9bde845ceb685f1b43286a124620c21fdf891a01" - integrity sha512-WFEMujpKTVAMvAgLBnMdw8ou9PxsbM4Oa9Dq+DAUsWpPACsMWOProyHLsdRxJyvHlgGfwVjo5MEusvStjI4j6g== +"@budibase/pro@1.0.207-alpha.10": + version "1.0.207-alpha.10" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.10.tgz#5b1130676503eff7d84b62469e13111d3a896f09" + integrity sha512-42raVveRgHH6Qf6H1kS4lBhftTWuNJ+uhVAmDoPlHtYxuffkBVRcf6wOgYbHTN0Xx3suLrZ9yW+TupuIZQMM9Q== dependencies: - "@budibase/backend-core" "1.0.207-alpha.3" + "@budibase/backend-core" "1.0.207-alpha.10" node-fetch "^2.6.1" +"@budibase/types@^1.0.207-alpha.10": + version "1.0.210" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.210.tgz#b28f9d7b37d2cbc6b75ae8de201958ccc540cf06" + integrity sha512-16foNHGAlJxvh4IpPEPw9eZyIvA2n9nssrNu54Ag73E85A+mIiZ6NGHrw+mojrdC1yaSrj/xeQ461iYXdxT6/g== + "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" From 40a2e95185b5fc3b67c1056f60d37b1013729522 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 24 Jun 2022 21:15:41 +0100 Subject: [PATCH 03/12] Internal table support for oneOf --- .../src/api/controllers/row/internalSearch.js | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/server/src/api/controllers/row/internalSearch.js b/packages/server/src/api/controllers/row/internalSearch.js index 5f1dc25faa..23cbdb4bec 100644 --- a/packages/server/src/api/controllers/row/internalSearch.js +++ b/packages/server/src/api/controllers/row/internalSearch.js @@ -17,6 +17,7 @@ class QueryBuilder { notEqual: {}, empty: {}, notEmpty: {}, + oneOf: {}, ...base, } this.limit = 50 @@ -112,6 +113,11 @@ class QueryBuilder { return this } + addOneOf(key, value) { + this.query.oneOf[key] = value + return this + } + /** * Preprocesses a value before going into a lucene search. * Transforms strings to lowercase and wraps strings and bools in quotes. @@ -220,6 +226,28 @@ class QueryBuilder { if (this.query.notEmpty) { build(this.query.notEmpty, key => `${key}:["" TO *]`) } + if (this.query.oneOf) { + build(this.query.oneOf, (key, value) => { + if (!Array.isArray(value)) { + if (typeof value === "string") { + value = value.replace(/(\s)*,(\s)*/g, ",").split(",") + } else { + return "" + } + } + const preprocess = item => { + return builder.preprocess(item, { + escape: true, + lowercase: true, + }) + } + let orStatement = `"${preprocess(value[0])}"` + for (let i = 1; i < value.length; i++) { + orStatement += ` OR "${preprocess(value[i])}"` + } + return `${key}:(${orStatement})` + }) + } return query } From abcf9752c1ee99486584deafda602a6f467e0d91 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 24 Jun 2022 21:29:19 +0100 Subject: [PATCH 04/12] Removed notOneOf --- packages/frontend-core/src/constants.js | 4 ---- packages/frontend-core/src/utils/lucene.js | 9 +-------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index ddc327411c..f69590f720 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -46,10 +46,6 @@ export const OperatorOptions = { value: "oneOf", label: "Is in", }, - NotIn: { - value: "notOneOf", - label: "Is not in", - }, } // Cookie names diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index 270de05011..a99ff6cf8b 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -94,7 +94,6 @@ export const buildLuceneQuery = filter => { contains: {}, notContains: {}, oneOf: {}, - notOneOf: {}, } if (Array.isArray(filter)) { filter.forEach(expression => { @@ -219,11 +218,6 @@ export const runLuceneQuery = (docs, query) => { return !testValue?.includes(docValue) }) - // Process a not included match (fails if the value is included) - const notOneOf = match("notOneOf", (docValue, testValue) => { - return testValue?.includes(docValue) - }) - // Match a document against all criteria const docMatch = doc => { return ( @@ -234,8 +228,7 @@ export const runLuceneQuery = (docs, query) => { notEqualMatch(doc) && emptyMatch(doc) && notEmptyMatch(doc) && - oneOf(doc) && - notOneOf(doc) + oneOf(doc) ) } From ea94983e69a3c4aeb6127340fd27591529f0a9b7 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 24 Jun 2022 21:36:38 +0100 Subject: [PATCH 05/12] Remove NotIn constant --- packages/frontend-core/src/utils/lucene.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index a99ff6cf8b..ec0225a6f9 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -15,7 +15,6 @@ export const getValidOperatorsForType = type => { Op.Empty, Op.NotEmpty, Op.In, - Op.NotIn, ] const numOps = [ Op.Equals, From 988bc12dea66c4ec9f6e59ba3b8fb31930848022 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 24 Jun 2022 21:37:58 +0100 Subject: [PATCH 06/12] Remove regex --- packages/server/src/api/controllers/row/internalSearch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/internalSearch.js b/packages/server/src/api/controllers/row/internalSearch.js index 23cbdb4bec..731b726654 100644 --- a/packages/server/src/api/controllers/row/internalSearch.js +++ b/packages/server/src/api/controllers/row/internalSearch.js @@ -230,7 +230,7 @@ class QueryBuilder { build(this.query.oneOf, (key, value) => { if (!Array.isArray(value)) { if (typeof value === "string") { - value = value.replace(/(\s)*,(\s)*/g, ",").split(",") + value = value.split(",") } else { return "" } From c5cfe0539c910b8285ada3b8f0bbf28a925da6e0 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Mon, 27 Jun 2022 13:36:03 +0100 Subject: [PATCH 07/12] Add 'Is in' filter option to number type --- packages/frontend-core/src/utils/lucene.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index ec0225a6f9..2183b4828f 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -23,6 +23,7 @@ export const getValidOperatorsForType = type => { Op.LessThan, Op.Empty, Op.NotEmpty, + Op.In, ] if (type === "string") { return stringOps From de01b684f0aa1226acee01835f7f0d36b1d01825 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Mon, 27 Jun 2022 13:38:43 +0100 Subject: [PATCH 08/12] Split string by comma if not array --- packages/frontend-core/src/utils/lucene.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index 2183b4828f..e7bc8e8afa 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -215,6 +215,9 @@ export const runLuceneQuery = (docs, query) => { // Process an includes match (fails if the value is not included) const oneOf = match("oneOf", (docValue, testValue) => { + if (typeof testValue === "string") { + testValue = testValue.split(",") + } return !testValue?.includes(docValue) }) From 08f1ddddd0b50fb1fdcc90427da3af1efa6f3f65 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Mon, 27 Jun 2022 13:54:03 +0100 Subject: [PATCH 09/12] Handle number types custom query --- packages/frontend-core/src/utils/lucene.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index e7bc8e8afa..e4acf1160d 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -217,6 +217,9 @@ export const runLuceneQuery = (docs, query) => { const oneOf = match("oneOf", (docValue, testValue) => { if (typeof testValue === "string") { testValue = testValue.split(",") + if (typeof docValue === "number") { + testValue = testValue.map(item => Number(item)) + } } return !testValue?.includes(docValue) }) From f84bf5f3372b197e563209de20bd3537cf8c987a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Mon, 27 Jun 2022 14:20:51 +0100 Subject: [PATCH 10/12] Using allPreProcessingOpts --- .../src/api/controllers/row/internalSearch.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/internalSearch.js b/packages/server/src/api/controllers/row/internalSearch.js index 731b726654..6fba0e47aa 100644 --- a/packages/server/src/api/controllers/row/internalSearch.js +++ b/packages/server/src/api/controllers/row/internalSearch.js @@ -235,15 +235,15 @@ class QueryBuilder { return "" } } - const preprocess = item => { - return builder.preprocess(item, { - escape: true, - lowercase: true, - }) - } - let orStatement = `"${preprocess(value[0])}"` + let orStatement = `${builder.preprocess( + value[0], + allPreProcessingOpts + )}` for (let i = 1; i < value.length; i++) { - orStatement += ` OR "${preprocess(value[i])}"` + orStatement += ` OR ${builder.preprocess( + value[i], + allPreProcessingOpts + )}` } return `${key}:(${orStatement})` }) From 3a65c877141421d5f4ca9c67bd0320874fa62f86 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Mon, 27 Jun 2022 14:32:20 +0100 Subject: [PATCH 11/12] Handle number 'Is in' --- packages/frontend-core/src/utils/lucene.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index e4acf1160d..9fa5a72317 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -102,8 +102,12 @@ export const buildLuceneQuery = filter => { if (type === "datetime" && value) { value = new Date(value).toISOString() } - if (type === "number") { - value = parseFloat(value) + if (type === "number" && !Array.isArray(value)) { + if (operator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } } if (type === "boolean") { value = `${value}`?.toLowerCase() === "true" @@ -137,6 +141,7 @@ export const buildLuceneQuery = filter => { query[operator][field] = value } } else { + console.log("VAL ", value) query[operator][field] = value } } From c0967ffe01f03c41fc8dc8027a029fff442fa64e Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Mon, 27 Jun 2022 14:37:15 +0100 Subject: [PATCH 12/12] Remove console log --- packages/frontend-core/src/utils/lucene.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/frontend-core/src/utils/lucene.js b/packages/frontend-core/src/utils/lucene.js index 9fa5a72317..1001ec26a8 100644 --- a/packages/frontend-core/src/utils/lucene.js +++ b/packages/frontend-core/src/utils/lucene.js @@ -141,7 +141,6 @@ export const buildLuceneQuery = filter => { query[operator][field] = value } } else { - console.log("VAL ", value) query[operator][field] = value } } @@ -223,7 +222,7 @@ export const runLuceneQuery = (docs, query) => { if (typeof testValue === "string") { testValue = testValue.split(",") if (typeof docValue === "number") { - testValue = testValue.map(item => Number(item)) + testValue = testValue.map(item => parseFloat(item)) } } return !testValue?.includes(docValue)