From 95f5844a44aead711a4725cadc33120db909ce6f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 15:02:16 +0100 Subject: [PATCH 01/25] Get in-memory searching into the search tests. --- .../src/api/routes/tests/search.spec.ts | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index aac43874a0..51349e7647 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -22,30 +22,27 @@ import { import _ from "lodash" import tk from "timekeeper" import { encodeJSBinding } from "@budibase/string-templates" +import { dataFilters } from "@budibase/shared-core" describe.each([ - ["lucene", undefined], - ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], -])("/api/:sourceId/search (%s)", (name, dsProvider) => { + ["in-memory", undefined], + // ["lucene", undefined], + // ["sqs", undefined], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], +])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" + const isInMemory = name === "in-memory" const isInternal = isSqs || isLucene const config = setup.getConfig() let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let table: Table - - const snippets = [ - { - name: "WeeksAgo", - code: `return function (weeks) {\n const currentTime = new Date(${Date.now()});\n currentTime.setDate(currentTime.getDate()-(7 * (weeks || 1)));\n return currentTime.toISOString();\n}`, - }, - ] + let rows: Row[] beforeAll(async () => { if (isSqs) { @@ -55,7 +52,12 @@ describe.each([ if (config.app?.appId) { config.app = await config.api.application.update(config.app?.appId, { - snippets, + snippets: [ + { + name: "WeeksAgo", + code: `return function (weeks) {\n const currentTime = new Date(${Date.now()});\n currentTime.setDate(currentTime.getDate()-(7 * (weeks || 1)));\n return currentTime.toISOString();\n}`, + }, + ], }) } @@ -79,14 +81,30 @@ describe.each([ ) } - async function createRows(rows: Record[]) { + async function createRows(arr: Record[]) { // Shuffling to avoid false positives given a fixed order - await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) }) + await config.api.row.bulkImport(table._id!, { + rows: _.shuffle(arr), + }) + rows = await config.api.row.fetch(table._id!) } class SearchAssertion { constructor(private readonly query: RowSearchParams) {} + private async performSearch(): Promise { + if (isInMemory) { + return dataFilters.runQuery(rows, this.query.query) + } else { + return ( + await config.api.row.search(table._id!, { + ...this.query, + tableId: table._id!, + }) + ).rows + } + } + // We originally used _.isMatch to compare rows, but found that when // comparing arrays it would return true if the source array was a subset of // the target array. This would sometimes create false matches. This @@ -157,10 +175,7 @@ describe.each([ // different to the one passed in will cause the assertion to fail. Extra // rows returned by the query will also cause the assertion to fail. async toMatchExactly(expectedRows: any[]) { - const { rows: foundRows } = await config.api.row.search(table._id!, { - ...this.query, - tableId: table._id!, - }) + const foundRows = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(expectedRows.length) @@ -176,10 +191,7 @@ describe.each([ // passed in. The order of the rows is not important, but extra rows will // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { - const { rows: foundRows } = await config.api.row.search(table._id!, { - ...this.query, - tableId: table._id!, - }) + const foundRows = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(expectedRows.length) @@ -197,10 +209,7 @@ describe.each([ // The order of the rows is not important. Extra rows will not cause the // assertion to fail. async toContain(expectedRows: any[]) { - const { rows: foundRows } = await config.api.row.search(table._id!, { - ...this.query, - tableId: table._id!, - }) + const foundRows = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect([...foundRows]).toEqual( @@ -217,10 +226,7 @@ describe.each([ } async toHaveLength(length: number) { - const { rows: foundRows } = await config.api.row.search(table._id!, { - ...this.query, - tableId: table._id!, - }) + const foundRows = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(length) From 7e4f571eb36092efccbc0c4ac7b5d3a6ce9fe54b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 15:23:35 +0100 Subject: [PATCH 02/25] wip --- packages/server/src/api/routes/tests/search.spec.ts | 13 ++++++++++++- packages/shared-core/src/filters.ts | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 51349e7647..d6e764192d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -94,7 +94,18 @@ describe.each([ private async performSearch(): Promise { if (isInMemory) { - return dataFilters.runQuery(rows, this.query.query) + let result = dataFilters.runQuery(rows, this.query.query) + if (this.query.sort) { + result = dataFilters.sort( + result, + this.query.sort, + this.query.sortOrder || SortOrder.ASCENDING + ) + } + if (this.query.limit) { + result = dataFilters.limit(result, this.query.limit.toString()) + } + return result } else { return ( await config.api.row.search(table._id!, { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index f714f1cef6..862c2ea9bd 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -7,9 +7,9 @@ import { SearchFilters, SearchQueryFields, SearchFilterOperator, - SortDirection, SortType, FieldConstraints, + SortOrder, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -454,7 +454,7 @@ export const runQuery = (docs: any[], query?: SearchFilters) => { export const sort = ( docs: any[], sort: string, - sortOrder: SortDirection, + sortOrder: SortOrder, sortType = SortType.STRING ) => { if (!sort || !sortOrder || !sortType) { From ae6539161fb966abe2fc4d8b30d0967a3ae84261 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:28:03 +0100 Subject: [PATCH 03/25] Get rid of negation in predicate. --- .../src/api/routes/tests/search.spec.ts | 2 +- packages/shared-core/src/filters.ts | 213 ++++++++++-------- 2 files changed, 117 insertions(+), 98 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d6e764192d..3cafec1b9a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1706,7 +1706,7 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => + it.only("successfully finds a row", () => expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 862c2ea9bd..e95015d340 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -264,7 +264,10 @@ export const buildQuery = (filter: SearchFilter[]) => { * @param docs the data * @param query the JSON query */ -export const runQuery = (docs: any[], query?: SearchFilters) => { +export const runQuery = ( + docs: Record[], + query?: SearchFilters +) => { if (!docs || !Array.isArray(docs)) { return [] } @@ -272,50 +275,48 @@ export const runQuery = (docs: any[], query?: SearchFilters) => { return docs } - // Make query consistent first query = cleanupQuery(query) - // Iterates over a set of filters and evaluates a fail function against a doc const match = ( type: SearchFilterOperator, - failFn: (docValue: any, testValue: any) => boolean + test: (docValue: any, testValue: any) => boolean ) => - (doc: any) => { - const filters = Object.entries(query![type] || {}) - for (let i = 0; i < filters.length; i++) { - const [key, testValue] = filters[i] - const docValue = deepGet(doc, removeKeyNumbering(key)) - if (failFn(docValue, testValue)) { + (doc: Record) => { + for (const [key, testValue] of Object.entries(query[type] || {})) { + if (!test(deepGet(doc, removeKeyNumbering(key)), testValue)) { return false } } return true } - // Process a string match (fails if the value does not start with the string) const stringMatch = match( SearchFilterOperator.STRING, - (docValue: string, testValue: string) => { - return ( - !docValue || - !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) - ) + (docValue: any, testValue: any) => { + if (!(typeof docValue === "string")) { + return false + } + if (!(typeof testValue === "string")) { + return false + } + return docValue.toLowerCase().startsWith(testValue.toLowerCase()) } ) - // Process a fuzzy match (treat the same as starts with when running locally) const fuzzyMatch = match( SearchFilterOperator.FUZZY, - (docValue: string, testValue: string) => { - return ( - !docValue || - !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) - ) + (docValue: any, testValue: any) => { + if (!(typeof docValue === "string")) { + return false + } + if (!(typeof testValue === "string")) { + return false + } + return docValue.toLowerCase().includes(testValue.toLowerCase()) } ) - // Process a range match const rangeMatch = match( SearchFilterOperator.RANGE, ( @@ -323,54 +324,47 @@ export const runQuery = (docs: any[], query?: SearchFilters) => { testValue: { low: number; high: number } ) => { if (docValue == null || docValue === "") { - return true + return false } if (!isNaN(+docValue)) { - return +docValue < testValue.low || +docValue > testValue.high + return +docValue >= testValue.low || +docValue <= testValue.high } if (dayjs(docValue).isValid()) { return ( - new Date(docValue).getTime() < new Date(testValue.low).getTime() || - new Date(docValue).getTime() > new Date(testValue.high).getTime() + new Date(docValue).getTime() >= new Date(testValue.low).getTime() || + new Date(docValue).getTime() <= new Date(testValue.high).getTime() ) } return false } ) - // Process an equal match (fails if the value is different) - const equalMatch = match( - SearchFilterOperator.EQUAL, - (docValue: any, testValue: string | null) => { - return testValue != null && testValue !== "" && docValue !== testValue - } - ) + const not = + (f: (...args: T) => boolean) => + (...args: T): boolean => + !f(...args) - // Process a not-equal match (fails if the value is the same) - const notEqualMatch = match( - SearchFilterOperator.NOT_EQUAL, - (docValue: any, testValue: string | null) => { - return testValue != null && testValue !== "" && docValue === testValue - } - ) + const _equal = (docValue: any, testValue: any) => docValue === testValue - // Process an empty match (fails if the value is not empty) - const emptyMatch = match( - SearchFilterOperator.EMPTY, - (docValue: string | null) => { - return docValue != null && docValue !== "" - } - ) + const equalMatch = match(SearchFilterOperator.EQUAL, _equal) + const notEqualMatch = match(SearchFilterOperator.NOT_EQUAL, not(_equal)) - // Process a not-empty match (fails is the value is empty) - const notEmptyMatch = match( - SearchFilterOperator.NOT_EMPTY, - (docValue: string | null) => { - return docValue == null || docValue === "" + const _empty = (docValue: any) => { + if (typeof docValue === "string") { + return docValue === "" } - ) + if (Array.isArray(docValue)) { + return docValue.length === 0 + } + if (typeof docValue === "object") { + return Object.keys(docValue).length === 0 + } + return docValue == null + } + + const emptyMatch = match(SearchFilterOperator.EMPTY, _empty) + const notEmptyMatch = match(SearchFilterOperator.NOT_EMPTY, not(_empty)) - // Process an includes match (fails if the value is not included) const oneOf = match( SearchFilterOperator.ONE_OF, (docValue: any, testValue: any) => { @@ -380,61 +374,86 @@ export const runQuery = (docs: any[], query?: SearchFilters) => { testValue = testValue.map((item: string) => parseFloat(item)) } } - return !testValue?.includes(docValue) + + if (!Array.isArray(testValue)) { + return false + } + + return testValue.includes(docValue) } ) const containsAny = match( SearchFilterOperator.CONTAINS_ANY, (docValue: any, testValue: any) => { - return !docValue?.includes(...testValue) - } - ) - - const contains = match( - SearchFilterOperator.CONTAINS, - (docValue: string | any[], testValue: any[]) => { - return !testValue?.every((item: any) => docValue?.includes(item)) - } - ) - - const notContains = match( - SearchFilterOperator.NOT_CONTAINS, - (docValue: string | any[], testValue: any[]) => { - return testValue?.every((item: any) => docValue?.includes(item)) - } - ) - - const docMatch = (doc: any) => { - const filterFunctions: Record boolean> = - { - string: stringMatch, - fuzzy: fuzzyMatch, - range: rangeMatch, - equal: equalMatch, - notEqual: notEqualMatch, - empty: emptyMatch, - notEmpty: notEmptyMatch, - oneOf: oneOf, - contains: contains, - containsAny: containsAny, - notContains: notContains, + if (!Array.isArray(docValue)) { + return false } - const activeFilterKeys: SearchFilterOperator[] = Object.entries(query || {}) + if (typeof testValue === "string") { + testValue = testValue.split(",") + if (typeof docValue[0] === "number") { + testValue = testValue.map((item: string) => parseFloat(item)) + } + } + + if (!Array.isArray(testValue)) { + return false + } + + return testValue.some(item => docValue.includes(item)) + } + ) + + const _contains = (docValue: any, testValue: any) => { + if (!Array.isArray(docValue)) { + return false + } + + if (typeof testValue === "string") { + testValue = testValue.split(",") + if (typeof docValue[0] === "number") { + testValue = testValue.map((item: string) => parseFloat(item)) + } + } + + if (!Array.isArray(testValue)) { + return false + } + + return testValue.every(item => docValue.includes(item)) + } + + const contains = match(SearchFilterOperator.CONTAINS, _contains) + const notContains = match(SearchFilterOperator.NOT_CONTAINS, not(_contains)) + + const docMatch = (doc: Record) => { + const filterFunctions = { + string: stringMatch, + fuzzy: fuzzyMatch, + range: rangeMatch, + equal: equalMatch, + notEqual: notEqualMatch, + empty: emptyMatch, + notEmpty: notEmptyMatch, + oneOf: oneOf, + contains: contains, + containsAny: containsAny, + notContains: notContains, + } + + const results = Object.entries(query || {}) .filter( - ([key, value]: [string, any]) => + ([key, value]) => !["allOr", "onEmptyFilter"].includes(key) && value && - Object.keys(value as Record).length > 0 + Object.keys(value).length > 0 ) - .map(([key]) => key as any) + .map(([key]) => { + return filterFunctions[key as SearchFilterOperator]?.(doc) ?? false + }) - const results: boolean[] = activeFilterKeys.map(filterKey => { - return filterFunctions[filterKey]?.(doc) ?? false - }) - - if (query!.allOr) { + if (query.allOr) { return results.some(result => result === true) } else { return results.every(result => result === true) From 22bf0d05ad3dfca3a1deec7975d14dab547421c8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:58:13 +0100 Subject: [PATCH 04/25] Making progress. --- .../src/api/routes/tests/search.spec.ts | 19 +++-- packages/shared-core/src/filters.ts | 70 +++++++++++-------- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3cafec1b9a..8f4d96d5e8 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1706,7 +1706,7 @@ describe.each([ }) describe("contains", () => { - it.only("successfully finds a row", () => + it("successfully finds a row", () => expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, @@ -1763,9 +1763,12 @@ describe.each([ // This will never work for Lucene. !isLucene && + // It also can't work for in-memory searching because the related table name + // isn't available. + !isInMemory && describe("relations", () => { let otherTable: Table - let rows: Row[] + let otherRows: Row[] beforeAll(async () => { otherTable = await createTable({ @@ -1785,7 +1788,7 @@ describe.each([ }, }) - rows = await Promise.all([ + otherRows = await Promise.all([ config.api.row.save(otherTable._id!, { one: "foo" }), config.api.row.save(otherTable._id!, { one: "bar" }), ]) @@ -1793,18 +1796,22 @@ describe.each([ await Promise.all([ config.api.row.save(table._id!, { two: "foo", - other: [rows[0]._id], + other: [otherRows[0]._id], }), config.api.row.save(table._id!, { two: "bar", - other: [rows[1]._id], + other: [otherRows[1]._id], }), ]) + + rows = await config.api.row.fetch(table._id!) }) it("can search through relations", () => expectQuery({ equal: { [`${otherTable.name}.one`]: "foo" }, - }).toContainExactly([{ two: "foo", other: [{ _id: rows[0]._id }] }])) + }).toContainExactly([ + { two: "foo", other: [{ _id: otherRows[0]._id }] }, + ])) }) }) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index e95015d340..4ccbc60641 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -14,6 +14,7 @@ import { import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet, schema } from "./helpers" +import _ from "lodash" const HBS_REGEX = /{{([^{].*?)}}/g @@ -339,15 +340,36 @@ export const runQuery = ( } ) + // This function exists to check that either the docValue is equal to the + // testValue, or if the docValue is an object or array of objects, that the + // _id of the docValue is equal to the testValue. + const _valueMatches = (docValue: any, testValue: any) => { + if (Array.isArray(docValue)) { + for (const item of docValue) { + if (_valueMatches(item, testValue)) { + return true + } + } + return false + } + + if (typeof docValue === "object" && typeof testValue === "string") { + return docValue._id === testValue + } + + return docValue === testValue + } + const not = (f: (...args: T) => boolean) => (...args: T): boolean => !f(...args) - const _equal = (docValue: any, testValue: any) => docValue === testValue - - const equalMatch = match(SearchFilterOperator.EQUAL, _equal) - const notEqualMatch = match(SearchFilterOperator.NOT_EQUAL, not(_equal)) + const equalMatch = match(SearchFilterOperator.EQUAL, _valueMatches) + const notEqualMatch = match( + SearchFilterOperator.NOT_EQUAL, + not(_valueMatches) + ) const _empty = (docValue: any) => { if (typeof docValue === "string") { @@ -379,13 +401,12 @@ export const runQuery = ( return false } - return testValue.includes(docValue) + return testValue.some(item => _valueMatches(docValue, item)) } ) - const containsAny = match( - SearchFilterOperator.CONTAINS_ANY, - (docValue: any, testValue: any) => { + const _contains = + (f: "some" | "every") => (docValue: any, testValue: any) => { if (!Array.isArray(docValue)) { return false } @@ -401,31 +422,18 @@ export const runQuery = ( return false } - return testValue.some(item => docValue.includes(item)) + return testValue[f](item => _valueMatches(docValue, item)) } + + const contains = match(SearchFilterOperator.CONTAINS, _contains("every")) + const notContains = match( + SearchFilterOperator.NOT_CONTAINS, + not(_contains("every")) + ) + const containsAny = match( + SearchFilterOperator.CONTAINS_ANY, + _contains("some") ) - - const _contains = (docValue: any, testValue: any) => { - if (!Array.isArray(docValue)) { - return false - } - - if (typeof testValue === "string") { - testValue = testValue.split(",") - if (typeof docValue[0] === "number") { - testValue = testValue.map((item: string) => parseFloat(item)) - } - } - - if (!Array.isArray(testValue)) { - return false - } - - return testValue.every(item => docValue.includes(item)) - } - - const contains = match(SearchFilterOperator.CONTAINS, _contains) - const notContains = match(SearchFilterOperator.NOT_CONTAINS, not(_contains)) const docMatch = (doc: Record) => { const filterFunctions = { From 7e69f85e779bcb0e9155702205b0c9e9e93587cf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 18:07:46 +0100 Subject: [PATCH 05/25] More progress. --- .../src/api/routes/tests/search.spec.ts | 2 ++ packages/shared-core/src/filters.ts | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 8f4d96d5e8..f3353be3b2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1279,6 +1279,8 @@ describe.each([ { numbers: ["three"] }, ])) + // Not sure if this is correct behaviour but changing it would be a + // breaking change. it("finds all with empty list", () => expectQuery({ notContains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 4ccbc60641..2e7afa882b 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -425,10 +425,26 @@ export const runQuery = ( return testValue[f](item => _valueMatches(docValue, item)) } - const contains = match(SearchFilterOperator.CONTAINS, _contains("every")) + const contains = match( + SearchFilterOperator.CONTAINS, + (docValue: any, testValue: any) => { + if (Array.isArray(testValue) && testValue.length === 0) { + return true + } + return _contains("every")(docValue, testValue) + } + ) const notContains = match( SearchFilterOperator.NOT_CONTAINS, - not(_contains("every")) + (docValue: any, testValue: any) => { + // Not sure if this is logically correct, but at the time this code was + // written the search endpoint behaved this way and we wanted to make this + // local search match its behaviour, so we had to do this. + if (Array.isArray(testValue) && testValue.length === 0) { + return true + } + return not(_contains("every"))(docValue, testValue) + } ) const containsAny = match( SearchFilterOperator.CONTAINS_ANY, From 6a2b65b75bebd830d5181b61aa091c7ab8cc42f7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 09:56:33 +0100 Subject: [PATCH 06/25] Down to 75 failures. Started at 91. --- packages/shared-core/src/filters.ts | 34 +++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 2e7afa882b..1699d28b5e 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -320,21 +320,33 @@ export const runQuery = ( const rangeMatch = match( SearchFilterOperator.RANGE, - ( - docValue: string | number | null, - testValue: { low: number; high: number } - ) => { + (docValue: any, testValue: any) => { if (docValue == null || docValue === "") { return false } + if (testValue.low == null && testValue.high == null) { + return false + } if (!isNaN(+docValue)) { - return +docValue >= testValue.low || +docValue <= testValue.high + if (!isNaN(+testValue.low) && !isNaN(+testValue.high)) { + return +docValue >= testValue.low && +docValue <= testValue.high + } else if (!isNaN(+testValue.low)) { + return +docValue >= testValue.low + } else if (!isNaN(+testValue.high)) { + return +docValue <= testValue.high + } } if (dayjs(docValue).isValid()) { - return ( - new Date(docValue).getTime() >= new Date(testValue.low).getTime() || - new Date(docValue).getTime() <= new Date(testValue.high).getTime() - ) + if (dayjs(testValue.low).isValid() && dayjs(testValue.high).isValid()) { + return ( + dayjs(docValue).isAfter(testValue.low) && + dayjs(docValue).isBefore(testValue.high) + ) + } else if (dayjs(testValue.low).isValid()) { + return dayjs(docValue).isAfter(testValue.low) + } else if (dayjs(testValue.high).isValid()) { + return dayjs(docValue).isBefore(testValue.high) + } } return false } @@ -422,6 +434,10 @@ export const runQuery = ( return false } + if (testValue.length === 0) { + return true + } + return testValue[f](item => _valueMatches(docValue, item)) } From cb6acd8c0b26801578c020b4bc234569e9ab1a6f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 12:24:27 +0100 Subject: [PATCH 07/25] Down to 71 failures. --- .../src/api/routes/tests/search.spec.ts | 12 +++++------ packages/shared-core/src/filters.ts | 21 ++++++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f3353be3b2..ec6c2bf304 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1079,13 +1079,13 @@ describe.each([ !isInternal && describe("datetime - time only", () => { - const T_1000 = "10:00" - const T_1045 = "10:45" - const T_1200 = "12:00" - const T_1530 = "15:30" - const T_0000 = "00:00" + const T_1000 = "10:00:00" + const T_1045 = "10:45:00" + const T_1200 = "12:00:00" + const T_1530 = "15:30:00" + const T_0000 = "00:00:00" - const UNEXISTING_TIME = "10:01" + const UNEXISTING_TIME = "10:01:00" const NULL_TIME__ID = `null_time__id` diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 1699d28b5e..a773524ae4 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -519,18 +519,29 @@ export const sort = ( if (!sort || !sortOrder || !sortType) { return docs } - const parse = - sortType === "string" ? (x: any) => `${x}` : (x: string) => parseFloat(x) + + const parse = (x: any) => { + if (x == null) { + return x + } + if (sortType === "string") { + return `${x}` + } + return parseFloat(x) + } + return docs .slice() .sort((a: { [x: string]: any }, b: { [x: string]: any }) => { const colA = parse(a[sort]) const colB = parse(b[sort]) + + const result = colB == null || colA > colB ? 1 : -1 if (sortOrder.toLowerCase() === "descending") { - return colA > colB ? -1 : 1 - } else { - return colA > colB ? 1 : -1 + return result * -1 } + + return result }) } From 69ab1ce44f4e9b0320582d11524facf2c1adad24 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 12:30:36 +0100 Subject: [PATCH 08/25] Down to 66 failures. --- packages/shared-core/src/filters.ts | 32 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index a773524ae4..43c99c8374 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -324,9 +324,11 @@ export const runQuery = ( if (docValue == null || docValue === "") { return false } + if (testValue.low == null && testValue.high == null) { return false } + if (!isNaN(+docValue)) { if (!isNaN(+testValue.low) && !isNaN(+testValue.high)) { return +docValue >= testValue.low && +docValue <= testValue.high @@ -336,18 +338,32 @@ export const runQuery = ( return +docValue <= testValue.high } } - if (dayjs(docValue).isValid()) { - if (dayjs(testValue.low).isValid() && dayjs(testValue.high).isValid()) { + + const docDate = dayjs(docValue) + if (docDate.isValid()) { + const lowDate = dayjs(testValue.low) + const highDate = dayjs(testValue.high) + if (lowDate.isValid() && highDate.isValid()) { return ( - dayjs(docValue).isAfter(testValue.low) && - dayjs(docValue).isBefore(testValue.high) + (docDate.isAfter(lowDate) && docDate.isBefore(highDate)) || + docDate.isSame(lowDate) || + docDate.isSame(highDate) ) - } else if (dayjs(testValue.low).isValid()) { - return dayjs(docValue).isAfter(testValue.low) - } else if (dayjs(testValue.high).isValid()) { - return dayjs(docValue).isBefore(testValue.high) + } else if (lowDate.isValid()) { + return docDate.isAfter(lowDate) || docDate.isSame(lowDate) + } else if (highDate.isValid()) { + return docDate.isBefore(highDate) || docDate.isSame(highDate) } } + + if (testValue.low && testValue.high) { + return docValue >= testValue.low && docValue <= testValue.high + } else if (testValue.low) { + return docValue >= testValue.low + } else if (testValue.high) { + return docValue <= testValue.high + } + return false } ) From a82da51b30a79b2974d4b9943b4476be1c32a91d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 12:34:22 +0100 Subject: [PATCH 09/25] Down to 60 failures. --- packages/shared-core/src/filters.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 43c99c8374..0b452c8cb8 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -381,7 +381,11 @@ export const runQuery = ( return false } - if (typeof docValue === "object" && typeof testValue === "string") { + if ( + docValue && + typeof docValue === "object" && + typeof testValue === "string" + ) { return docValue._id === testValue } From 854347f9f5c57ee04f1be94b02386275ffd5f23d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 14:42:34 +0100 Subject: [PATCH 10/25] Down to 59 failures. --- packages/shared-core/src/filters.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 0b452c8cb8..ae6dd8e15b 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -341,8 +341,8 @@ export const runQuery = ( const docDate = dayjs(docValue) if (docDate.isValid()) { - const lowDate = dayjs(testValue.low) - const highDate = dayjs(testValue.high) + const lowDate = dayjs(testValue.low || "0000-00-00T00:00:00.000Z") + const highDate = dayjs(testValue.high || "9999-00-00T00:00:00.000Z") if (lowDate.isValid() && highDate.isValid()) { return ( (docDate.isAfter(lowDate) && docDate.isBefore(highDate)) || @@ -356,11 +356,11 @@ export const runQuery = ( } } - if (testValue.low && testValue.high) { + if (testValue.low != null && testValue.high != null) { return docValue >= testValue.low && docValue <= testValue.high - } else if (testValue.low) { + } else if (testValue.low != null) { return docValue >= testValue.low - } else if (testValue.high) { + } else if (testValue.high != null) { return docValue <= testValue.high } From 746ee711ae949fd4fbcce02f50689f02e54001eb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 14:45:43 +0100 Subject: [PATCH 11/25] Down to 19 failures. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ec6c2bf304..1c0747ba3f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -94,7 +94,7 @@ describe.each([ private async performSearch(): Promise { if (isInMemory) { - let result = dataFilters.runQuery(rows, this.query.query) + let result = dataFilters.runQuery(_.cloneDeep(rows), this.query.query) if (this.query.sort) { result = dataFilters.sort( result, From c01c2c7cc34be144654f313ac421862b6c347aab Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 16:23:16 +0100 Subject: [PATCH 12/25] Down to 4 failures. --- .../src/api/routes/tests/search.spec.ts | 586 +++++++++--------- packages/shared-core/src/filters.ts | 25 +- 2 files changed, 315 insertions(+), 296 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1c0747ba3f..afc575b2c4 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -313,351 +313,359 @@ describe.each([ }) }) - // Ensure all bindings resolve and perform as expected - describe("bindings", () => { - let globalUsers: any = [] + // We've decided not to try and support binding for in-memory search just now. + !isInMemory && + describe("bindings", () => { + let globalUsers: any = [] - const serverTime = new Date() + const serverTime = new Date() - // In MariaDB and MySQL we only store dates to second precision, so we need - // to remove milliseconds from the server time to ensure searches work as - // expected. - serverTime.setMilliseconds(0) + // In MariaDB and MySQL we only store dates to second precision, so we need + // to remove milliseconds from the server time to ensure searches work as + // expected. + serverTime.setMilliseconds(0) - const future = new Date(serverTime.getTime() + 1000 * 60 * 60 * 24 * 30) + const future = new Date(serverTime.getTime() + 1000 * 60 * 60 * 24 * 30) - const rows = (currentUser: User) => { - return [ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - { name: currentUser.firstName, appointment: future.toISOString() }, - { name: "serverDate", appointment: serverTime.toISOString() }, - { - name: "single user, session user", - single_user: JSON.stringify(currentUser), - }, - { - name: "single user", - single_user: JSON.stringify(globalUsers[0]), - }, - { - name: "deprecated single user, session user", - deprecated_single_user: JSON.stringify([currentUser]), - }, - { - name: "deprecated single user", - deprecated_single_user: JSON.stringify([globalUsers[0]]), - }, - { - name: "multi user", - multi_user: JSON.stringify(globalUsers), - }, - { - name: "multi user with session user", - multi_user: JSON.stringify([...globalUsers, currentUser]), - }, - { - name: "deprecated multi user", - deprecated_multi_user: JSON.stringify(globalUsers), - }, - { - name: "deprecated multi user with session user", - deprecated_multi_user: JSON.stringify([...globalUsers, currentUser]), - }, - ] - } - - beforeAll(async () => { - // Set up some global users - globalUsers = await Promise.all( - Array(2) - .fill(0) - .map(async () => { - const globalUser = await config.globalUser() - const userMedataId = globalUser._id - ? dbCore.generateUserMetadataID(globalUser._id) - : null - return { - _id: globalUser._id, - _meta: userMedataId, - } - }) - ) - - table = await createTable({ - name: { name: "name", type: FieldType.STRING }, - appointment: { name: "appointment", type: FieldType.DATETIME }, - single_user: { - name: "single_user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, - deprecated_single_user: { - name: "deprecated_single_user", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, - multi_user: { - name: "multi_user", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - constraints: { - type: "array", + const rows = (currentUser: User) => { + return [ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, + { name: currentUser.firstName, appointment: future.toISOString() }, + { name: "serverDate", appointment: serverTime.toISOString() }, + { + name: "single user, session user", + single_user: JSON.stringify(currentUser), }, - }, - deprecated_multi_user: { - name: "deprecated_multi_user", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, - constraints: { - type: "array", + { + name: "single user", + single_user: JSON.stringify(globalUsers[0]), }, - }, - }) - await createRows(rows(config.getUser())) - }) + { + name: "deprecated single user, session user", + deprecated_single_user: JSON.stringify([currentUser]), + }, + { + name: "deprecated single user", + deprecated_single_user: JSON.stringify([globalUsers[0]]), + }, + { + name: "multi user", + multi_user: JSON.stringify(globalUsers), + }, + { + name: "multi user with session user", + multi_user: JSON.stringify([...globalUsers, currentUser]), + }, + { + name: "deprecated multi user", + deprecated_multi_user: JSON.stringify(globalUsers), + }, + { + name: "deprecated multi user with session user", + deprecated_multi_user: JSON.stringify([ + ...globalUsers, + currentUser, + ]), + }, + ] + } - // !! Current User is auto generated per run - it("should return all rows matching the session user firstname", async () => { - await expectQuery({ - equal: { name: "{{ [user].firstName }}" }, - }).toContainExactly([ - { - name: config.getUser().firstName, - appointment: future.toISOString(), - }, - ]) - }) + beforeAll(async () => { + // Set up some global users + globalUsers = await Promise.all( + Array(2) + .fill(0) + .map(async () => { + const globalUser = await config.globalUser() + const userMedataId = globalUser._id + ? dbCore.generateUserMetadataID(globalUser._id) + : null + return { + _id: globalUser._id, + _meta: userMedataId, + } + }) + ) - it("should parse the date binding and return all rows after the resolved value", async () => { - await tk.withFreeze(serverTime, async () => { - await expectQuery({ - range: { - appointment: { - low: "{{ [now] }}", - high: "9999-00-00T00:00:00.000Z", + table = await createTable({ + name: { name: "name", type: FieldType.STRING }, + appointment: { name: "appointment", type: FieldType.DATETIME }, + single_user: { + name: "single_user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + deprecated_single_user: { + name: "deprecated_single_user", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + }, + multi_user: { + name: "multi_user", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: "array", }, }, + deprecated_multi_user: { + name: "deprecated_multi_user", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USERS, + constraints: { + type: "array", + }, + }, + }) + await createRows(rows(config.getUser())) + }) + + // !! Current User is auto generated per run + it("should return all rows matching the session user firstname", async () => { + await expectQuery({ + equal: { name: "{{ [user].firstName }}" }, }).toContainExactly([ { name: config.getUser().firstName, appointment: future.toISOString(), }, + ]) + }) + + it("should parse the date binding and return all rows after the resolved value", async () => { + await tk.withFreeze(serverTime, async () => { + await expectQuery({ + range: { + appointment: { + low: "{{ [now] }}", + high: "9999-00-00T00:00:00.000Z", + }, + }, + }).toContainExactly([ + { + name: config.getUser().firstName, + appointment: future.toISOString(), + }, + { name: "serverDate", appointment: serverTime.toISOString() }, + ]) + }) + }) + + it("should parse the date binding and return all rows before the resolved value", async () => { + await expectQuery({ + range: { + appointment: { + low: "0000-00-00T00:00:00.000Z", + high: "{{ [now] }}", + }, + }, + }).toContainExactly([ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, { name: "serverDate", appointment: serverTime.toISOString() }, ]) }) - }) - it("should parse the date binding and return all rows before the resolved value", async () => { - await expectQuery({ - range: { - appointment: { - low: "0000-00-00T00:00:00.000Z", - high: "{{ [now] }}", + it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => { + const jsBinding = "return snippets.WeeksAgo();" + const encodedBinding = encodeJSBinding(jsBinding) + + await expectQuery({ + range: { + appointment: { + low: "0000-00-00T00:00:00.000Z", + high: encodedBinding, + }, }, - }, - }).toContainExactly([ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - { name: "serverDate", appointment: serverTime.toISOString() }, - ]) - }) + }).toContainExactly([ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, + ]) + }) - it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => { - const jsBinding = "return snippets.WeeksAgo();" - const encodedBinding = encodeJSBinding(jsBinding) + it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { + const jsBinding = `const currentTime = new Date(${Date.now()})\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();` + const encodedBinding = encodeJSBinding(jsBinding) - await expectQuery({ - range: { - appointment: { - low: "0000-00-00T00:00:00.000Z", - high: encodedBinding, + await expectQuery({ + range: { + appointment: { + low: "0000-00-00T00:00:00.000Z", + high: encodedBinding, + }, }, - }, - }).toContainExactly([ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - ]) - }) + }).toContainExactly([ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, + ]) + }) - it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { - const jsBinding = `const currentTime = new Date(${Date.now()})\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();` - const encodedBinding = encodeJSBinding(jsBinding) - - await expectQuery({ - range: { - appointment: { - low: "0000-00-00T00:00:00.000Z", - high: encodedBinding, + it("should match a single user row by the session user id", async () => { + await expectQuery({ + equal: { single_user: "{{ [user]._id }}" }, + }).toContainExactly([ + { + name: "single user, session user", + single_user: { _id: config.getUser()._id }, }, - }, - }).toContainExactly([ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - ]) - }) + ]) + }) - it("should match a single user row by the session user id", async () => { - await expectQuery({ - equal: { single_user: "{{ [user]._id }}" }, - }).toContainExactly([ - { - name: "single user, session user", - single_user: { _id: config.getUser()._id }, - }, - ]) - }) + it("should match a deprecated single user row by the session user id", async () => { + await expectQuery({ + equal: { deprecated_single_user: "{{ [user]._id }}" }, + }).toContainExactly([ + { + name: "deprecated single user, session user", + deprecated_single_user: [{ _id: config.getUser()._id }], + }, + ]) + }) - it("should match a deprecated single user row by the session user id", async () => { - await expectQuery({ - equal: { deprecated_single_user: "{{ [user]._id }}" }, - }).toContainExactly([ - { - name: "deprecated single user, session user", - deprecated_single_user: [{ _id: config.getUser()._id }], - }, - ]) - }) + // TODO(samwho): fix for SQS + !isSqs && + it("should match the session user id in a multi user field", async () => { + const allUsers = [...globalUsers, config.getUser()].map( + (user: any) => { + return { _id: user._id } + } + ) - // TODO(samwho): fix for SQS - !isSqs && - it("should match the session user id in a multi user field", async () => { - const allUsers = [...globalUsers, config.getUser()].map((user: any) => { - return { _id: user._id } + await expectQuery({ + contains: { multi_user: ["{{ [user]._id }}"] }, + }).toContainExactly([ + { + name: "multi user with session user", + multi_user: allUsers, + }, + ]) }) - await expectQuery({ - contains: { multi_user: ["{{ [user]._id }}"] }, - }).toContainExactly([ - { - name: "multi user with session user", - multi_user: allUsers, - }, - ]) - }) + // TODO(samwho): fix for SQS + !isSqs && + it("should match the session user id in a deprecated multi user field", async () => { + const allUsers = [...globalUsers, config.getUser()].map( + (user: any) => { + return { _id: user._id } + } + ) - // TODO(samwho): fix for SQS - !isSqs && - it("should match the session user id in a deprecated multi user field", async () => { - const allUsers = [...globalUsers, config.getUser()].map((user: any) => { - return { _id: user._id } + await expectQuery({ + contains: { deprecated_multi_user: ["{{ [user]._id }}"] }, + }).toContainExactly([ + { + name: "deprecated multi user with session user", + deprecated_multi_user: allUsers, + }, + ]) }) + // TODO(samwho): fix for SQS + !isSqs && + it("should not match the session user id in a multi user field", async () => { + await expectQuery({ + notContains: { multi_user: ["{{ [user]._id }}"] }, + notEmpty: { multi_user: true }, + }).toContainExactly([ + { + name: "multi user", + multi_user: globalUsers.map((user: any) => { + return { _id: user._id } + }), + }, + ]) + }) + + // TODO(samwho): fix for SQS + !isSqs && + it("should not match the session user id in a deprecated multi user field", async () => { + await expectQuery({ + notContains: { deprecated_multi_user: ["{{ [user]._id }}"] }, + notEmpty: { deprecated_multi_user: true }, + }).toContainExactly([ + { + name: "deprecated multi user", + deprecated_multi_user: globalUsers.map((user: any) => { + return { _id: user._id } + }), + }, + ]) + }) + + it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => { await expectQuery({ - contains: { deprecated_multi_user: ["{{ [user]._id }}"] }, + oneOf: { + single_user: [ + "{{ default [user]._id '_empty_' }}", + globalUsers[0]._id, + ], + }, }).toContainExactly([ { - name: "deprecated multi user with session user", - deprecated_multi_user: allUsers, + name: "single user, session user", + single_user: { _id: config.getUser()._id }, + }, + { + name: "single user", + single_user: { _id: globalUsers[0]._id }, }, ]) }) - // TODO(samwho): fix for SQS - !isSqs && - it("should not match the session user id in a multi user field", async () => { + it("should match the session user id and a user table row id using helpers, user binding and a static user id. (deprecated single user)", async () => { await expectQuery({ - notContains: { multi_user: ["{{ [user]._id }}"] }, - notEmpty: { multi_user: true }, + oneOf: { + deprecated_single_user: [ + "{{ default [user]._id '_empty_' }}", + globalUsers[0]._id, + ], + }, }).toContainExactly([ { - name: "multi user", - multi_user: globalUsers.map((user: any) => { - return { _id: user._id } - }), + name: "deprecated single user, session user", + deprecated_single_user: [{ _id: config.getUser()._id }], + }, + { + name: "deprecated single user", + deprecated_single_user: [{ _id: globalUsers[0]._id }], }, ]) }) - // TODO(samwho): fix for SQS - !isSqs && - it("should not match the session user id in a deprecated multi user field", async () => { + it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => { await expectQuery({ - notContains: { deprecated_multi_user: ["{{ [user]._id }}"] }, - notEmpty: { deprecated_multi_user: true }, + oneOf: { + single_user: [ + "{{ default [user]._idx '_empty_' }}", + globalUsers[0]._id, + ], + }, }).toContainExactly([ { - name: "deprecated multi user", - deprecated_multi_user: globalUsers.map((user: any) => { - return { _id: user._id } - }), + name: "single user", + single_user: { _id: globalUsers[0]._id }, }, ]) }) - it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => { - await expectQuery({ - oneOf: { - single_user: [ - "{{ default [user]._id '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "single user, session user", - single_user: { _id: config.getUser()._id }, - }, - { - name: "single user", - single_user: { _id: globalUsers[0]._id }, - }, - ]) + it("should resolve 'default' helper to '_empty_' when binding resolves to nothing (deprecated single user)", async () => { + await expectQuery({ + oneOf: { + deprecated_single_user: [ + "{{ default [user]._idx '_empty_' }}", + globalUsers[0]._id, + ], + }, + }).toContainExactly([ + { + name: "deprecated single user", + deprecated_single_user: [{ _id: globalUsers[0]._id }], + }, + ]) + }) }) - it("should match the session user id and a user table row id using helpers, user binding and a static user id. (deprecated single user)", async () => { - await expectQuery({ - oneOf: { - deprecated_single_user: [ - "{{ default [user]._id '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "deprecated single user, session user", - deprecated_single_user: [{ _id: config.getUser()._id }], - }, - { - name: "deprecated single user", - deprecated_single_user: [{ _id: globalUsers[0]._id }], - }, - ]) - }) - - it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => { - await expectQuery({ - oneOf: { - single_user: [ - "{{ default [user]._idx '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "single user", - single_user: { _id: globalUsers[0]._id }, - }, - ]) - }) - - it("should resolve 'default' helper to '_empty_' when binding resolves to nothing (deprecated single user)", async () => { - await expectQuery({ - oneOf: { - deprecated_single_user: [ - "{{ default [user]._idx '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "deprecated single user", - deprecated_single_user: [{ _id: globalUsers[0]._id }], - }, - ]) - }) - }) - describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { beforeAll(async () => { table = await createTable({ diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ae6dd8e15b..391b9a795f 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -325,17 +325,28 @@ export const runQuery = ( return false } + if (_.isObject(testValue.low) && _.isEmpty(testValue.low)) { + testValue.low = undefined + } + + if (_.isObject(testValue.high) && _.isEmpty(testValue.high)) { + testValue.high = undefined + } + if (testValue.low == null && testValue.high == null) { return false } - if (!isNaN(+docValue)) { - if (!isNaN(+testValue.low) && !isNaN(+testValue.high)) { - return +docValue >= testValue.low && +docValue <= testValue.high - } else if (!isNaN(+testValue.low)) { - return +docValue >= testValue.low - } else if (!isNaN(+testValue.high)) { - return +docValue <= testValue.high + const docNum = +docValue + if (!isNaN(docNum)) { + const lowNum = +testValue.low + const highNum = +testValue.high + if (!isNaN(lowNum) && !isNaN(highNum)) { + return docNum >= lowNum && docNum <= highNum + } else if (!isNaN(lowNum)) { + return docNum >= lowNum + } else if (!isNaN(highNum)) { + return docNum <= highNum } } From f909cdee43e3a880537006d0a0af5dbaf4e77eb5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 17:05:02 +0100 Subject: [PATCH 13/25] Down to 2 failures. --- packages/shared-core/src/filters.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 391b9a795f..d63975f8fc 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -285,7 +285,10 @@ export const runQuery = ( ) => (doc: Record) => { for (const [key, testValue] of Object.entries(query[type] || {})) { - if (!test(deepGet(doc, removeKeyNumbering(key)), testValue)) { + const result = test(deepGet(doc, removeKeyNumbering(key)), testValue) + if (query.allOr && result) { + return true + } else if (!query.allOr && !result) { return false } } From 1e34411d667e6ff9f69812e2ec834005071d25f8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jun 2024 17:29:22 +0100 Subject: [PATCH 14/25] Adding the correct link for the migration pages. --- .../pages/builder/maintenance/index.svelte | 3 ++- yarn.lock | 24 +++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/pages/builder/maintenance/index.svelte b/packages/builder/src/pages/builder/maintenance/index.svelte index e4c379885a..f7eb16ab81 100644 --- a/packages/builder/src/pages/builder/maintenance/index.svelte +++ b/packages/builder/src/pages/builder/maintenance/index.svelte @@ -33,7 +33,8 @@ {/if} diff --git a/yarn.lock b/yarn.lock index 426fa2275d..d71dd4da78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3495,10 +3495,10 @@ dependencies: lodash "^4.17.21" -"@koa/cors@^3.1.0": - version "3.4.3" - resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-3.4.3.tgz#d669ee6e8d6e4f0ec4a7a7b0a17e7a3ed3752ebb" - integrity sha512-WPXQUaAeAMVaLTEFpoq3T2O1C+FstkjJnDQqy95Ck1UdILajsRhu6mhJ8H2f4NFPRBoCNN+qywTJfq/gGki5mw== +"@koa/cors@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-5.0.0.tgz#0029b5f057fa0d0ae0e37dd2c89ece315a0daffd" + integrity sha512-x/iUDjcS90W69PryLDIMgFyV21YLTnG9zOpPXS7Bkt2b8AsY3zZsIpOLBkYr9fBcF3HbkKaER5hOBZLfpLgYNw== dependencies: vary "^1.1.2" @@ -5817,10 +5817,10 @@ "@types/koa-compose" "*" "@types/node" "*" -"@types/koa__cors@^3.1.1": - version "3.3.1" - resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.1.tgz#0ec7543c4c620fd23451bfdd3e21b9a6aadedccd" - integrity sha512-aFGYhTFW7651KhmZZ05VG0QZJre7QxBxDj2LF1lf6GA/wSXEfKVAJxiQQWzRV4ZoMzQIO8vJBXKsUcRuvYK9qw== +"@types/koa__cors@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-5.0.0.tgz#74567a045b599266e2cd3940cef96cedecc2ef1f" + integrity sha512-LCk/n25Obq5qlernGOK/2LUwa/2YJb2lxHUkkvYFDOpLXlVI6tKcdfCHRBQnOY4LwH6el5WOLs6PD/a8Uzau6g== dependencies: "@types/koa" "*" @@ -16343,10 +16343,10 @@ node-source-walk@^5.0.0: dependencies: "@babel/parser" "^7.0.0" -nodemailer@6.7.2: - version "6.7.2" - resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0" - integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q== +nodemailer@6.9.13: + version "6.9.13" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.13.tgz#5b292bf1e92645f4852ca872c56a6ba6c4a3d3d6" + integrity sha512-7o38Yogx6krdoBf3jCAqnIN4oSQFx+fMa0I7dK1D+me9kBxx12D+/33wSb+fhOCtIxvYJ+4x4IMEhmhCKfAiOA== nodemailer@6.9.9: version "6.9.9" From 1161c185e21b3eb48cd357920382914b5b8fe17f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 13 Jun 2024 17:46:03 +0100 Subject: [PATCH 15/25] Down to 0 failures. --- .../src/api/routes/tests/search.spec.ts | 25 ++++++------------- packages/shared-core/src/filters.ts | 20 +++++++++++++++ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index afc575b2c4..22971c9c1f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -26,12 +26,12 @@ import { dataFilters } from "@budibase/shared-core" describe.each([ ["in-memory", undefined], - // ["lucene", undefined], - // ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + ["lucene", undefined], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -94,18 +94,7 @@ describe.each([ private async performSearch(): Promise { if (isInMemory) { - let result = dataFilters.runQuery(_.cloneDeep(rows), this.query.query) - if (this.query.sort) { - result = dataFilters.sort( - result, - this.query.sort, - this.query.sortOrder || SortOrder.ASCENDING - ) - } - if (this.query.limit) { - result = dataFilters.limit(result, this.query.limit.toString()) - } - return result + return dataFilters.search(_.cloneDeep(rows), this.query) } else { return ( await config.api.row.search(table._id!, { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index d63975f8fc..14b9f004d3 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -10,6 +10,8 @@ import { SortType, FieldConstraints, SortOrder, + RowSearchParams, + EmptyFilterOption, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -260,6 +262,17 @@ export const buildQuery = (filter: SearchFilter[]) => { return query } +export const search = (docs: Record[], query: RowSearchParams) => { + let result = runQuery(docs, query.query) + if (query.sort) { + result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) + } + if (query.limit) { + result = limit(result, query.limit.toString()) + } + return result +} + /** * Performs a client-side search on an array of data * @param docs the data @@ -278,6 +291,13 @@ export const runQuery = ( query = cleanupQuery(query) + if ( + !hasFilters(query) && + query.onEmptyFilter === EmptyFilterOption.RETURN_NONE + ) { + return [] + } + const match = ( type: SearchFilterOperator, From 8970705b399d86e01285d61167ea704a55233dad Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jun 2024 17:49:41 +0100 Subject: [PATCH 16/25] Adding a minimum time to the app migration screen and adding a link to documentation. --- .../frontend-core/src/components/Updating.svelte | 15 ++++++++++++--- .../src/appMigrations/appMigrationMetadata.ts | 9 ++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 7d4a101fee..7d14e57aba 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -1,18 +1,22 @@