From d51bbb7952fcc268335f5d86eed14cb44f300e49 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 10 Apr 2024 16:38:52 +0100 Subject: [PATCH 01/24] When looking through the search parameters to build up information about validating search inputs better on the API (recently created linear issue) found the public docs weren't fully up to date - fixing this. --- packages/server/specs/openapi.json | 35 +++++++++++++++++++++- packages/server/specs/openapi.yaml | 31 ++++++++++++++++++- packages/server/specs/resources/misc.ts | 27 ++++++++++++++++- packages/server/src/definitions/openapi.ts | 20 ++++++++++++- 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 1cf2d0ebce..7eb3edfd33 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -853,6 +853,7 @@ "array", "datetime", "attachment", + "attachment_single", "link", "formula", "auto", @@ -1059,6 +1060,7 @@ "array", "datetime", "attachment", + "attachment_single", "link", "formula", "auto", @@ -1276,6 +1278,7 @@ "array", "datetime", "attachment", + "attachment_single", "link", "formula", "auto", @@ -1752,7 +1755,7 @@ }, "fuzzy": { "type": "object", - "description": "A fuzzy search, only supported by internal tables." + "description": "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'." }, "range": { "type": "object", @@ -1786,6 +1789,36 @@ "oneOf": { "type": "object", "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]." + }, + "contains": { + "type": "object", + "description": "Searches for a value, or set of values in an array column types (such as a multi-select), if an array of search options is provided then it must match all.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "notContains": { + "type": "object", + "description": "As with the contains search, only functions for array column types, but searches for columns missing the supplied values.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "containsAny": { + "type": "object", + "description": "As with the contains search, only functions for array column types, but searches for any of the provided values when given an array.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } } } }, diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index a1615e7426..2ca35fe7af 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -775,6 +775,7 @@ components: - array - datetime - attachment + - attachment_single - link - formula - auto @@ -940,6 +941,7 @@ components: - array - datetime - attachment + - attachment_single - link - formula - auto @@ -1112,6 +1114,7 @@ components: - array - datetime - attachment + - attachment_single - link - formula - auto @@ -1492,7 +1495,8 @@ components: description: The value to search for in the column. fuzzy: type: object - description: A fuzzy search, only supported by internal tables. + description: Searches for a sub-string within a string column, e.g. searching + for 'dib' will match 'Budibase'. range: type: object description: Searches within a range, the format of this must be in the format @@ -1524,6 +1528,31 @@ components: description: Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. + contains: + type: object + description: Searches for a value, or set of values in an array column types + (such as a multi-select), if an array of search options is + provided then it must match all. + example: + arrayColumn: + - a + - b + notContains: + type: object + description: As with the contains search, only functions for array column types, + but searches for columns missing the supplied values. + example: + arrayColumn: + - a + - b + containsAny: + type: object + description: As with the contains search, only functions for array column types, + but searches for any of the provided values when given an array. + example: + arrayColumn: + - a + - b paginate: type: boolean description: Enables pagination, by default this is disabled. diff --git a/packages/server/specs/resources/misc.ts b/packages/server/specs/resources/misc.ts index 16cb831c86..4272ba44e1 100644 --- a/packages/server/specs/resources/misc.ts +++ b/packages/server/specs/resources/misc.ts @@ -27,7 +27,8 @@ export default new Resource().setSchemas({ }, fuzzy: { type: "object", - description: "A fuzzy search, only supported by internal tables.", + description: + "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.", }, range: { type: "object", @@ -67,6 +68,30 @@ export default new Resource().setSchemas({ description: "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].", }, + contains: { + type: "object", + description: + "Searches for a value, or set of values in an array column types (such as a multi-select), if an array of search options is provided then it must match all.", + example: { + arrayColumn: ["a", "b"], + }, + }, + notContains: { + type: "object", + description: + "As with the contains search, only functions for array column types, but searches for columns missing the supplied values.", + example: { + arrayColumn: ["a", "b"], + }, + }, + containsAny: { + type: "object", + description: + "As with the contains search, only functions for array column types, but searches for any of the provided values when given an array.", + example: { + arrayColumn: ["a", "b"], + }, + }, }, }, paginate: { diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 34014ba626..2828ca85bc 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -273,6 +273,7 @@ export interface components { | "array" | "datetime" | "attachment" + | "attachment_single" | "link" | "formula" | "auto" @@ -381,6 +382,7 @@ export interface components { | "array" | "datetime" | "attachment" + | "attachment_single" | "link" | "formula" | "auto" @@ -491,6 +493,7 @@ export interface components { | "array" | "datetime" | "attachment" + | "attachment_single" | "link" | "formula" | "auto" @@ -693,7 +696,7 @@ export interface components { * @example [object Object] */ string?: { [key: string]: string }; - /** @description A fuzzy search, only supported by internal tables. */ + /** @description Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'. */ fuzzy?: { [key: string]: unknown }; /** * @description Searches within a range, the format of this must be in the format of an object with a "low" and "high" property. @@ -713,6 +716,21 @@ export interface components { notEmpty?: { [key: string]: unknown }; /** @description Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. */ oneOf?: { [key: string]: unknown }; + /** + * @description Searches for a value, or set of values in an array column types (such as a multi-select), if an array of search options is provided then it must match all. + * @example [object Object] + */ + contains?: { [key: string]: unknown }; + /** + * @description As with the contains search, only functions for array column types, but searches for columns missing the supplied values. + * @example [object Object] + */ + notContains?: { [key: string]: unknown }; + /** + * @description As with the contains search, only functions for array column types, but searches for any of the provided values when given an array. + * @example [object Object] + */ + containsAny?: { [key: string]: unknown }; }; /** @description Enables pagination, by default this is disabled. */ paginate?: boolean; From 4d3d78f7a0b58511ce017e5db844a96e65db2e55 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 10 Apr 2024 17:02:03 +0100 Subject: [PATCH 02/24] PR comments. --- packages/server/specs/openapi.json | 6 +++--- packages/server/specs/openapi.yaml | 15 ++++++++------- packages/server/specs/resources/misc.ts | 6 +++--- packages/server/src/definitions/openapi.ts | 6 +++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 7eb3edfd33..7d07b424f0 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -1792,7 +1792,7 @@ }, "contains": { "type": "object", - "description": "Searches for a value, or set of values in an array column types (such as a multi-select), if an array of search options is provided then it must match all.", + "description": "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", "example": { "arrayColumn": [ "a", @@ -1802,7 +1802,7 @@ }, "notContains": { "type": "object", - "description": "As with the contains search, only functions for array column types, but searches for columns missing the supplied values.", + "description": "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", "example": { "arrayColumn": [ "a", @@ -1812,7 +1812,7 @@ }, "containsAny": { "type": "object", - "description": "As with the contains search, only functions for array column types, but searches for any of the provided values when given an array.", + "description": "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", "example": { "arrayColumn": [ "a", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 2ca35fe7af..3a798c424b 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -1530,25 +1530,26 @@ components: [value1, value2]. contains: type: object - description: Searches for a value, or set of values in an array column types - (such as a multi-select), if an array of search options is - provided then it must match all. + description: Searches for a value, or set of values in array column types (such + as a multi-select). If an array of search options is provided + then it must match all. example: arrayColumn: - a - b notContains: type: object - description: As with the contains search, only functions for array column types, - but searches for columns missing the supplied values. + description: The logical inverse of contains. Only works on array column types. + If an array of values is passed, the row must not match any of + them to be returned in the response. example: arrayColumn: - a - b containsAny: type: object - description: As with the contains search, only functions for array column types, - but searches for any of the provided values when given an array. + description: As with the contains search, only works on array column types and + searches for any of the provided values when given an array. example: arrayColumn: - a diff --git a/packages/server/specs/resources/misc.ts b/packages/server/specs/resources/misc.ts index 4272ba44e1..f56dff3301 100644 --- a/packages/server/specs/resources/misc.ts +++ b/packages/server/specs/resources/misc.ts @@ -71,7 +71,7 @@ export default new Resource().setSchemas({ contains: { type: "object", description: - "Searches for a value, or set of values in an array column types (such as a multi-select), if an array of search options is provided then it must match all.", + "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", example: { arrayColumn: ["a", "b"], }, @@ -79,7 +79,7 @@ export default new Resource().setSchemas({ notContains: { type: "object", description: - "As with the contains search, only functions for array column types, but searches for columns missing the supplied values.", + "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", example: { arrayColumn: ["a", "b"], }, @@ -87,7 +87,7 @@ export default new Resource().setSchemas({ containsAny: { type: "object", description: - "As with the contains search, only functions for array column types, but searches for any of the provided values when given an array.", + "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", example: { arrayColumn: ["a", "b"], }, diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 2828ca85bc..cc3c8a7d4d 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -717,17 +717,17 @@ export interface components { /** @description Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. */ oneOf?: { [key: string]: unknown }; /** - * @description Searches for a value, or set of values in an array column types (such as a multi-select), if an array of search options is provided then it must match all. + * @description Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all. * @example [object Object] */ contains?: { [key: string]: unknown }; /** - * @description As with the contains search, only functions for array column types, but searches for columns missing the supplied values. + * @description The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response. * @example [object Object] */ notContains?: { [key: string]: unknown }; /** - * @description As with the contains search, only functions for array column types, but searches for any of the provided values when given an array. + * @description As with the contains search, only works on array column types and searches for any of the provided values when given an array. * @example [object Object] */ containsAny?: { [key: string]: unknown }; From 20da8bb816762a93d995ad30011708b53979a7d7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 10 Apr 2024 17:36:57 +0100 Subject: [PATCH 03/24] Adding support for SQS prepared statement API. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 11 +++++++++-- packages/backend-core/src/db/instrumentation.ts | 8 ++++++-- packages/server/src/sdk/app/rows/search/sqs.ts | 7 ++++--- packages/types/src/sdk/db.ts | 6 +++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c1347f4f2b..d220d0a8ac 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -12,6 +12,7 @@ import { isDocument, RowResponse, RowValue, + SqlQueryBinding, } from "@budibase/types" import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" @@ -248,14 +249,20 @@ export class DatabaseImpl implements Database { }) } - async sql(sql: string): Promise { + async sql( + sql: string, + parameters?: SqlQueryBinding + ): Promise { const dbName = this.name const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}` const response = await directCouchUrlCall({ url: `${this.couchInfo.sqlUrl}/${url}`, method: "POST", cookie: this.couchInfo.cookie, - body: sql, + body: { + query: sql, + args: parameters, + }, }) if (response.status > 300) { throw new Error(await response.text()) diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index 880f0a3c72..32ba81ebd8 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -13,6 +13,7 @@ import { DatabaseQueryOpts, Document, RowValue, + SqlQueryBinding, } from "@budibase/types" import tracer from "dd-trace" import { Writable } from "stream" @@ -150,10 +151,13 @@ export class DDInstrumentedDatabase implements Database { }) } - sql(sql: string): Promise { + sql( + sql: string, + parameters?: SqlQueryBinding + ): Promise { return tracer.trace("db.sql", span => { span?.addTags({ db_name: this.name }) - return this.db.sql(sql) + return this.db.sql(sql, parameters) }) } } diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 89dae7628f..f270353821 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -11,6 +11,7 @@ import { SortOrder, SortType, Table, + SqlQuery, } from "@budibase/types" import SqlQueryBuilder from "../../../../integrations/base/sql" import { SqlClient } from "../../../../integrations/utils" @@ -156,21 +157,21 @@ export async function search( try { const query = builder._query(request, { disableReturning: true, - disableBindings: true, }) if (Array.isArray(query)) { throw new Error("SQS cannot currently handle multiple queries") } - let sql = query.sql + let sql = query.sql, + bindings = query.bindings // quick hack for docIds sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") const db = context.getAppDB() - const rows = await db.sql(sql) + const rows = await db.sql(sql, bindings) return { rows: await sqlOutputProcessing( diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 692ddcf737..c723f5f8d6 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -4,6 +4,7 @@ import { AnyDocument, Document, RowValue, + SqlQueryBinding, ViewTemplateOpts, } from "../" import { Writable } from "stream" @@ -143,7 +144,10 @@ export interface Database { opts?: DatabasePutOpts ): Promise bulkDocs(documents: AnyDocument[]): Promise - sql(sql: string): Promise + sql( + sql: string, + parameters?: SqlQueryBinding + ): Promise allDocs( params: DatabaseQueryOpts ): Promise> From 2e3e512433e9ae9a899ed2c8bbc3c1bfe81115bf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 10 Apr 2024 17:40:12 +0100 Subject: [PATCH 04/24] Linting. --- packages/server/src/sdk/app/rows/search/sqs.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index f270353821..5b0b6e3bc7 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -11,7 +11,6 @@ import { SortOrder, SortType, Table, - SqlQuery, } from "@budibase/types" import SqlQueryBuilder from "../../../../integrations/base/sql" import { SqlClient } from "../../../../integrations/utils" From a1164ac581961c9c6e7810953087f0576d57a510 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 10 Apr 2024 17:50:18 +0100 Subject: [PATCH 05/24] Working towards getting date tests working for SQS. --- .../src/api/routes/tests/search.spec.ts | 131 ++++++++---------- .../server/src/sdk/app/rows/search/sqs.ts | 2 +- packages/types/src/sdk/datasources.ts | 1 + 3 files changed, 63 insertions(+), 71 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f5d107d0de..5215595c8f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -13,12 +13,12 @@ import { jest.unmock("mssql") describe.each([ - ["internal", undefined], + // ["internal", undefined], ["internal-sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" const config = setup.getConfig() @@ -83,9 +83,6 @@ describe.each([ { query: { equal: { name: "foo" } }, expected: [rows[0]] }, { query: { notEqual: { name: "foo" } }, expected: [rows[1]] }, { query: { oneOf: { name: ["foo"] } }, expected: [rows[0]] }, - // { query: { contains: { name: "f" } }, expected: [0] }, - // { query: { notContains: { name: ["f"] } }, expected: [1] }, - // { query: { containsAny: { name: ["f"] } }, expected: [0] }, ] it.each(stringSearchTests)( @@ -171,7 +168,7 @@ describe.each([ ) }) - describe("dates", () => { + describe.only("dates", () => { beforeEach(async () => { table = await config.api.table.save( tableForDatasource(datasource, { @@ -186,8 +183,8 @@ describe.each([ }) const rows = [ - { dob: new Date("2020-01-01") }, - { dob: new Date("2020-01-10") }, + { dob: new Date("2020-01-01").toISOString() }, + { dob: new Date("2020-01-10").toISOString() }, ] interface DateSearchTest { @@ -196,70 +193,66 @@ describe.each([ } const dateSearchTests: DateSearchTest[] = [ - { query: {}, expected: rows }, + //{ query: {}, expected: rows }, + //{ + // query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, + // expected: rows, + //}, + //{ + // query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, + // expected: [], + //}, { - query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, - expected: rows, - }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, - expected: [], - }, - { - query: { equal: { dob: new Date("2020-01-01") } }, + query: { equal: { dob: new Date("2020-01-01").toISOString() } }, expected: [rows[0]], }, - { query: { equal: { dob: new Date("2020-01-02") } }, expected: [] }, - { - query: { notEqual: { dob: new Date("2020-01-01") } }, - expected: [rows[1]], - }, - { - query: { oneOf: { dob: [new Date("2020-01-01")] } }, - expected: [rows[0]], - }, - { - query: { - range: { - dob: { - low: new Date("2020-01-01").toISOString(), - high: new Date("2020-01-05").toISOString(), - }, - }, - }, - expected: [rows[0]], - }, - { - query: { - range: { - dob: { - low: new Date("2020-01-01").toISOString(), - high: new Date("2020-01-10").toISOString(), - }, - }, - }, - expected: rows, - }, - { - query: { - range: { - dob: { - low: new Date("2020-01-05").toISOString(), - high: new Date("2020-01-10").toISOString(), - }, - }, - }, - expected: [rows[1]], - }, + // { query: { equal: { dob: new Date("2020-01-02") } }, expected: [] }, + // { + // query: { notEqual: { dob: new Date("2020-01-01") } }, + // expected: [rows[1]], + // }, + // { + // query: { oneOf: { dob: [new Date("2020-01-01")] } }, + // expected: [rows[0]], + // }, + // { + // query: { + // range: { + // dob: { + // low: new Date("2020-01-01").toISOString(), + // high: new Date("2020-01-05").toISOString(), + // }, + // }, + // }, + // expected: [rows[0]], + // }, + // { + // query: { + // range: { + // dob: { + // low: new Date("2020-01-01").toISOString(), + // high: new Date("2020-01-10").toISOString(), + // }, + // }, + // }, + // expected: rows, + // }, + // { + // query: { + // range: { + // dob: { + // low: new Date("2020-01-05").toISOString(), + // high: new Date("2020-01-10").toISOString(), + // }, + // }, + // }, + // expected: [rows[1]], + // }, ] it.each(dateSearchTests)( `should be able to run query: $query`, async ({ query, expected }) => { - // TODO(samwho): most of these work for SQS, but not all. Fix 'em. - if (isSqs) { - return - } const savedRows = await Promise.all( rows.map(r => config.api.row.save(table._id!, r)) ) @@ -270,9 +263,7 @@ describe.each([ expect(foundRows).toEqual( expect.arrayContaining( expected.map(r => - expect.objectContaining( - savedRows.find(sr => sr.dob === r.dob.toISOString())! - ) + expect.objectContaining(savedRows.find(sr => sr.dob === r.dob)!) ) ) ) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 89dae7628f..565286ec1e 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -156,7 +156,7 @@ export async function search( try { const query = builder._query(request, { disableReturning: true, - disableBindings: true, + disableBindings: false, }) if (Array.isArray(query)) { diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index e1a012d81e..fff6b75dcc 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -74,6 +74,7 @@ export enum FilterType { EMPTY = "empty", NOT_EMPTY = "notEmpty", ONE_OF = "oneOf", + CONTAINS = "contains", } export enum DatasourceFeature { From ed8f0960e04165cf25ee96e378b3ca147dd79996 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 10 Apr 2024 17:54:45 +0100 Subject: [PATCH 06/24] All search tests for dates working across all datasources. --- .../src/api/routes/tests/search.spec.ts | 117 +++++++++--------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5215595c8f..96c3855f00 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -13,12 +13,12 @@ import { jest.unmock("mssql") describe.each([ - // ["internal", undefined], + ["internal", undefined], ["internal-sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" const config = setup.getConfig() @@ -168,7 +168,7 @@ describe.each([ ) }) - describe.only("dates", () => { + describe("dates", () => { beforeEach(async () => { table = await config.api.table.save( tableForDatasource(datasource, { @@ -193,61 +193,64 @@ describe.each([ } const dateSearchTests: DateSearchTest[] = [ - //{ query: {}, expected: rows }, - //{ - // query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, - // expected: rows, - //}, - //{ - // query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, - // expected: [], - //}, + { query: {}, expected: rows }, + { + query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, + expected: rows, + }, + { + query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, + expected: [], + }, { query: { equal: { dob: new Date("2020-01-01").toISOString() } }, expected: [rows[0]], }, - // { query: { equal: { dob: new Date("2020-01-02") } }, expected: [] }, - // { - // query: { notEqual: { dob: new Date("2020-01-01") } }, - // expected: [rows[1]], - // }, - // { - // query: { oneOf: { dob: [new Date("2020-01-01")] } }, - // expected: [rows[0]], - // }, - // { - // query: { - // range: { - // dob: { - // low: new Date("2020-01-01").toISOString(), - // high: new Date("2020-01-05").toISOString(), - // }, - // }, - // }, - // expected: [rows[0]], - // }, - // { - // query: { - // range: { - // dob: { - // low: new Date("2020-01-01").toISOString(), - // high: new Date("2020-01-10").toISOString(), - // }, - // }, - // }, - // expected: rows, - // }, - // { - // query: { - // range: { - // dob: { - // low: new Date("2020-01-05").toISOString(), - // high: new Date("2020-01-10").toISOString(), - // }, - // }, - // }, - // expected: [rows[1]], - // }, + { + query: { equal: { dob: new Date("2020-01-02").toISOString() } }, + expected: [], + }, + { + query: { notEqual: { dob: new Date("2020-01-01").toISOString() } }, + expected: [rows[1]], + }, + { + query: { oneOf: { dob: [new Date("2020-01-01").toISOString()] } }, + expected: [rows[0]], + }, + { + query: { + range: { + dob: { + low: new Date("2020-01-01").toISOString(), + high: new Date("2020-01-05").toISOString(), + }, + }, + }, + expected: [rows[0]], + }, + { + query: { + range: { + dob: { + low: new Date("2020-01-01").toISOString(), + high: new Date("2020-01-10").toISOString(), + }, + }, + }, + expected: rows, + }, + { + query: { + range: { + dob: { + low: new Date("2020-01-05").toISOString(), + high: new Date("2020-01-10").toISOString(), + }, + }, + }, + expected: [rows[1]], + }, ] it.each(dateSearchTests)( From 432b11a7f640570a03a8e0dc3878939714381a45 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 10 Apr 2024 17:56:57 +0100 Subject: [PATCH 07/24] Revert unneeded change to types. --- packages/types/src/sdk/datasources.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index fff6b75dcc..e1a012d81e 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -74,7 +74,6 @@ export enum FilterType { EMPTY = "empty", NOT_EMPTY = "notEmpty", ONE_OF = "oneOf", - CONTAINS = "contains", } export enum DatasourceFeature { From e6c3fd2951d41ba304f67aba062b74568c5c3250 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 10 Apr 2024 18:01:25 +0100 Subject: [PATCH 08/24] Make pro submodule match master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f8e8f87bd5..ef186d0024 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f8e8f87bd52081e1303a5ae92c432ea5b38f3bb4 +Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a From 6613dfad4436def0b59c61603894363ca70f7fa2 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 11 Apr 2024 08:40:50 +0100 Subject: [PATCH 09/24] Add us.i.posthog.com to CSP (#13453) --- hosting/proxy/nginx.prod.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 65cc3ff390..5b31d86fe3 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -55,7 +55,7 @@ http { set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; + set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; set $csp_frame "frame-src 'self' https:"; set $csp_img "img-src http: https: data: blob:"; From a24f5964aec756d7ff08838e280e511c22109461 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 11 Apr 2024 07:55:02 +0000 Subject: [PATCH 10/24] Bump version to 2.23.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index e5a2623766..da049a0e61 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.0", + "version": "2.23.1", "npmClient": "yarn", "packages": [ "packages/*", From 229bbc0d103f949f9e76522b88d4fc60a8fd05d5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 09:53:54 +0100 Subject: [PATCH 11/24] Assert length on search tests, fixes bug in SQS around on empty return none. --- .../src/api/routes/tests/search.spec.ts | 59 ++++++++++--------- packages/server/src/integrations/base/sql.ts | 26 ++++++++ 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 96c3855f00..3fabbfbef9 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -6,6 +6,7 @@ import { Datasource, EmptyFilterOption, FieldType, + Row, SearchFilters, Table, } from "@budibase/types" @@ -47,7 +48,7 @@ describe.each([ }) describe("strings", () => { - beforeEach(async () => { + beforeAll(async () => { table = await config.api.table.save( tableForDatasource(datasource, { schema: { @@ -61,6 +62,13 @@ describe.each([ }) const rows = [{ name: "foo" }, { name: "bar" }] + let savedRows: Row[] + + beforeAll(async () => { + savedRows = await Promise.all( + rows.map(r => config.api.row.save(table._id!, r)) + ) + }) interface StringSearchTest { query: SearchFilters @@ -68,6 +76,8 @@ describe.each([ } const stringSearchTests: StringSearchTest[] = [ + // These three test cases are generic and don't really need + // to be repeated for all data types, so we just do them here. { query: {}, expected: rows }, { query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, @@ -77,6 +87,7 @@ describe.each([ query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, expected: [], }, + // The rest of these tests are specific to strings. { query: { string: { name: "foo" } }, expected: [rows[0]] }, { query: { string: { name: "none" } }, expected: [] }, { query: { fuzzy: { name: "oo" } }, expected: [rows[0]] }, @@ -88,13 +99,11 @@ describe.each([ it.each(stringSearchTests)( `should be able to run query: $query`, async ({ query, expected }) => { - const savedRows = await Promise.all( - rows.map(r => config.api.row.save(table._id!, r)) - ) const { rows: foundRows } = await config.api.row.search(table._id!, { tableId: table._id!, query, }) + expect(foundRows).toHaveLength(expected.length) expect(foundRows).toEqual( expect.arrayContaining( expected.map(r => @@ -107,7 +116,7 @@ describe.each([ }) describe("number", () => { - beforeEach(async () => { + beforeAll(async () => { table = await config.api.table.save( tableForDatasource(datasource, { schema: { @@ -121,6 +130,13 @@ describe.each([ }) const rows = [{ age: 1 }, { age: 10 }] + let savedRows: Row[] + + beforeAll(async () => { + savedRows = await Promise.all( + rows.map(r => config.api.row.save(table._id!, r)) + ) + }) interface NumberSearchTest { query: SearchFilters @@ -128,15 +144,6 @@ describe.each([ } const numberSearchTests: NumberSearchTest[] = [ - { query: {}, expected: rows }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, - expected: rows, - }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, - expected: [], - }, { query: { equal: { age: 1 } }, expected: [rows[0]] }, { query: { equal: { age: 2 } }, expected: [] }, { query: { notEqual: { age: 1 } }, expected: [rows[1]] }, @@ -150,13 +157,11 @@ describe.each([ it.each(numberSearchTests)( `should be able to run query: $query`, async ({ query, expected }) => { - const savedRows = await Promise.all( - rows.map(r => config.api.row.save(table._id!, r)) - ) const { rows: foundRows } = await config.api.row.search(table._id!, { tableId: table._id!, query, }) + expect(foundRows).toHaveLength(expected.length) expect(foundRows).toEqual( expect.arrayContaining( expected.map(r => @@ -186,6 +191,13 @@ describe.each([ { dob: new Date("2020-01-01").toISOString() }, { dob: new Date("2020-01-10").toISOString() }, ] + let savedRows: Row[] + + beforeEach(async () => { + savedRows = await Promise.all( + rows.map(r => config.api.row.save(table._id!, r)) + ) + }) interface DateSearchTest { query: SearchFilters @@ -193,15 +205,6 @@ describe.each([ } const dateSearchTests: DateSearchTest[] = [ - { query: {}, expected: rows }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, - expected: rows, - }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, - expected: [], - }, { query: { equal: { dob: new Date("2020-01-01").toISOString() } }, expected: [rows[0]], @@ -256,13 +259,11 @@ describe.each([ it.each(dateSearchTests)( `should be able to run query: $query`, async ({ query, expected }) => { - const savedRows = await Promise.all( - rows.map(r => config.api.row.save(table._id!, r)) - ) const { rows: foundRows } = await config.api.row.search(table._id!, { tableId: table._id!, query, }) + expect(foundRows).toHaveLength(expected.length) expect(foundRows).toEqual( expect.arrayContaining( expected.map(r => diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index e99e34ab0f..abf12b35b2 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -22,6 +22,7 @@ import { SortDirection, SqlQueryBinding, Table, + EmptyFilterOption, } from "@budibase/types" import environment from "../../environment" @@ -243,6 +244,7 @@ class InternalBuilder { return query } filters = parseFilters(filters) + let noFilters = true // if all or specified in filters, then everything is an or const allOr = filters.allOr if (filters.oneOf) { @@ -250,6 +252,7 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" query = query[fnc](key, Array.isArray(array) ? array : [array]) }) + noFilters = false } if (filters.string) { iterate(filters.string, (key, value) => { @@ -265,9 +268,11 @@ class InternalBuilder { ]) } }) + noFilters = false } if (filters.fuzzy) { iterate(filters.fuzzy, like) + noFilters = false } if (filters.range) { iterate(filters.range, (key, value) => { @@ -300,40 +305,61 @@ class InternalBuilder { query = query[fnc](key, "<", value.high) } }) + noFilters = false } if (filters.equal) { iterate(filters.equal, (key, value) => { const fnc = allOr ? "orWhere" : "where" query = query[fnc]({ [key]: value }) }) + + // Somewhere above us in the stack adds `{ type: "row" }` to the `equal` + // key before we get here, so we need to still consider it empty when + // that's the case. + const equalEmpty = + Object.keys(filters.equal).length === 1 && filters.equal.type === "row" + if (!equalEmpty) { + noFilters = false + } } if (filters.notEqual) { iterate(filters.notEqual, (key, value) => { const fnc = allOr ? "orWhereNot" : "whereNot" query = query[fnc]({ [key]: value }) }) + noFilters = false } if (filters.empty) { iterate(filters.empty, key => { const fnc = allOr ? "orWhereNull" : "whereNull" query = query[fnc](key) }) + noFilters = false } if (filters.notEmpty) { iterate(filters.notEmpty, key => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" query = query[fnc](key) }) + noFilters = false } if (filters.contains) { contains(filters.contains) + noFilters = false } if (filters.notContains) { contains(filters.notContains) + noFilters = false } if (filters.containsAny) { contains(filters.containsAny, true) + noFilters = false } + + if (noFilters && filters.onEmptyFilter === EmptyFilterOption.RETURN_NONE) { + query = query.whereRaw("1=0") + } + return query } From 187e7b281a05a852bb1992046454ca58321a5306 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:57:36 +0100 Subject: [PATCH 12/24] Chore/update csp posthog (#13455) * Add us.i.posthog.com to CSP * Allow posthog survey scripts in CSP --- hosting/proxy/nginx.prod.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 5b31d86fe3..79007da311 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -51,7 +51,7 @@ http { proxy_buffering off; set $csp_default "default-src 'self'"; - set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net"; + set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com"; set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; From ad913929af5dff13cc3002f33f928b3fe6f2ab7d Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 11 Apr 2024 08:58:10 +0000 Subject: [PATCH 13/24] Bump version to 2.23.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index da049a0e61..f7ed11cebd 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.1", + "version": "2.23.2", "npmClient": "yarn", "packages": [ "packages/*", From 672025e17602d94dde9bb48e8e13a64f45195965 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 10:11:03 +0100 Subject: [PATCH 14/24] Solve onEmptyFilter in a nicer way. --- .../src/api/controllers/row/external.ts | 13 ---------- packages/server/src/integrations/base/sql.ts | 25 ------------------- packages/server/src/sdk/app/rows/search.ts | 12 +++++++++ 3 files changed, 12 insertions(+), 38 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 8ce9ff0f9d..e0e3cb6c18 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -17,11 +17,9 @@ import { Row, Table, UserCtx, - EmptyFilterOption, } from "@budibase/types" import sdk from "../../../sdk" import * as utils from "./utils" -import { dataFilters } from "@budibase/shared-core" import { inputProcessing, outputProcessing, @@ -33,17 +31,6 @@ export async function handleRequest( tableId: string, opts?: RunConfig ): Promise> { - // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string - if (opts && opts.filters) { - opts.filters = sdk.rows.removeEmptyFilters(opts.filters) - } - if ( - !dataFilters.hasFilters(opts?.filters) && - opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE - ) { - return [] as any - } - return new ExternalRequest(operation, tableId, opts?.datasource).run( opts || {} ) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index abf12b35b2..f5828f9419 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -22,7 +22,6 @@ import { SortDirection, SqlQueryBinding, Table, - EmptyFilterOption, } from "@budibase/types" import environment from "../../environment" @@ -244,7 +243,6 @@ class InternalBuilder { return query } filters = parseFilters(filters) - let noFilters = true // if all or specified in filters, then everything is an or const allOr = filters.allOr if (filters.oneOf) { @@ -252,7 +250,6 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" query = query[fnc](key, Array.isArray(array) ? array : [array]) }) - noFilters = false } if (filters.string) { iterate(filters.string, (key, value) => { @@ -268,11 +265,9 @@ class InternalBuilder { ]) } }) - noFilters = false } if (filters.fuzzy) { iterate(filters.fuzzy, like) - noFilters = false } if (filters.range) { iterate(filters.range, (key, value) => { @@ -305,59 +300,39 @@ class InternalBuilder { query = query[fnc](key, "<", value.high) } }) - noFilters = false } if (filters.equal) { iterate(filters.equal, (key, value) => { const fnc = allOr ? "orWhere" : "where" query = query[fnc]({ [key]: value }) }) - - // Somewhere above us in the stack adds `{ type: "row" }` to the `equal` - // key before we get here, so we need to still consider it empty when - // that's the case. - const equalEmpty = - Object.keys(filters.equal).length === 1 && filters.equal.type === "row" - if (!equalEmpty) { - noFilters = false - } } if (filters.notEqual) { iterate(filters.notEqual, (key, value) => { const fnc = allOr ? "orWhereNot" : "whereNot" query = query[fnc]({ [key]: value }) }) - noFilters = false } if (filters.empty) { iterate(filters.empty, key => { const fnc = allOr ? "orWhereNull" : "whereNull" query = query[fnc](key) }) - noFilters = false } if (filters.notEmpty) { iterate(filters.notEmpty, key => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" query = query[fnc](key) }) - noFilters = false } if (filters.contains) { contains(filters.contains) - noFilters = false } if (filters.notContains) { contains(filters.notContains) - noFilters = false } if (filters.containsAny) { contains(filters.containsAny, true) - noFilters = false - } - - if (noFilters && filters.onEmptyFilter === EmptyFilterOption.RETURN_NONE) { - query = query.whereRaw("1=0") } return query diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 928c0f6780..65fd19e427 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,4 +1,5 @@ import { + EmptyFilterOption, Row, RowSearchParams, SearchFilters, @@ -11,6 +12,7 @@ import { NoEmptyFilterStrings } from "../../../constants" import * as sqs from "./search/sqs" import env from "../../../environment" import { ExportRowsParams, ExportRowsResult } from "./search/types" +import { dataFilters } from "@budibase/shared-core" export { isValidFilter } from "../../../integrations/utils" @@ -60,6 +62,16 @@ export async function search( options: RowSearchParams ): Promise> { const isExternalTable = isExternalTableID(options.tableId) + options.query = removeEmptyFilters(options.query) + if ( + !dataFilters.hasFilters(options.query) && + options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE + ) { + return { + rows: [], + } + } + if (isExternalTable) { return external.search(options) } else if (env.SQS_SEARCH_ENABLE) { From 5a36422b97784808d21d6b5f90c77146c41c49e6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 10:21:30 +0100 Subject: [PATCH 15/24] Fix postgres tests. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 65fd19e427..f681bfeb90 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -62,7 +62,7 @@ export async function search( options: RowSearchParams ): Promise> { const isExternalTable = isExternalTableID(options.tableId) - options.query = removeEmptyFilters(options.query) + options.query = removeEmptyFilters(options.query || {}) if ( !dataFilters.hasFilters(options.query) && options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE From ba171bb5a229c9e089484ef8926c6a04245bd410 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 11:58:21 +0100 Subject: [PATCH 16/24] Reduce duplication in search.spec.ts --- .../src/api/routes/tests/search.spec.ts | 292 +++++++++--------- 1 file changed, 144 insertions(+), 148 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3fabbfbef9..5be65553e4 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -6,11 +6,18 @@ import { Datasource, EmptyFilterOption, FieldType, - Row, - SearchFilters, + RowSearchParams, + SortOrder, Table, } from "@budibase/types" +function leftContainsRight< + A extends Record, + B extends Record +>(left: A, right: B) { + return Object.entries(right).every(([k, v]) => left[k] === v) +} + jest.unmock("mssql") describe.each([ @@ -47,10 +54,79 @@ describe.each([ } }) - describe("strings", () => { - beforeAll(async () => { - table = await config.api.table.save( - tableForDatasource(datasource, { + async function testSearch>( + test: SearchTest, + table: Table + ) { + const expected = test.expectToFind + delete test.expectToFind + const { rows: foundRows } = await config.api.row.search(table._id!, { + ...test, + tableId: table._id!, + }) + if (!expected) { + return + } + expect(foundRows).toHaveLength(expected.length) + expect(foundRows).toEqual( + expect.arrayContaining( + expected.map(expectedRow => + expect.objectContaining( + foundRows.find(foundRow => leftContainsRight(foundRow, expectedRow)) + ) + ) + ) + ) + } + + function searchTests>( + name: string, + opts: { + table: (ds?: Datasource) => Promise + rows: T[] + tests: SearchTest[] + } + ) { + let table: Table + + for (const test of opts.tests) { + test.toString = () => { + const queryStr = JSON.stringify({ + query: test.query, + limit: test.limit, + sort: test.sort, + sortOrder: test.sortOrder, + }) + const expectStr = JSON.stringify(test.expectToFind) + return `should run: ${queryStr} and find ${expectStr}` + } + } + + // eslint-disable-next-line jest/valid-title + describe(name, () => { + beforeAll(async () => { + table = await opts.table(datasource) + }) + + beforeAll(async () => { + await Promise.all( + opts.rows.map(r => config.api.row.save(table._id!, r)) + ) + }) + + it.each(opts.tests)(`%s`, test => testSearch(test, table)) + }) + } + + interface SearchTest> + extends Omit { + expectToFind?: RowType[] + } + + searchTests("strings", { + table: async ds => { + return await config.api.table.save( + tableForDatasource(ds, { schema: { name: { name: "name", @@ -59,66 +135,38 @@ describe.each([ }, }) ) - }) + }, + rows: [{ name: "foo" }, { name: "bar" }], + tests: [ + // These test cases are generic and don't really need to be repeated for + // all data types, so we just do them here. - const rows = [{ name: "foo" }, { name: "bar" }] - let savedRows: Row[] - - beforeAll(async () => { - savedRows = await Promise.all( - rows.map(r => config.api.row.save(table._id!, r)) - ) - }) - - interface StringSearchTest { - query: SearchFilters - expected: (typeof rows)[number][] - } - - const stringSearchTests: StringSearchTest[] = [ - // These three test cases are generic and don't really need - // to be repeated for all data types, so we just do them here. - { query: {}, expected: rows }, + // @ts-expect-error - intentionally not passing a query to make sure the + // API can handle it. + { expectToFind: [{ name: "foo" }, { name: "bar" }] }, + { query: {}, expectToFind: [{ name: "foo" }, { name: "bar" }] }, { query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, - expected: rows, + expectToFind: [{ name: "foo" }, { name: "bar" }], }, { query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, - expected: [], + expectToFind: [], }, // The rest of these tests are specific to strings. - { query: { string: { name: "foo" } }, expected: [rows[0]] }, - { query: { string: { name: "none" } }, expected: [] }, - { query: { fuzzy: { name: "oo" } }, expected: [rows[0]] }, - { query: { equal: { name: "foo" } }, expected: [rows[0]] }, - { query: { notEqual: { name: "foo" } }, expected: [rows[1]] }, - { query: { oneOf: { name: ["foo"] } }, expected: [rows[0]] }, - ] - - it.each(stringSearchTests)( - `should be able to run query: $query`, - async ({ query, expected }) => { - const { rows: foundRows } = await config.api.row.search(table._id!, { - tableId: table._id!, - query, - }) - expect(foundRows).toHaveLength(expected.length) - expect(foundRows).toEqual( - expect.arrayContaining( - expected.map(r => - expect.objectContaining(savedRows.find(sr => sr.name === r.name)!) - ) - ) - ) - } - ) + { query: { string: { name: "foo" } }, expectToFind: [{ name: "foo" }] }, + { query: { string: { name: "none" } }, expectToFind: [] }, + { query: { fuzzy: { name: "oo" } }, expectToFind: [{ name: "foo" }] }, + { query: { equal: { name: "foo" } }, expectToFind: [{ name: "foo" }] }, + { query: { notEqual: { name: "foo" } }, expectToFind: [{ name: "bar" }] }, + { query: { oneOf: { name: ["foo"] } }, expectToFind: [{ name: "foo" }] }, + ], }) - describe("number", () => { - beforeAll(async () => { - table = await config.api.table.save( - tableForDatasource(datasource, { + searchTests("numbers", { + table: async ds => { + return await config.api.table.save( + tableForDatasource(ds, { schema: { age: { name: "age", @@ -127,56 +175,33 @@ describe.each([ }, }) ) - }) - - const rows = [{ age: 1 }, { age: 10 }] - let savedRows: Row[] - - beforeAll(async () => { - savedRows = await Promise.all( - rows.map(r => config.api.row.save(table._id!, r)) - ) - }) - - interface NumberSearchTest { - query: SearchFilters - expected: (typeof rows)[number][] - } - - const numberSearchTests: NumberSearchTest[] = [ - { query: { equal: { age: 1 } }, expected: [rows[0]] }, - { query: { equal: { age: 2 } }, expected: [] }, - { query: { notEqual: { age: 1 } }, expected: [rows[1]] }, - { query: { oneOf: { age: [1] } }, expected: [rows[0]] }, - { query: { range: { age: { low: 1, high: 5 } } }, expected: [rows[0]] }, - { query: { range: { age: { low: 0, high: 1 } } }, expected: [rows[0]] }, - { query: { range: { age: { low: 3, high: 4 } } }, expected: [] }, - { query: { range: { age: { low: 0, high: 11 } } }, expected: rows }, - ] - - it.each(numberSearchTests)( - `should be able to run query: $query`, - async ({ query, expected }) => { - const { rows: foundRows } = await config.api.row.search(table._id!, { - tableId: table._id!, - query, - }) - expect(foundRows).toHaveLength(expected.length) - expect(foundRows).toEqual( - expect.arrayContaining( - expected.map(r => - expect.objectContaining(savedRows.find(sr => sr.age === r.age)!) - ) - ) - ) - } - ) + }, + rows: [{ age: 1 }, { age: 10 }], + tests: [ + { query: { equal: { age: 1 } }, expectToFind: [{ age: 1 }] }, + { query: { equal: { age: 2 } }, expectToFind: [] }, + { query: { notEqual: { age: 1 } }, expectToFind: [{ age: 10 }] }, + { query: { oneOf: { age: [1] } }, expectToFind: [{ age: 1 }] }, + { + query: { range: { age: { low: 1, high: 5 } } }, + expectToFind: [{ age: 1 }], + }, + { + query: { range: { age: { low: 0, high: 1 } } }, + expectToFind: [{ age: 1 }], + }, + { query: { range: { age: { low: 3, high: 4 } } }, expectToFind: [] }, + { + query: { range: { age: { low: 0, high: 11 } } }, + expectToFind: [{ age: 1 }, { age: 10 }], + }, + ], }) - describe("dates", () => { - beforeEach(async () => { - table = await config.api.table.save( - tableForDatasource(datasource, { + searchTests("dates", { + table: async ds => { + return await config.api.table.save( + tableForDatasource(ds, { schema: { dob: { name: "dob", @@ -185,41 +210,27 @@ describe.each([ }, }) ) - }) - - const rows = [ + }, + rows: [ { dob: new Date("2020-01-01").toISOString() }, { dob: new Date("2020-01-10").toISOString() }, - ] - let savedRows: Row[] - - beforeEach(async () => { - savedRows = await Promise.all( - rows.map(r => config.api.row.save(table._id!, r)) - ) - }) - - interface DateSearchTest { - query: SearchFilters - expected: (typeof rows)[number][] - } - - const dateSearchTests: DateSearchTest[] = [ + ], + tests: [ { query: { equal: { dob: new Date("2020-01-01").toISOString() } }, - expected: [rows[0]], + expectToFind: [{ dob: new Date("2020-01-01").toISOString() }], }, { query: { equal: { dob: new Date("2020-01-02").toISOString() } }, - expected: [], + expectToFind: [], }, { query: { notEqual: { dob: new Date("2020-01-01").toISOString() } }, - expected: [rows[1]], + expectToFind: [{ dob: new Date("2020-01-10").toISOString() }], }, { query: { oneOf: { dob: [new Date("2020-01-01").toISOString()] } }, - expected: [rows[0]], + expectToFind: [{ dob: new Date("2020-01-01").toISOString() }], }, { query: { @@ -230,7 +241,7 @@ describe.each([ }, }, }, - expected: [rows[0]], + expectToFind: [{ dob: new Date("2020-01-01").toISOString() }], }, { query: { @@ -241,7 +252,10 @@ describe.each([ }, }, }, - expected: rows, + expectToFind: [ + { dob: new Date("2020-01-01").toISOString() }, + { dob: new Date("2020-01-10").toISOString() }, + ], }, { query: { @@ -252,26 +266,8 @@ describe.each([ }, }, }, - expected: [rows[1]], + expectToFind: [{ dob: new Date("2020-01-10").toISOString() }], }, - ] - - it.each(dateSearchTests)( - `should be able to run query: $query`, - async ({ query, expected }) => { - const { rows: foundRows } = await config.api.row.search(table._id!, { - tableId: table._id!, - query, - }) - expect(foundRows).toHaveLength(expected.length) - expect(foundRows).toEqual( - expect.arrayContaining( - expected.map(r => - expect.objectContaining(savedRows.find(sr => sr.dob === r.dob)!) - ) - ) - ) - } - ) + ], }) }) From 0d564a8b4cd025415b4e0f8ca25c409f81e0d373 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 11:58:49 +0100 Subject: [PATCH 17/24] Remove unused variables. --- packages/server/src/api/routes/tests/search.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5be65553e4..79399a9272 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,7 +7,6 @@ import { EmptyFilterOption, FieldType, RowSearchParams, - SortOrder, Table, } from "@budibase/types" @@ -32,7 +31,6 @@ describe.each([ const config = setup.getConfig() let envCleanup: (() => void) | undefined - let table: Table let datasource: Datasource | undefined beforeAll(async () => { From eb56140ce25e7ad7cb36bbb301c0c8dae03d0e2a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 12:03:34 +0100 Subject: [PATCH 18/24] Convert dates to strings, looks nicer and makes no difference. --- .../src/api/routes/tests/search.spec.ts | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 79399a9272..a99df1a744 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -210,61 +210,61 @@ describe.each([ ) }, rows: [ - { dob: new Date("2020-01-01").toISOString() }, - { dob: new Date("2020-01-10").toISOString() }, + { dob: "2020-01-01T00:00:00.000Z" }, + { dob: "2020-01-10T00:00:00.000Z" }, ], tests: [ { - query: { equal: { dob: new Date("2020-01-01").toISOString() } }, - expectToFind: [{ dob: new Date("2020-01-01").toISOString() }], + query: { equal: { dob: "2020-01-01T00:00:00.000Z" } }, + expectToFind: [{ dob: "2020-01-01T00:00:00.000Z" }], }, { - query: { equal: { dob: new Date("2020-01-02").toISOString() } }, + query: { equal: { dob: "2020-01-02T00:00:00.000Z" } }, expectToFind: [], }, { - query: { notEqual: { dob: new Date("2020-01-01").toISOString() } }, - expectToFind: [{ dob: new Date("2020-01-10").toISOString() }], + query: { notEqual: { dob: "2020-01-01T00:00:00.000Z" } }, + expectToFind: [{ dob: "2020-01-10T00:00:00.000Z" }], }, { - query: { oneOf: { dob: [new Date("2020-01-01").toISOString()] } }, - expectToFind: [{ dob: new Date("2020-01-01").toISOString() }], + query: { oneOf: { dob: ["2020-01-01T00:00:00.000Z"] } }, + expectToFind: [{ dob: "2020-01-01T00:00:00.000Z" }], }, { query: { range: { dob: { - low: new Date("2020-01-01").toISOString(), - high: new Date("2020-01-05").toISOString(), + low: "2020-01-01T00:00:00.000Z", + high: "2020-01-05T00:00:00.000Z", }, }, }, - expectToFind: [{ dob: new Date("2020-01-01").toISOString() }], + expectToFind: [{ dob: "2020-01-01T00:00:00.000Z" }], }, { query: { range: { dob: { - low: new Date("2020-01-01").toISOString(), - high: new Date("2020-01-10").toISOString(), + low: "2020-01-01T00:00:00.000Z", + high: "2020-01-10T00:00:00.000Z", }, }, }, expectToFind: [ - { dob: new Date("2020-01-01").toISOString() }, - { dob: new Date("2020-01-10").toISOString() }, + { dob: "2020-01-01T00:00:00.000Z" }, + { dob: "2020-01-10T00:00:00.000Z" }, ], }, { query: { range: { dob: { - low: new Date("2020-01-05").toISOString(), - high: new Date("2020-01-10").toISOString(), + low: "2020-01-05T00:00:00.000Z", + high: "2020-01-10T00:00:00.000Z", }, }, }, - expectToFind: [{ dob: new Date("2020-01-10").toISOString() }], + expectToFind: [{ dob: "2020-01-10T00:00:00.000Z" }], }, ], }) From 8c32127a6f247fd1258720c59d3eb03efda4eb7e Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 11 Apr 2024 11:15:05 +0000 Subject: [PATCH 19/24] Bump version to 2.23.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index f7ed11cebd..385d86209a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.2", + "version": "2.23.3", "npmClient": "yarn", "packages": [ "packages/*", From c07882b4526899912ec31787e496c1c06fc03872 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Apr 2024 15:16:26 +0100 Subject: [PATCH 20/24] Restructure search.spec.ts to be much more readable. --- .../src/api/routes/tests/search.spec.ts | 406 +++++++++--------- 1 file changed, 200 insertions(+), 206 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index a99df1a744..fdf1ed7603 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,15 +7,11 @@ import { EmptyFilterOption, FieldType, RowSearchParams, + SearchFilters, Table, + TableSchema, } from "@budibase/types" - -function leftContainsRight< - A extends Record, - B extends Record ->(left: A, right: B) { - return Object.entries(right).every(([k, v]) => left[k] === v) -} +import _ from "lodash" jest.unmock("mssql") @@ -32,6 +28,7 @@ describe.each([ let envCleanup: (() => void) | undefined let datasource: Datasource | undefined + let table: Table beforeAll(async () => { if (isSqs) { @@ -52,220 +49,217 @@ describe.each([ } }) - async function testSearch>( - test: SearchTest, - table: Table - ) { - const expected = test.expectToFind - delete test.expectToFind - const { rows: foundRows } = await config.api.row.search(table._id!, { - ...test, - tableId: table._id!, - }) - if (!expected) { - return - } - expect(foundRows).toHaveLength(expected.length) - expect(foundRows).toEqual( - expect.arrayContaining( - expected.map(expectedRow => - expect.objectContaining( - foundRows.find(foundRow => leftContainsRight(foundRow, expectedRow)) - ) - ) - ) + async function createTable(schema: TableSchema) { + table = await config.api.table.save( + tableForDatasource(datasource, { schema }) ) } - function searchTests>( - name: string, - opts: { - table: (ds?: Datasource) => Promise
- rows: T[] - tests: SearchTest[] - } - ) { - let table: Table + async function createRows(rows: Record[]) { + await Promise.all(rows.map(r => config.api.row.save(table._id!, r))) + } - for (const test of opts.tests) { - test.toString = () => { - const queryStr = JSON.stringify({ - query: test.query, - limit: test.limit, - sort: test.sort, - sortOrder: test.sortOrder, - }) - const expectStr = JSON.stringify(test.expectToFind) - return `should run: ${queryStr} and find ${expectStr}` - } - } + class SearchAssertion { + constructor(private readonly query: RowSearchParams) {} - // eslint-disable-next-line jest/valid-title - describe(name, () => { - beforeAll(async () => { - table = await opts.table(datasource) + async toFind(expectedRows: any[]) { + const { rows: foundRows } = await config.api.row.search(table._id!, { + ...this.query, + tableId: table._id!, }) - beforeAll(async () => { - await Promise.all( - opts.rows.map(r => config.api.row.save(table._id!, r)) + // eslint-disable-next-line jest/no-standalone-expect + expect(foundRows).toHaveLength(expectedRows.length) + // eslint-disable-next-line jest/no-standalone-expect + expect(foundRows).toEqual( + expect.arrayContaining( + expectedRows.map((expectedRow: any) => + expect.objectContaining( + foundRows.find(foundRow => _.isMatch(foundRow, expectedRow)) + ) + ) ) + ) + } + + async toFindNothing() { + await this.toFind([]) + } + } + + function expectSearch(query: Omit) { + return new SearchAssertion({ ...query, tableId: table._id! }) + } + + function expectQuery(query: SearchFilters) { + return expectSearch({ query }) + } + + describe("strings", () => { + beforeAll(async () => { + await createTable({ + name: { name: "name", type: FieldType.STRING }, + }) + await createRows([{ name: "foo" }, { name: "bar" }]) + }) + + describe("misc", () => { + it("should return all if no query is passed", () => + expectSearch({} as RowSearchParams).toFind([ + { name: "foo" }, + { name: "bar" }, + ])) + + it("should return all if empty query is passed", () => + expectQuery({}).toFind([{ name: "foo" }, { name: "bar" }])) + + it("should return all if onEmptyFilter is RETURN_ALL", () => + expectQuery({ + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + }).toFind([{ name: "foo" }, { name: "bar" }])) + + it("should return nothing if onEmptyFilter is RETURN_NONE", () => + expectQuery({ + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }).toFindNothing()) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { name: "foo" } }).toFind([{ name: "foo" }])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { name: "none" } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { name: "foo" } }).toFind([{ name: "bar" }])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { name: "bar" } }).toFind([{ name: "foo" }])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { name: ["foo"] } }).toFind([{ name: "foo" }])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()) + }) + + describe("fuzzy", () => { + it("successfully finds a row", () => + expectQuery({ fuzzy: { name: "oo" } }).toFind([{ name: "foo" }])) + + it("fails to find nonexistent row", () => + expectQuery({ fuzzy: { name: "none" } }).toFindNothing()) + }) + }) + + describe("numbers", () => { + beforeAll(async () => { + await createTable({ + age: { name: "age", type: FieldType.NUMBER }, + }) + await createRows([{ age: 1 }, { age: 10 }]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { age: 1 } }).toFind([{ age: 1 }])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { age: 2 } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { age: 1 } }).toFind([{ age: 10 }])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { age: 10 } }).toFind([{ age: 1 }])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { age: [1] } }).toFind([{ age: 1 }])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { age: [2] } }).toFindNothing()) + }) + + describe("range", () => { + it("successfully finds a row", () => + expectQuery({ + range: { age: { low: 1, high: 5 } }, + }).toFind([{ age: 1 }])) + + it("successfully finds multiple rows", () => + expectQuery({ + range: { age: { low: 1, high: 10 } }, + }).toFind([{ age: 1 }, { age: 10 }])) + + it("successfully finds a row with a high bound", () => + expectQuery({ + range: { age: { low: 5, high: 10 } }, + }).toFind([{ age: 10 }])) + }) + }) + + describe("dates", () => { + const JAN_1ST = "2020-01-01T00:00:00.000Z" + const JAN_2ND = "2020-01-02T00:00:00.000Z" + const JAN_5TH = "2020-01-05T00:00:00.000Z" + const JAN_10TH = "2020-01-10T00:00:00.000Z" + + beforeAll(async () => { + await createTable({ + dob: { name: "dob", type: FieldType.DATETIME }, }) - it.each(opts.tests)(`%s`, test => testSearch(test, table)) + await createRows([{ dob: JAN_1ST }, { dob: JAN_10TH }]) }) - } - interface SearchTest> - extends Omit { - expectToFind?: RowType[] - } + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { dob: JAN_1ST } }).toFind([{ dob: JAN_1ST }])) - searchTests("strings", { - table: async ds => { - return await config.api.table.save( - tableForDatasource(ds, { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, - }) - ) - }, - rows: [{ name: "foo" }, { name: "bar" }], - tests: [ - // These test cases are generic and don't really need to be repeated for - // all data types, so we just do them here. + it("fails to find nonexistent row", () => + expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing()) + }) - // @ts-expect-error - intentionally not passing a query to make sure the - // API can handle it. - { expectToFind: [{ name: "foo" }, { name: "bar" }] }, - { query: {}, expectToFind: [{ name: "foo" }, { name: "bar" }] }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL }, - expectToFind: [{ name: "foo" }, { name: "bar" }], - }, - { - query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE }, - expectToFind: [], - }, - // The rest of these tests are specific to strings. - { query: { string: { name: "foo" } }, expectToFind: [{ name: "foo" }] }, - { query: { string: { name: "none" } }, expectToFind: [] }, - { query: { fuzzy: { name: "oo" } }, expectToFind: [{ name: "foo" }] }, - { query: { equal: { name: "foo" } }, expectToFind: [{ name: "foo" }] }, - { query: { notEqual: { name: "foo" } }, expectToFind: [{ name: "bar" }] }, - { query: { oneOf: { name: ["foo"] } }, expectToFind: [{ name: "foo" }] }, - ], - }) + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { dob: JAN_1ST } }).toFind([{ dob: JAN_10TH }])) - searchTests("numbers", { - table: async ds => { - return await config.api.table.save( - tableForDatasource(ds, { - schema: { - age: { - name: "age", - type: FieldType.NUMBER, - }, - }, - }) - ) - }, - rows: [{ age: 1 }, { age: 10 }], - tests: [ - { query: { equal: { age: 1 } }, expectToFind: [{ age: 1 }] }, - { query: { equal: { age: 2 } }, expectToFind: [] }, - { query: { notEqual: { age: 1 } }, expectToFind: [{ age: 10 }] }, - { query: { oneOf: { age: [1] } }, expectToFind: [{ age: 1 }] }, - { - query: { range: { age: { low: 1, high: 5 } } }, - expectToFind: [{ age: 1 }], - }, - { - query: { range: { age: { low: 0, high: 1 } } }, - expectToFind: [{ age: 1 }], - }, - { query: { range: { age: { low: 3, high: 4 } } }, expectToFind: [] }, - { - query: { range: { age: { low: 0, high: 11 } } }, - expectToFind: [{ age: 1 }, { age: 10 }], - }, - ], - }) + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { dob: JAN_10TH } }).toFind([{ dob: JAN_1ST }])) + }) - searchTests("dates", { - table: async ds => { - return await config.api.table.save( - tableForDatasource(ds, { - schema: { - dob: { - name: "dob", - type: FieldType.DATETIME, - }, - }, - }) - ) - }, - rows: [ - { dob: "2020-01-01T00:00:00.000Z" }, - { dob: "2020-01-10T00:00:00.000Z" }, - ], - tests: [ - { - query: { equal: { dob: "2020-01-01T00:00:00.000Z" } }, - expectToFind: [{ dob: "2020-01-01T00:00:00.000Z" }], - }, - { - query: { equal: { dob: "2020-01-02T00:00:00.000Z" } }, - expectToFind: [], - }, - { - query: { notEqual: { dob: "2020-01-01T00:00:00.000Z" } }, - expectToFind: [{ dob: "2020-01-10T00:00:00.000Z" }], - }, - { - query: { oneOf: { dob: ["2020-01-01T00:00:00.000Z"] } }, - expectToFind: [{ dob: "2020-01-01T00:00:00.000Z" }], - }, - { - query: { - range: { - dob: { - low: "2020-01-01T00:00:00.000Z", - high: "2020-01-05T00:00:00.000Z", - }, - }, - }, - expectToFind: [{ dob: "2020-01-01T00:00:00.000Z" }], - }, - { - query: { - range: { - dob: { - low: "2020-01-01T00:00:00.000Z", - high: "2020-01-10T00:00:00.000Z", - }, - }, - }, - expectToFind: [ - { dob: "2020-01-01T00:00:00.000Z" }, - { dob: "2020-01-10T00:00:00.000Z" }, - ], - }, - { - query: { - range: { - dob: { - low: "2020-01-05T00:00:00.000Z", - high: "2020-01-10T00:00:00.000Z", - }, - }, - }, - expectToFind: [{ dob: "2020-01-10T00:00:00.000Z" }], - }, - ], + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { dob: [JAN_1ST] } }).toFind([{ dob: JAN_1ST }])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing()) + }) + + describe("range", () => { + it("successfully finds a row", () => + expectQuery({ + range: { dob: { low: JAN_1ST, high: JAN_5TH } }, + }).toFind([{ dob: JAN_1ST }])) + + it("successfully finds multiple rows", () => + expectQuery({ + range: { dob: { low: JAN_1ST, high: JAN_10TH } }, + }).toFind([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + + it("successfully finds a row with a high bound", () => + expectQuery({ + range: { dob: { low: JAN_5TH, high: JAN_10TH } }, + }).toFind([{ dob: JAN_10TH }])) + }) }) }) From a044ba226a76f3a5960f4cae580967b4e01861d8 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:25:36 +0100 Subject: [PATCH 21/24] ignore key action if posthog survey is focused (#13466) --- .../_components/ComponentList/ComponentKeyHandler.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte index 7e9c113a77..6b27d79c15 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte @@ -137,8 +137,12 @@ const activeTag = document.activeElement?.tagName.toLowerCase() const inCodeEditor = document.activeElement?.classList?.contains("cm-content") + const inPosthogSurvey = + document.activeElement?.classList?.[0]?.startsWith("PostHogSurvey") if ( - (inCodeEditor || ["input", "textarea"].indexOf(activeTag) !== -1) && + (inCodeEditor || + inPosthogSurvey || + ["input", "textarea"].indexOf(activeTag) !== -1) && e.key !== "Escape" ) { return From 13bb099d40f2506951fdf03163fcfd62680f07c8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 11 Apr 2024 14:28:26 +0000 Subject: [PATCH 22/24] Bump version to 2.23.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 385d86209a..78a3aa13e9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.3", + "version": "2.23.4", "npmClient": "yarn", "packages": [ "packages/*", From f9e1f4b8c4a5d63d2bbea184b29c69ad5279575c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Apr 2024 15:58:22 +0100 Subject: [PATCH 23/24] Fixing issue with OSS build, if the user is OSS don't attempt to account portal build. --- .github/workflows/budibase_ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 6120290d0d..536992d655 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -64,10 +64,11 @@ jobs: - run: yarn --frozen-lockfile # Run build all the projects - - name: Build - run: | - yarn build:oss - yarn build:account-portal + - name: Build OSS + run: yarn build:oss + - name: Build account portal + run: yarn build:account-portal + if: inputs.run_as_oss != true # Check the types of the projects built via esbuild - name: Check types run: | From b8685dc24558deffa253080b4cc9dc32835c86ef Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 11 Apr 2024 16:24:11 +0100 Subject: [PATCH 24/24] Updating if statement. --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 536992d655..fd4d8cf7c8 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -68,7 +68,7 @@ jobs: run: yarn build:oss - name: Build account portal run: yarn build:account-portal - if: inputs.run_as_oss != true + if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} # Check the types of the projects built via esbuild - name: Check types run: |