From b6bcf6719fe866f3f8de26932352349543cd9dd9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:27:49 +0100 Subject: [PATCH 01/17] Fixes an issue with fetch information being passed up from DatabaseImpl, making sure errors are fully sanitised. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d220d0a8ac..d54e23217b 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -3,11 +3,11 @@ import { AllDocsResponse, AnyDocument, Database, - DatabaseOpts, - DatabaseQueryOpts, - DatabasePutOpts, DatabaseCreateIndexOpts, DatabaseDeleteIndexOpts, + DatabaseOpts, + DatabasePutOpts, + DatabaseQueryOpts, Document, isDocument, RowResponse, @@ -17,7 +17,7 @@ import { import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" -import { WriteStream, ReadStream } from "fs" +import { ReadStream, WriteStream } from "fs" import { newid } from "../../docIds/newid" import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" @@ -38,6 +38,34 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise +class CouchDBError extends Error { + status: number + statusCode: number + reason: string + name: string + errid: string | undefined + description: string | undefined + + constructor( + message: string, + info: { + status: number + name: string + errid: string + description: string + reason: string + } + ) { + super(message) + this.status = info.status + this.statusCode = info.status + this.reason = info.reason + this.name = info.name + this.errid = info.errid + this.description = info.description + } +} + export function DatabaseWithConnection( dbName: string, connection: string, @@ -119,7 +147,7 @@ export class DatabaseImpl implements Database { } catch (err: any) { // Handling race conditions if (err.statusCode !== 412) { - throw err + throw new CouchDBError(err.message, err) } } } @@ -138,10 +166,15 @@ export class DatabaseImpl implements Database { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() return await this.performCall(call) - } else if (err.statusCode) { - err.status = err.statusCode } - throw err + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, { + status: err.status || err.statusCode, + name: err.name, + errid: err.errid, + description: err.description, + reason: err.reason, + }) } } @@ -281,16 +314,9 @@ export class DatabaseImpl implements Database { } async destroy() { - try { - return await this.nano().db.destroy(this.name) - } catch (err: any) { - // didn't exist, don't worry - if (err.statusCode === 404) { - return - } else { - throw { ...err, status: err.statusCode } - } - } + return this.performCall(async () => { + return () => this.nano().db.destroy(this.name) + }) } async compact() { From f036776a907d12204163a7ce3764afa64465013d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:32:57 +0100 Subject: [PATCH 02/17] One small change to keep 404 functionality on destroy DB. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d54e23217b..ca8a22b54e 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -314,9 +314,16 @@ export class DatabaseImpl implements Database { } async destroy() { - return this.performCall(async () => { - return () => this.nano().db.destroy(this.name) - }) + try { + return await this.nano().db.destroy(this.name) + } catch (err: any) { + // didn't exist, don't worry + if (err.statusCode === 404) { + return + } else { + throw new CouchDBError(err.message, err) + } + } } async compact() { From c9ec06b5b1d4e790c29542a9d708aabb1709237d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:51:57 +0100 Subject: [PATCH 03/17] Adding error field. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ca8a22b54e..c520f4d81f 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -43,8 +43,9 @@ class CouchDBError extends Error { statusCode: number reason: string name: string - errid: string | undefined - description: string | undefined + errid: string + error: string + description: string constructor( message: string, @@ -54,6 +55,7 @@ class CouchDBError extends Error { errid: string description: string reason: string + error: string } ) { super(message) @@ -63,6 +65,7 @@ class CouchDBError extends Error { this.name = info.name this.errid = info.errid this.description = info.description + this.error = info.error } } @@ -168,13 +171,7 @@ export class DatabaseImpl implements Database { return await this.performCall(call) } // stripping the error down the props which are safe/useful, drop everything else - throw new CouchDBError(`CouchDB error: ${err.message}`, { - status: err.status || err.statusCode, - name: err.name, - errid: err.errid, - description: err.description, - reason: err.reason, - }) + throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } From a1a50de61c611d7bcff22cbded4af61d9ce661f5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:59:11 +0100 Subject: [PATCH 04/17] Final final fix. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c520f4d81f..ef351f7d4d 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -50,7 +50,8 @@ class CouchDBError extends Error { constructor( message: string, info: { - status: number + status: number | undefined + statusCode: number | undefined name: string errid: string description: string @@ -59,8 +60,9 @@ class CouchDBError extends Error { } ) { super(message) - this.status = info.status - this.statusCode = info.status + const statusCode = info.status || info.statusCode || 500 + this.status = statusCode + this.statusCode = statusCode this.reason = info.reason this.name = info.name this.errid = info.errid From 1365d190488e816f96f49a0568e74e4f05d861cc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 12:03:24 +0100 Subject: [PATCH 05/17] Updating pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 479879246a..ff397e5454 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 479879246aac5dd3073cc695945c62c41fae5b0e +Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409 From bec7b782775df65e298a483ce5f141e60c54320b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 10 May 2024 11:10:28 +0000 Subject: [PATCH 06/17] Bump version to 2.24.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9c5a6c6bab..7daf0b039b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.2", + "version": "2.24.3", "npmClient": "yarn", "packages": [ "packages/*", From ad57776b7fe780296695a93a1c22e190157edc2e Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 10 May 2024 11:13:00 +0000 Subject: [PATCH 07/17] Bump version to 2.25.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7daf0b039b..16dc73aa30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.3", + "version": "2.25.0", "npmClient": "yarn", "packages": [ "packages/*", From e8b8e6e8b4385cb5e167b1467e1d91ad6768e377 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 10 May 2024 13:18:30 +0100 Subject: [PATCH 08/17] Allow Fancy Input validation to be triggered onBlur (#13658) * Add free_trial to deploy camunda script * Allow for more validation customisation on fancy input --- packages/bbui/src/FancyForm/FancyInput.svelte | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/FancyForm/FancyInput.svelte b/packages/bbui/src/FancyForm/FancyInput.svelte index 0c58b9b045..f665fa5724 100644 --- a/packages/bbui/src/FancyForm/FancyInput.svelte +++ b/packages/bbui/src/FancyForm/FancyInput.svelte @@ -11,6 +11,7 @@ export let error = null export let validate = null export let suffix = null + export let validateOn = "change" const dispatch = createEventDispatcher() @@ -24,7 +25,16 @@ const newValue = e.target.value dispatch("change", newValue) value = newValue - if (validate) { + if (validate && (error || validateOn === "change")) { + error = validate(newValue) + } + } + + const onBlur = e => { + focused = false + const newValue = e.target.value + dispatch("blur", newValue) + if (validate && validateOn === "blur") { error = validate(newValue) } } @@ -61,7 +71,7 @@ type={type || "text"} on:input={onChange} on:focus={() => (focused = true)} - on:blur={() => (focused = false)} + on:blur={onBlur} class:placeholder bind:this={ref} /> From 902613d6007a2b3c49d3cac30af9aa583046365f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 12:00:08 +0100 Subject: [PATCH 09/17] Working towards user relationship tests passing. --- .../src/api/routes/tests/search.spec.ts | 134 +++++++++++++++++- packages/server/src/integrations/base/sql.ts | 2 +- .../server/src/sdk/app/rows/search/sqs.ts | 3 +- 3 files changed, 131 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d036da646e..426f383ad0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,6 +1,6 @@ import { tableForDatasource } from "../../../tests/utilities/structures" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, utils } from "@budibase/backend-core" import * as setup from "./utilities" import { @@ -25,12 +25,12 @@ const serverTime = new Date("2024-05-06T00:00:00.000Z") tk.freeze(serverTime) describe.each([ - ["lucene", undefined], + //["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -1152,4 +1152,126 @@ describe.each([ ])) }) }) + + describe("user", () => { + let user1: User + let user2: User + + beforeAll(async () => { + user1 = await config.createUser({ _id: `us_${utils.newid()}` }) + user2 = await config.createUser({ _id: `us_${utils.newid()}` }) + + await createTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + }) + + await createRows([ + { user: JSON.stringify(user1) }, + { user: JSON.stringify(user2) }, + ]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { user: user1._id } }).toContainExactly([ + { user: { _id: user1._id } }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { user: "us_none" } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ + { user: { _id: user2._id } }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ + { user: { _id: user1._id } }, + { user: { _id: user2._id } }, + ])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ + { user: { _id: user1._id } }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) + }) + }) + + describe("multi user", () => { + let user1: User + let user2: User + + beforeAll(async () => { + user1 = await config.createUser({ _id: `us_${utils.newid()}` }) + user2 = await config.createUser({ _id: `us_${utils.newid()}` }) + + await createTable({ + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + }, + }) + + await createRows([ + { users: JSON.stringify([user1]) }, + { users: JSON.stringify([user2]) }, + { users: JSON.stringify([user1, user2]) }, + { users: JSON.stringify([]) }, + ]) + }) + + describe("contains", () => { + it("successfully finds a row", () => + expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()) + }) + + describe("notContains", () => { + it("successfully finds a row", () => + expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ + { users: [{ _id: user2._id }] }, + { users: [] }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user2._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + { users: [] }, + ])) + }) + + describe("containsAny", () => { + it("successfully finds rows", () => + expectQuery({ + containsAny: { users: [user1._id, user2._id] }, + }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user2._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) + }) + }) }) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 7a2b819007..33e276c81b 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -271,7 +271,7 @@ class InternalBuilder { } statement += (statement ? andOr : "") + - `LOWER(${likeKey(this.client, key)}) LIKE ?` + `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` } if (statement === "") { diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 05b1a3bd96..7aaaa6bd6c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -172,7 +172,8 @@ export async function search( sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") const db = context.getAppDB() - return await db.sql(sql, bindings) + const rows = await db.sql(sql, bindings) + return rows }) // process from the format of tableId.column to expected format From 1562e7b1f10ae920ece4c6006a75bb496dde93bc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 12:05:01 +0100 Subject: [PATCH 10/17] Working towards user relationship tests passing. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/integrations/base/sql.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f20e0534e5..f777eb6db1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -406,7 +406,7 @@ describe.each([ ]) }) - it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { + it.only("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { const jsBinding = "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();" const encodedBinding = encodeJSBinding(jsBinding) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 1140a1ac54..1c0c252b1c 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -272,7 +272,8 @@ class InternalBuilder { } statement += (statement ? andOr : "") + - `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` + // `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` + `LOWER(${likeKey(this.client, key)}) LIKE ?` } if (statement === "") { From e2a1ab7eaf40ff7140a0380778fba40b6ca6ce48 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 17:01:52 +0100 Subject: [PATCH 11/17] All tests passing. --- .../src/api/routes/tests/search.spec.ts | 53 +++++++++++++++---- packages/server/src/integrations/base/sql.ts | 50 +++++++++++------ 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3886822c21..0321cdf49e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -22,12 +22,12 @@ import tk from "timekeeper" import { encodeJSBinding } from "@budibase/string-templates" describe.each([ - //["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -1288,6 +1288,7 @@ describe.each([ await createRows([ { user: JSON.stringify(user1) }, { user: JSON.stringify(user2) }, + { user: null }, ]) }) @@ -1305,12 +1306,14 @@ describe.each([ it("successfully finds a row", () => expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ { user: { _id: user2._id } }, + {}, ])) it("fails to find nonexistent row", () => expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, + { user: {} }, ])) }) @@ -1323,6 +1326,19 @@ describe.each([ it("fails to find nonexistent row", () => expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) }) + + describe("empty", () => { + it("finds empty rows", () => + expectQuery({ empty: { user: null } }).toContainExactly([{}])) + }) + + describe("notEmpty", () => { + it("finds non-empty rows", () => + expectQuery({ notEmpty: { user: null } }).toContainExactly([ + { user: { _id: user1._id } }, + { user: { _id: user2._id } }, + ])) + }) }) describe("multi user", () => { @@ -1338,14 +1354,19 @@ describe.each([ name: "users", type: FieldType.BB_REFERENCE, subtype: BBReferenceFieldSubType.USER, + constraints: { type: "array" }, + }, + number: { + name: "number", + type: FieldType.NUMBER, }, }) await createRows([ - { users: JSON.stringify([user1]) }, - { users: JSON.stringify([user2]) }, - { users: JSON.stringify([user1, user2]) }, - { users: JSON.stringify([]) }, + { number: 1, users: JSON.stringify([user1]) }, + { number: 2, users: JSON.stringify([user2]) }, + { number: 3, users: JSON.stringify([user1, user2]) }, + { number: 4, users: JSON.stringify([]) }, ]) }) @@ -1389,5 +1410,19 @@ describe.each([ it("fails to find nonexistent row", () => expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) }) + + describe("multi-column equals", () => { + it("successfully finds a row", () => + expectQuery({ + equal: { number: 1 }, + contains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }])) + + it("fails to find nonexistent row", () => + expectQuery({ + equal: { number: 2 }, + contains: { users: [user1._id] }, + }).toFindNothing()) + }) }) }) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 1c0c252b1c..c3292cf424 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -226,8 +226,7 @@ class InternalBuilder { } const contains = (mode: object, any: boolean = false) => { - const fnc = allOr ? "orWhere" : "where" - const rawFnc = `${fnc}Raw` + const rawFnc = allOr ? "orWhereRaw" : "whereRaw" const not = mode === filters?.notContains ? "NOT " : "" function stringifyArray(value: Array, quoteStyle = '"'): string { for (let i in value) { @@ -240,24 +239,24 @@ class InternalBuilder { if (this.client === SqlClient.POSTGRES) { iterate(mode, (key: string, value: Array) => { const wrap = any ? "" : "'" - const containsOp = any ? "\\?| array" : "@>" + const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) - const tableName = fieldNames[0] - const columnName = fieldNames[1] - // @ts-ignore + const table = fieldNames[0] + const col = fieldNames[1] query = query[rawFnc]( - `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray( + `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray( value, any ? "'" : '"' - )}${wrap}` + )}${wrap}, FALSE)` ) }) } else if (this.client === SqlClient.MY_SQL) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, (key: string, value: Array) => { - // @ts-ignore query = query[rawFnc]( - `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')` + `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( + value + )}'), FALSE)` ) }) } else { @@ -272,8 +271,7 @@ class InternalBuilder { } statement += (statement ? andOr : "") + - // `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` - `LOWER(${likeKey(this.client, key)}) LIKE ?` + `COALESCE(LOWER(${likeKey(this.client, key)}), '') LIKE ?` } if (statement === "") { @@ -338,14 +336,34 @@ class InternalBuilder { } if (filters.equal) { iterate(filters.equal, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc]({ [key]: value }) + const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (this.client === SqlClient.MS_SQL) { + query = query[fnc]( + `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 1`, + [value] + ) + } else { + query = query[fnc]( + `COALESCE(${likeKey(this.client, key)} = ?, FALSE)`, + [value] + ) + } }) } if (filters.notEqual) { iterate(filters.notEqual, (key, value) => { - const fnc = allOr ? "orWhereNot" : "whereNot" - query = query[fnc]({ [key]: value }) + const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (this.client === SqlClient.MS_SQL) { + query = query[fnc]( + `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 0`, + [value] + ) + } else { + query = query[fnc]( + `COALESCE(${likeKey(this.client, key)} != ?, TRUE)`, + [value] + ) + } }) } if (filters.empty) { From 2131cc689cd01eaaaa5ae7eb3d76de5bd1bf0fe5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 17:13:12 +0100 Subject: [PATCH 12/17] Put pro back in line with master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index ff397e5454..d3c3077011 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409 +Subproject commit d3c3077011a8e20ed3c48dcd6301caca4120b6ac From a9f8a72ebd33e341f1d10511a7cc0f0c52428444 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 22 May 2024 18:00:32 +0100 Subject: [PATCH 13/17] Attempting to fix tests. --- .../server/src/api/routes/tests/search.spec.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index e4fe461999..26bf58dbf3 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -87,8 +87,11 @@ describe.each([ class SearchAssertion { constructor(private readonly query: RowSearchParams) {} - private popRow(expectedRow: any, foundRows: any[]) { - const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow)) + private popRow( + expectedRow: T, + foundRows: T[] + ): NonNullable { + const row = foundRows.find(row => _.isMatch(row, expectedRow)) if (!row) { const fields = Object.keys(expectedRow) // To make the error message more readable, we only include the fields @@ -130,6 +133,8 @@ describe.each([ // passed in. The order of the rows is not important, but extra rows will // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { + expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) + const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -151,6 +156,8 @@ describe.each([ // The order of the rows is not important. Extra rows will not cause the // assertion to fail. async toContain(expectedRows: any[]) { + expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) + const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -1055,6 +1062,7 @@ describe.each([ describe("notEqual", () => { it("successfully finds a row", () => expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + { timeid: NULL_TIME__ID }, { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, @@ -1064,6 +1072,7 @@ describe.each([ it("return all when requesting non-existing", () => expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly( [ + { timeid: NULL_TIME__ID }, { time: "10:00:00" }, { time: "10:45:00" }, { time: "12:00:00" }, @@ -1668,7 +1677,7 @@ describe.each([ }) describe("containsAny", () => { - it("successfully finds rows", () => + it.only("successfully finds rows", () => expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ From 120f240f01e0178ddae12cb228b2058b9bb367a4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 10:43:44 +0100 Subject: [PATCH 14/17] Fix tests. --- .../src/api/routes/tests/search.spec.ts | 67 +++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 26bf58dbf3..aac43874a0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -87,24 +87,67 @@ describe.each([ class SearchAssertion { constructor(private readonly query: RowSearchParams) {} - private popRow( + // We originally used _.isMatch to compare rows, but found that when + // comparing arrays it would return true if the source array was a subset of + // the target array. This would sometimes create false matches. This + // function is a more strict version of _.isMatch that only returns true if + // the source array is an exact match of the target. + // + // _.isMatch("100", "1") also returns true which is not what we want. + private isMatch>(expected: T, found: T) { + if (!expected) { + throw new Error("Expected is undefined") + } + if (!found) { + return false + } + + for (const key of Object.keys(expected)) { + if (Array.isArray(expected[key])) { + if (!Array.isArray(found[key])) { + return false + } + if (expected[key].length !== found[key].length) { + return false + } + if (!_.isMatch(found[key], expected[key])) { + return false + } + } else if (typeof expected[key] === "object") { + if (!this.isMatch(expected[key], found[key])) { + return false + } + } else { + if (expected[key] !== found[key]) { + return false + } + } + } + return true + } + + // This function exists to ensure that the same row is not matched twice. + // When a row gets matched, we make sure to remove it from the list of rows + // we're matching against. + private popRow( expectedRow: T, foundRows: T[] ): NonNullable { - const row = foundRows.find(row => _.isMatch(row, expectedRow)) + const row = foundRows.find(row => this.isMatch(expectedRow, row)) if (!row) { const fields = Object.keys(expectedRow) // To make the error message more readable, we only include the fields // that are present in the expected row. const searchedObjects = foundRows.map(row => _.pick(row, fields)) throw new Error( - `Failed to find row: ${JSON.stringify( - expectedRow - )} in ${JSON.stringify(searchedObjects)}` + `Failed to find row:\n\n${JSON.stringify( + expectedRow, + null, + 2 + )}\n\nin\n\n${JSON.stringify(searchedObjects, null, 2)}` ) } - // Ensuring the same row is not matched twice foundRows.splice(foundRows.indexOf(row), 1) return row } @@ -133,8 +176,6 @@ describe.each([ // passed in. The order of the rows is not important, but extra rows will // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { - expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) - const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -156,8 +197,6 @@ describe.each([ // The order of the rows is not important. Extra rows will not cause the // assertion to fail. async toContain(expectedRows: any[]) { - expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) - const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -1592,7 +1631,7 @@ describe.each([ expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, - { user: {} }, + {}, ])) }) @@ -1664,7 +1703,7 @@ describe.each([ it("successfully finds a row", () => expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ { users: [{ _id: user2._id }] }, - { users: [] }, + {}, ])) it("fails to find nonexistent row", () => @@ -1672,12 +1711,12 @@ describe.each([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - { users: [] }, + {}, ])) }) describe("containsAny", () => { - it.only("successfully finds rows", () => + it("successfully finds rows", () => expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ From a6d2f82e7b185656e661d7a7821b3965d00b118f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 15:16:52 +0100 Subject: [PATCH 15/17] Fix tests. --- packages/server/src/api/controllers/view/exporters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index 3b5f951dca..9cf114f4e5 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -57,5 +57,5 @@ export function isFormat(format: any): format is Format { } export function parseCsvExport(value: string) { - return JSON.parse(value?.replace(/'/g, '"')) as T + return JSON.parse(value) as T } From 325819ebae9fea081f2bd46a94952d7b51fab58a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 16:13:07 +0100 Subject: [PATCH 16/17] Fix tests (take 3). --- .../src/integrations/tests/sqlAlias.spec.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 0de4d0a151..f907e1ab8e 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -77,7 +77,7 @@ describe("Captures of real examples", () => { "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 + left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where COALESCE("b"."taskname" = $2, FALSE) order by "a"."productname" asc nulls first limit $3`), }) }) @@ -137,12 +137,12 @@ describe("Captures of real examples", () => { "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname" - from (select * from "tasks" as "a" where not "a"."completed" = $1 + from (select * from "tasks" as "a" where COALESCE("a"."completed" != $1, TRUE) order by "a"."taskname" asc nulls first limit $2) as "a" left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" left join "products" as "b" on "b"."productid" = "d"."productid" left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" - where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`), + where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE) order by "a"."taskname" asc nulls first limit $6`), }) }) }) @@ -154,7 +154,7 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`), + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), }) }) @@ -164,7 +164,7 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`), + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), }) }) }) @@ -175,8 +175,9 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) expect(query).toEqual({ bindings: ["ddd", ""], - sql: multiline(`delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 - returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), + sql: multiline(`delete from "compositetable" as "a" + where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE) + returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), }) }) }) @@ -197,7 +198,7 @@ describe("Captures of real examples", () => { returningQuery = input }, queryJson) expect(returningQuery).toEqual({ - sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]", + sql: "select * from (select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]", bindings: [1, "Test", 22], }) }) From ef60893df1677d58c450f70dcf7532f45ee305be Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 17:02:08 +0100 Subject: [PATCH 17/17] Fix tests (take 4). --- packages/server/src/integrations/tests/sql.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 9b84409e92..302b61fc74 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -189,7 +189,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit], - sql: `select * from (select * from (select * from "test" where (LOWER("test"."age") LIKE :1 AND LOWER("test"."age") LIKE :2) and (LOWER("test"."name") LIKE :3 AND LOWER("test"."name") LIKE :4)) where rownum <= :5) "test"`, + sql: `select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4)) where rownum <= :5) "test"`, }) query = new Sql(SqlClient.ORACLE, limit)._query(