From b6bcf6719fe866f3f8de26932352349543cd9dd9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:27:49 +0100 Subject: [PATCH 01/36] Fixes an issue with fetch information being passed up from DatabaseImpl, making sure errors are fully sanitised. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d220d0a8ac..d54e23217b 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -3,11 +3,11 @@ import { AllDocsResponse, AnyDocument, Database, - DatabaseOpts, - DatabaseQueryOpts, - DatabasePutOpts, DatabaseCreateIndexOpts, DatabaseDeleteIndexOpts, + DatabaseOpts, + DatabasePutOpts, + DatabaseQueryOpts, Document, isDocument, RowResponse, @@ -17,7 +17,7 @@ import { import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" -import { WriteStream, ReadStream } from "fs" +import { ReadStream, WriteStream } from "fs" import { newid } from "../../docIds/newid" import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" @@ -38,6 +38,34 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise +class CouchDBError extends Error { + status: number + statusCode: number + reason: string + name: string + errid: string | undefined + description: string | undefined + + constructor( + message: string, + info: { + status: number + name: string + errid: string + description: string + reason: string + } + ) { + super(message) + this.status = info.status + this.statusCode = info.status + this.reason = info.reason + this.name = info.name + this.errid = info.errid + this.description = info.description + } +} + export function DatabaseWithConnection( dbName: string, connection: string, @@ -119,7 +147,7 @@ export class DatabaseImpl implements Database { } catch (err: any) { // Handling race conditions if (err.statusCode !== 412) { - throw err + throw new CouchDBError(err.message, err) } } } @@ -138,10 +166,15 @@ export class DatabaseImpl implements Database { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() return await this.performCall(call) - } else if (err.statusCode) { - err.status = err.statusCode } - throw err + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, { + status: err.status || err.statusCode, + name: err.name, + errid: err.errid, + description: err.description, + reason: err.reason, + }) } } @@ -281,16 +314,9 @@ export class DatabaseImpl implements Database { } async destroy() { - try { - return await this.nano().db.destroy(this.name) - } catch (err: any) { - // didn't exist, don't worry - if (err.statusCode === 404) { - return - } else { - throw { ...err, status: err.statusCode } - } - } + return this.performCall(async () => { + return () => this.nano().db.destroy(this.name) + }) } async compact() { From f036776a907d12204163a7ce3764afa64465013d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:32:57 +0100 Subject: [PATCH 02/36] One small change to keep 404 functionality on destroy DB. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d54e23217b..ca8a22b54e 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -314,9 +314,16 @@ export class DatabaseImpl implements Database { } async destroy() { - return this.performCall(async () => { - return () => this.nano().db.destroy(this.name) - }) + try { + return await this.nano().db.destroy(this.name) + } catch (err: any) { + // didn't exist, don't worry + if (err.statusCode === 404) { + return + } else { + throw new CouchDBError(err.message, err) + } + } } async compact() { From c9ec06b5b1d4e790c29542a9d708aabb1709237d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:51:57 +0100 Subject: [PATCH 03/36] Adding error field. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ca8a22b54e..c520f4d81f 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -43,8 +43,9 @@ class CouchDBError extends Error { statusCode: number reason: string name: string - errid: string | undefined - description: string | undefined + errid: string + error: string + description: string constructor( message: string, @@ -54,6 +55,7 @@ class CouchDBError extends Error { errid: string description: string reason: string + error: string } ) { super(message) @@ -63,6 +65,7 @@ class CouchDBError extends Error { this.name = info.name this.errid = info.errid this.description = info.description + this.error = info.error } } @@ -168,13 +171,7 @@ export class DatabaseImpl implements Database { return await this.performCall(call) } // stripping the error down the props which are safe/useful, drop everything else - throw new CouchDBError(`CouchDB error: ${err.message}`, { - status: err.status || err.statusCode, - name: err.name, - errid: err.errid, - description: err.description, - reason: err.reason, - }) + throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } From a1a50de61c611d7bcff22cbded4af61d9ce661f5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:59:11 +0100 Subject: [PATCH 04/36] Final final fix. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c520f4d81f..ef351f7d4d 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -50,7 +50,8 @@ class CouchDBError extends Error { constructor( message: string, info: { - status: number + status: number | undefined + statusCode: number | undefined name: string errid: string description: string @@ -59,8 +60,9 @@ class CouchDBError extends Error { } ) { super(message) - this.status = info.status - this.statusCode = info.status + const statusCode = info.status || info.statusCode || 500 + this.status = statusCode + this.statusCode = statusCode this.reason = info.reason this.name = info.name this.errid = info.errid From 1365d190488e816f96f49a0568e74e4f05d861cc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 12:03:24 +0100 Subject: [PATCH 05/36] Updating pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 479879246a..ff397e5454 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 479879246aac5dd3073cc695945c62c41fae5b0e +Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409 From bec7b782775df65e298a483ce5f141e60c54320b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 10 May 2024 11:10:28 +0000 Subject: [PATCH 06/36] Bump version to 2.24.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9c5a6c6bab..7daf0b039b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.2", + "version": "2.24.3", "npmClient": "yarn", "packages": [ "packages/*", From ad57776b7fe780296695a93a1c22e190157edc2e Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 10 May 2024 11:13:00 +0000 Subject: [PATCH 07/36] Bump version to 2.25.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7daf0b039b..16dc73aa30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.3", + "version": "2.25.0", "npmClient": "yarn", "packages": [ "packages/*", From e8b8e6e8b4385cb5e167b1467e1d91ad6768e377 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 10 May 2024 13:18:30 +0100 Subject: [PATCH 08/36] Allow Fancy Input validation to be triggered onBlur (#13658) * Add free_trial to deploy camunda script * Allow for more validation customisation on fancy input --- packages/bbui/src/FancyForm/FancyInput.svelte | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/FancyForm/FancyInput.svelte b/packages/bbui/src/FancyForm/FancyInput.svelte index 0c58b9b045..f665fa5724 100644 --- a/packages/bbui/src/FancyForm/FancyInput.svelte +++ b/packages/bbui/src/FancyForm/FancyInput.svelte @@ -11,6 +11,7 @@ export let error = null export let validate = null export let suffix = null + export let validateOn = "change" const dispatch = createEventDispatcher() @@ -24,7 +25,16 @@ const newValue = e.target.value dispatch("change", newValue) value = newValue - if (validate) { + if (validate && (error || validateOn === "change")) { + error = validate(newValue) + } + } + + const onBlur = e => { + focused = false + const newValue = e.target.value + dispatch("blur", newValue) + if (validate && validateOn === "blur") { error = validate(newValue) } } @@ -61,7 +71,7 @@ type={type || "text"} on:input={onChange} on:focus={() => (focused = true)} - on:blur={() => (focused = false)} + on:blur={onBlur} class:placeholder bind:this={ref} /> From 902613d6007a2b3c49d3cac30af9aa583046365f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 12:00:08 +0100 Subject: [PATCH 09/36] Working towards user relationship tests passing. --- .../src/api/routes/tests/search.spec.ts | 134 +++++++++++++++++- packages/server/src/integrations/base/sql.ts | 2 +- .../server/src/sdk/app/rows/search/sqs.ts | 3 +- 3 files changed, 131 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d036da646e..426f383ad0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1,6 +1,6 @@ import { tableForDatasource } from "../../../tests/utilities/structures" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, utils } from "@budibase/backend-core" import * as setup from "./utilities" import { @@ -25,12 +25,12 @@ const serverTime = new Date("2024-05-06T00:00:00.000Z") tk.freeze(serverTime) describe.each([ - ["lucene", undefined], + //["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -1152,4 +1152,126 @@ describe.each([ ])) }) }) + + describe("user", () => { + let user1: User + let user2: User + + beforeAll(async () => { + user1 = await config.createUser({ _id: `us_${utils.newid()}` }) + user2 = await config.createUser({ _id: `us_${utils.newid()}` }) + + await createTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + }) + + await createRows([ + { user: JSON.stringify(user1) }, + { user: JSON.stringify(user2) }, + ]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { user: user1._id } }).toContainExactly([ + { user: { _id: user1._id } }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { user: "us_none" } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ + { user: { _id: user2._id } }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ + { user: { _id: user1._id } }, + { user: { _id: user2._id } }, + ])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ + { user: { _id: user1._id } }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) + }) + }) + + describe("multi user", () => { + let user1: User + let user2: User + + beforeAll(async () => { + user1 = await config.createUser({ _id: `us_${utils.newid()}` }) + user2 = await config.createUser({ _id: `us_${utils.newid()}` }) + + await createTable({ + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + }, + }) + + await createRows([ + { users: JSON.stringify([user1]) }, + { users: JSON.stringify([user2]) }, + { users: JSON.stringify([user1, user2]) }, + { users: JSON.stringify([]) }, + ]) + }) + + describe("contains", () => { + it("successfully finds a row", () => + expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()) + }) + + describe("notContains", () => { + it("successfully finds a row", () => + expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ + { users: [{ _id: user2._id }] }, + { users: [] }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user2._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + { users: [] }, + ])) + }) + + describe("containsAny", () => { + it("successfully finds rows", () => + expectQuery({ + containsAny: { users: [user1._id, user2._id] }, + }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user2._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) + }) + }) }) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 7a2b819007..33e276c81b 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -271,7 +271,7 @@ class InternalBuilder { } statement += (statement ? andOr : "") + - `LOWER(${likeKey(this.client, key)}) LIKE ?` + `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` } if (statement === "") { diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 05b1a3bd96..7aaaa6bd6c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -172,7 +172,8 @@ export async function search( sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`") const db = context.getAppDB() - return await db.sql(sql, bindings) + const rows = await db.sql(sql, bindings) + return rows }) // process from the format of tableId.column to expected format From 1562e7b1f10ae920ece4c6006a75bb496dde93bc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 12:05:01 +0100 Subject: [PATCH 10/36] Working towards user relationship tests passing. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/integrations/base/sql.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f20e0534e5..f777eb6db1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -406,7 +406,7 @@ describe.each([ ]) }) - it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { + it.only("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { const jsBinding = "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();" const encodedBinding = encodeJSBinding(jsBinding) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 1140a1ac54..1c0c252b1c 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -272,7 +272,8 @@ class InternalBuilder { } statement += (statement ? andOr : "") + - `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` + // `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` + `LOWER(${likeKey(this.client, key)}) LIKE ?` } if (statement === "") { From e2a1ab7eaf40ff7140a0380778fba40b6ca6ce48 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 17:01:52 +0100 Subject: [PATCH 11/36] All tests passing. --- .../src/api/routes/tests/search.spec.ts | 53 +++++++++++++++---- packages/server/src/integrations/base/sql.ts | 50 +++++++++++------ 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3886822c21..0321cdf49e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -22,12 +22,12 @@ import tk from "timekeeper" import { encodeJSBinding } from "@budibase/string-templates" describe.each([ - //["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - //[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - //[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - //[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - //[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -1288,6 +1288,7 @@ describe.each([ await createRows([ { user: JSON.stringify(user1) }, { user: JSON.stringify(user2) }, + { user: null }, ]) }) @@ -1305,12 +1306,14 @@ describe.each([ it("successfully finds a row", () => expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ { user: { _id: user2._id } }, + {}, ])) it("fails to find nonexistent row", () => expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, + { user: {} }, ])) }) @@ -1323,6 +1326,19 @@ describe.each([ it("fails to find nonexistent row", () => expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) }) + + describe("empty", () => { + it("finds empty rows", () => + expectQuery({ empty: { user: null } }).toContainExactly([{}])) + }) + + describe("notEmpty", () => { + it("finds non-empty rows", () => + expectQuery({ notEmpty: { user: null } }).toContainExactly([ + { user: { _id: user1._id } }, + { user: { _id: user2._id } }, + ])) + }) }) describe("multi user", () => { @@ -1338,14 +1354,19 @@ describe.each([ name: "users", type: FieldType.BB_REFERENCE, subtype: BBReferenceFieldSubType.USER, + constraints: { type: "array" }, + }, + number: { + name: "number", + type: FieldType.NUMBER, }, }) await createRows([ - { users: JSON.stringify([user1]) }, - { users: JSON.stringify([user2]) }, - { users: JSON.stringify([user1, user2]) }, - { users: JSON.stringify([]) }, + { number: 1, users: JSON.stringify([user1]) }, + { number: 2, users: JSON.stringify([user2]) }, + { number: 3, users: JSON.stringify([user1, user2]) }, + { number: 4, users: JSON.stringify([]) }, ]) }) @@ -1389,5 +1410,19 @@ describe.each([ it("fails to find nonexistent row", () => expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) }) + + describe("multi-column equals", () => { + it("successfully finds a row", () => + expectQuery({ + equal: { number: 1 }, + contains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }])) + + it("fails to find nonexistent row", () => + expectQuery({ + equal: { number: 2 }, + contains: { users: [user1._id] }, + }).toFindNothing()) + }) }) }) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 1c0c252b1c..c3292cf424 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -226,8 +226,7 @@ class InternalBuilder { } const contains = (mode: object, any: boolean = false) => { - const fnc = allOr ? "orWhere" : "where" - const rawFnc = `${fnc}Raw` + const rawFnc = allOr ? "orWhereRaw" : "whereRaw" const not = mode === filters?.notContains ? "NOT " : "" function stringifyArray(value: Array, quoteStyle = '"'): string { for (let i in value) { @@ -240,24 +239,24 @@ class InternalBuilder { if (this.client === SqlClient.POSTGRES) { iterate(mode, (key: string, value: Array) => { const wrap = any ? "" : "'" - const containsOp = any ? "\\?| array" : "@>" + const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) - const tableName = fieldNames[0] - const columnName = fieldNames[1] - // @ts-ignore + const table = fieldNames[0] + const col = fieldNames[1] query = query[rawFnc]( - `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray( + `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray( value, any ? "'" : '"' - )}${wrap}` + )}${wrap}, FALSE)` ) }) } else if (this.client === SqlClient.MY_SQL) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, (key: string, value: Array) => { - // @ts-ignore query = query[rawFnc]( - `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')` + `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( + value + )}'), FALSE)` ) }) } else { @@ -272,8 +271,7 @@ class InternalBuilder { } statement += (statement ? andOr : "") + - // `COALESCE(LOWER(${likeKey(this.client, key)}) LIKE ?, FALSE)` - `LOWER(${likeKey(this.client, key)}) LIKE ?` + `COALESCE(LOWER(${likeKey(this.client, key)}), '') LIKE ?` } if (statement === "") { @@ -338,14 +336,34 @@ class InternalBuilder { } if (filters.equal) { iterate(filters.equal, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc]({ [key]: value }) + const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (this.client === SqlClient.MS_SQL) { + query = query[fnc]( + `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 1`, + [value] + ) + } else { + query = query[fnc]( + `COALESCE(${likeKey(this.client, key)} = ?, FALSE)`, + [value] + ) + } }) } if (filters.notEqual) { iterate(filters.notEqual, (key, value) => { - const fnc = allOr ? "orWhereNot" : "whereNot" - query = query[fnc]({ [key]: value }) + const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (this.client === SqlClient.MS_SQL) { + query = query[fnc]( + `CASE WHEN ${likeKey(this.client, key)} = ? THEN 1 ELSE 0 END = 0`, + [value] + ) + } else { + query = query[fnc]( + `COALESCE(${likeKey(this.client, key)} != ?, TRUE)`, + [value] + ) + } }) } if (filters.empty) { From 2131cc689cd01eaaaa5ae7eb3d76de5bd1bf0fe5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 May 2024 17:13:12 +0100 Subject: [PATCH 12/36] Put pro back in line with master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index ff397e5454..d3c3077011 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409 +Subproject commit d3c3077011a8e20ed3c48dcd6301caca4120b6ac From a9f8a72ebd33e341f1d10511a7cc0f0c52428444 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 22 May 2024 18:00:32 +0100 Subject: [PATCH 13/36] Attempting to fix tests. --- .../server/src/api/routes/tests/search.spec.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index e4fe461999..26bf58dbf3 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -87,8 +87,11 @@ describe.each([ class SearchAssertion { constructor(private readonly query: RowSearchParams) {} - private popRow(expectedRow: any, foundRows: any[]) { - const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow)) + private popRow( + expectedRow: T, + foundRows: T[] + ): NonNullable { + const row = foundRows.find(row => _.isMatch(row, expectedRow)) if (!row) { const fields = Object.keys(expectedRow) // To make the error message more readable, we only include the fields @@ -130,6 +133,8 @@ describe.each([ // passed in. The order of the rows is not important, but extra rows will // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { + expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) + const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -151,6 +156,8 @@ describe.each([ // The order of the rows is not important. Extra rows will not cause the // assertion to fail. async toContain(expectedRows: any[]) { + expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) + const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -1055,6 +1062,7 @@ describe.each([ describe("notEqual", () => { it("successfully finds a row", () => expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + { timeid: NULL_TIME__ID }, { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, @@ -1064,6 +1072,7 @@ describe.each([ it("return all when requesting non-existing", () => expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly( [ + { timeid: NULL_TIME__ID }, { time: "10:00:00" }, { time: "10:45:00" }, { time: "12:00:00" }, @@ -1668,7 +1677,7 @@ describe.each([ }) describe("containsAny", () => { - it("successfully finds rows", () => + it.only("successfully finds rows", () => expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ From 2cc329994a16ff3010259837782bdcf49aace95c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 May 2024 17:18:39 +0200 Subject: [PATCH 14/36] Don't allow chaning time only to datetime --- .../DataTable/modals/CreateEditColumn.svelte | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e88c28a9d9..26d0357b5f 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -604,27 +604,29 @@ - {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} -
-
- - - - + {#if !editableColumn.timeOnly} + {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} +
+
+ + + + +
+
- -
+ {/if} + {/if} - {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
From b3bea1e839b091efbe4beac0e50faa981c5e1716 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 May 2024 17:18:58 +0200 Subject: [PATCH 15/36] Block timeOnly changes in the api --- packages/server/src/sdk/app/tables/external/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 98e6e561c8..53302085fc 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -73,6 +73,16 @@ function validate(table: Table, oldTable?: Table) { `Column "${key}" has subtype "${column.subtype}" - this is not supported.` ) } + + if (column.type === FieldType.DATETIME) { + const oldColumn = oldTable?.schema[key] as typeof column + + if (column.timeOnly !== oldColumn.timeOnly) { + throw new Error( + `Column "${key}" can not change from time to datetime or viceversa.` + ) + } + } } } From 028afd9cca04808615b672f9979f62af469b13d5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 11:34:58 +0200 Subject: [PATCH 16/36] Fix checks --- packages/server/src/sdk/app/tables/external/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 53302085fc..842b6b5648 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -77,7 +77,7 @@ function validate(table: Table, oldTable?: Table) { if (column.type === FieldType.DATETIME) { const oldColumn = oldTable?.schema[key] as typeof column - if (column.timeOnly !== oldColumn.timeOnly) { + if (oldColumn && column.timeOnly !== oldColumn.timeOnly) { throw new Error( `Column "${key}" can not change from time to datetime or viceversa.` ) From 120f240f01e0178ddae12cb228b2058b9bb367a4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 10:43:44 +0100 Subject: [PATCH 17/36] Fix tests. --- .../src/api/routes/tests/search.spec.ts | 67 +++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 26bf58dbf3..aac43874a0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -87,24 +87,67 @@ describe.each([ class SearchAssertion { constructor(private readonly query: RowSearchParams) {} - private popRow( + // We originally used _.isMatch to compare rows, but found that when + // comparing arrays it would return true if the source array was a subset of + // the target array. This would sometimes create false matches. This + // function is a more strict version of _.isMatch that only returns true if + // the source array is an exact match of the target. + // + // _.isMatch("100", "1") also returns true which is not what we want. + private isMatch>(expected: T, found: T) { + if (!expected) { + throw new Error("Expected is undefined") + } + if (!found) { + return false + } + + for (const key of Object.keys(expected)) { + if (Array.isArray(expected[key])) { + if (!Array.isArray(found[key])) { + return false + } + if (expected[key].length !== found[key].length) { + return false + } + if (!_.isMatch(found[key], expected[key])) { + return false + } + } else if (typeof expected[key] === "object") { + if (!this.isMatch(expected[key], found[key])) { + return false + } + } else { + if (expected[key] !== found[key]) { + return false + } + } + } + return true + } + + // This function exists to ensure that the same row is not matched twice. + // When a row gets matched, we make sure to remove it from the list of rows + // we're matching against. + private popRow( expectedRow: T, foundRows: T[] ): NonNullable { - const row = foundRows.find(row => _.isMatch(row, expectedRow)) + const row = foundRows.find(row => this.isMatch(expectedRow, row)) if (!row) { const fields = Object.keys(expectedRow) // To make the error message more readable, we only include the fields // that are present in the expected row. const searchedObjects = foundRows.map(row => _.pick(row, fields)) throw new Error( - `Failed to find row: ${JSON.stringify( - expectedRow - )} in ${JSON.stringify(searchedObjects)}` + `Failed to find row:\n\n${JSON.stringify( + expectedRow, + null, + 2 + )}\n\nin\n\n${JSON.stringify(searchedObjects, null, 2)}` ) } - // Ensuring the same row is not matched twice foundRows.splice(foundRows.indexOf(row), 1) return row } @@ -133,8 +176,6 @@ describe.each([ // passed in. The order of the rows is not important, but extra rows will // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { - expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) - const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -156,8 +197,6 @@ describe.each([ // The order of the rows is not important. Extra rows will not cause the // assertion to fail. async toContain(expectedRows: any[]) { - expectedRows.sort((a, b) => Object.keys(b).length - Object.keys(a).length) - const { rows: foundRows } = await config.api.row.search(table._id!, { ...this.query, tableId: table._id!, @@ -1592,7 +1631,7 @@ describe.each([ expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, - { user: {} }, + {}, ])) }) @@ -1664,7 +1703,7 @@ describe.each([ it("successfully finds a row", () => expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ { users: [{ _id: user2._id }] }, - { users: [] }, + {}, ])) it("fails to find nonexistent row", () => @@ -1672,12 +1711,12 @@ describe.each([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - { users: [] }, + {}, ])) }) describe("containsAny", () => { - it.only("successfully finds rows", () => + it("successfully finds rows", () => expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ From 9b3f467a52f917c320a25c53e7cde3038fae5f63 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 11:51:02 +0200 Subject: [PATCH 18/36] Time-date only on constraint settings --- .../DataTable/modals/CreateEditColumn.svelte | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e88c28a9d9..b16e57e866 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -586,13 +586,17 @@ bind:constraints={editableColumn.constraints} bind:optionColors={editableColumn.optionColors} /> - {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn} + {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
- +
@@ -601,7 +605,11 @@
- +
{#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} From ac92aaeab3f8f5ad8a78f38abd9ef4cf8f2b03aa Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:00:50 +0200 Subject: [PATCH 19/36] Extract HH:mm --- packages/bbui/src/helpers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js index dd94d12f7f..0f912a7161 100644 --- a/packages/bbui/src/helpers.js +++ b/packages/bbui/src/helpers.js @@ -170,7 +170,8 @@ export const stringifyDate = ( const offset = referenceDate.getTimezoneOffset() * 60000 const date = new Date(value.valueOf() - offset) if (timeOnly) { - return date.toISOString().slice(11, 19) + // Extract HH:mm + return date.toISOString().slice(11, 16) } return date.toISOString().slice(0, -1) } From 402426a5f3b06ceb7cc78948135fc0fc0767351d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:07:45 +0200 Subject: [PATCH 20/36] Validate time only field constrains --- packages/server/src/sdk/app/rows/utils.ts | 60 +++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 5456b81858..78f499f895 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,8 +1,10 @@ -import cloneDeep from "lodash/cloneDeep" import validateJs from "validate.js" +import dayjs from "dayjs" +import cloneDeep from "lodash/fp/cloneDeep" import { Datasource, DatasourcePlusQueryResponse, + FieldConstraints, FieldType, QueryJson, Row, @@ -206,9 +208,7 @@ export async function validate({ errors[fieldName] = [`Contains invalid JSON`] } } else if (type === FieldType.DATETIME && column.timeOnly) { - if (row[fieldName] && !row[fieldName].match(/^(\d+)(:[0-5]\d){1,2}$/)) { - errors[fieldName] = [`${fieldName} is not a valid time`] - } + res = validateTimeOnlyField(fieldName, row[fieldName], constraints) } else { res = validateJs.single(row[fieldName], constraints) } @@ -216,3 +216,55 @@ export async function validate({ } return { valid: Object.keys(errors).length === 0, errors } } + +function validateTimeOnlyField( + fieldName: string, + value: any, + constraints: FieldConstraints | undefined +) { + let res + if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) { + res = [`${fieldName} is not a valid time`] + } + if (constraints) { + let castedValue = value + const stringTimeToDateISOString = (value: string) => { + const [hour, minute] = value.split(":").map((x: string) => +x) + return dayjs().hour(hour).minute(minute).toISOString() + } + + if (castedValue) { + castedValue = stringTimeToDateISOString(castedValue) + } + let castedConstraints = cloneDeep(constraints) + if (castedConstraints.datetime?.earliest) { + castedConstraints.datetime.earliest = stringTimeToDateISOString( + castedConstraints.datetime?.earliest + ) + } + if (castedConstraints.datetime?.latest) { + castedConstraints.datetime.latest = stringTimeToDateISOString( + castedConstraints.datetime?.latest + ) + } + + let jsValidation = validateJs.single(castedValue, castedConstraints) + jsValidation = jsValidation?.map((m: string) => + m + ?.replace( + castedConstraints.datetime?.earliest || "", + constraints.datetime?.earliest || "" + ) + .replace( + castedConstraints.datetime?.latest || "", + constraints.datetime?.latest || "" + ) + ) + if (jsValidation) { + res ??= [] + res.push(...jsValidation) + } + } + + return res +} From e169454490ba580302fdad8b75470dc1ad8bb0bb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:51:49 +0200 Subject: [PATCH 21/36] Move utils to backend-core --- packages/backend-core/src/docIds/ids.ts | 8 ++++++++ packages/server/src/db/utils.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts index 9627b2b94c..a828c1b91e 100644 --- a/packages/backend-core/src/docIds/ids.ts +++ b/packages/backend-core/src/docIds/ids.ts @@ -18,6 +18,14 @@ export const generateAppID = (tenantId?: string | null) => { return `${id}${newid()}` } +/** + * Generates a new table ID. + * @returns The new table ID which the table doc can be stored under. + */ +export function generateTableID() { + return `${DocumentType.TABLE}${SEPARATOR}${newid()}` +} + /** * Gets a new row ID for the specified table. * @param tableId The table which the row is being created for. diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index ce8d0accbb..e42f30c723 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -77,7 +77,7 @@ export function getTableParams(tableId?: Optional, otherProps = {}) { * @returns The new table ID which the table doc can be stored under. */ export function generateTableID() { - return `${DocumentType.TABLE}${SEPARATOR}${newid()}` + return dbCore.generateTableID() } /** From bed18615b54df076bbae18cbef147621a97c7d06 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:57:42 +0200 Subject: [PATCH 22/36] Add basic tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/server/src/sdk/app/rows/tests/utils.spec.ts diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts new file mode 100644 index 0000000000..38c2c64d6b --- /dev/null +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -0,0 +1,48 @@ +import { + FieldType, + INTERNAL_TABLE_SOURCE_ID, + Table, + TableSourceType, +} from "@budibase/types" +import { generateTableID } from "../../../../db/utils" +import { validate } from "../utils" + +describe("validate", () => { + describe("time only", () => { + const getTable = (): Table => ({ + type: "table", + _id: generateTableID(), + name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + time: { + name: "time", + type: FieldType.DATETIME, + timeOnly: true, + }, + }, + }) + + it("should accept empty fields", async () => { + const row = {} + const table = getTable() + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + expect(output.errors).toEqual({}) + }) + + describe("required", () => { + it("should reject empty fields", async () => { + const row = {} + const table = getTable() + table.schema.time.constraints = { + presence: true, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ["can't be blank"] }) + }) + }) + }) +}) From fb06254964acf4c1a67a034a0afe8e456998a4b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:00:15 +0200 Subject: [PATCH 23/36] Extra tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 38c2c64d6b..e7a9fa10eb 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -6,6 +6,7 @@ import { } from "@budibase/types" import { generateTableID } from "../../../../db/utils" import { validate } from "../utils" +import { generator } from "@budibase/backend-core/tests" describe("validate", () => { describe("time only", () => { @@ -24,7 +25,7 @@ describe("validate", () => { }, }) - it("should accept empty fields", async () => { + it("should accept empty values", async () => { const row = {} const table = getTable() const output = await validate({ table, tableId: table._id!, row }) @@ -32,8 +33,26 @@ describe("validate", () => { expect(output.errors).toEqual({}) }) + it("should accept valid times with HH:mm format", async () => { + const row = { + time: `${generator.hour()}:${generator.minute()}`, + } + const table = getTable() + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + + it("should accept valid times with HH:mm:ss format", async () => { + const row = { + time: `${generator.hour()}:${generator.minute()}:${generator.second()}`, + } + const table = getTable() + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + describe("required", () => { - it("should reject empty fields", async () => { + it("should reject empty values", async () => { const row = {} const table = getTable() table.schema.time.constraints = { @@ -43,6 +62,17 @@ describe("validate", () => { expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) + + it.each([undefined, null])("should reject %s values", async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ["can't be blank"] }) + }) }) }) }) From d58c144dce7d76215e6f0edbeea7bae945ea5569 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:27:31 +0200 Subject: [PATCH 24/36] Add extra tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 81 ++++++++++++++++++- packages/server/src/sdk/app/rows/utils.ts | 13 +-- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index e7a9fa10eb..bed2a7fbfb 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -9,6 +9,10 @@ import { validate } from "../utils" import { generator } from "@budibase/backend-core/tests" describe("validate", () => { + const hour = () => generator.hour().toString().padStart(2, "0") + const minute = () => generator.minute().toString().padStart(2, "0") + const second = minute + describe("time only", () => { const getTable = (): Table => ({ type: "table", @@ -35,7 +39,7 @@ describe("validate", () => { it("should accept valid times with HH:mm format", async () => { const row = { - time: `${generator.hour()}:${generator.minute()}`, + time: `${hour()}:${minute()}`, } const table = getTable() const output = await validate({ table, tableId: table._id!, row }) @@ -44,13 +48,86 @@ describe("validate", () => { it("should accept valid times with HH:mm:ss format", async () => { const row = { - time: `${generator.hour()}:${generator.minute()}:${generator.second()}`, + time: `${hour()}:${minute()}:${second()}`, } const table = getTable() const output = await validate({ table, tableId: table._id!, row }) expect(output.valid).toBe(true) }) + it.each([ + ["ISO datetimes", generator.date().toISOString()], + ["random values", generator.word()], + ])("should reject %s", async (_, time) => { + const row = { + time, + } + const table = getTable() + table.schema.time.constraints = { + presence: true, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ['"time" is not a valid time'] }) + }) + + describe("time constraints", () => { + it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "9:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ["must be no later than 15:00"] }) + }) + }) + describe("required", () => { it("should reject empty values", async () => { const row = {} diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 78f499f895..708bd6f503 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -224,13 +224,16 @@ function validateTimeOnlyField( ) { let res if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) { - res = [`${fieldName} is not a valid time`] - } - if (constraints) { + res = [`"${fieldName}" is not a valid time`] + } else if (constraints) { let castedValue = value const stringTimeToDateISOString = (value: string) => { - const [hour, minute] = value.split(":").map((x: string) => +x) - return dayjs().hour(hour).minute(minute).toISOString() + const [hour, minute, second] = value.split(":").map((x: string) => +x) + let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute) + if (!isNaN(second)) { + date = date.second(second) + } + return date.toISOString() } if (castedValue) { From 5b80e4fb6ecbb6ff51001ff739cd7d726788730b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:35:33 +0200 Subject: [PATCH 25/36] Add more tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 128 +++++++++++++----- 1 file changed, 93 insertions(+), 35 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index bed2a7fbfb..7361f7b13c 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -72,48 +72,76 @@ describe("validate", () => { }) describe("time constraints", () => { - it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( - "should accept values in range (%s)", - async time => { - const row = { time } - const table = getTable() - table.schema.time.constraints = { - presence: true, - datetime: { - earliest: "10:00", - latest: "15:00", - }, - } - const output = await validate({ table, tableId: table._id!, row }) - expect(output.valid).toBe(true) - } - ) - - it.each([ - "9:59:50", - `${generator.integer({ min: 0, max: 9 })}:${minute()}`, - ])("should reject values before range (%s)", async time => { - const row = { time } + describe("earliest only", () => { const table = getTable() table.schema.time.constraints = { presence: true, datetime: { earliest: "10:00", - latest: "15:00", + latest: "", }, } - const output = await validate({ table, tableId: table._id!, row }) - expect(output.valid).toBe(false) - expect(output.errors).toEqual({ - time: ["must be no earlier than 10:00"], + + it.each([ + "10:00", + "15:00", + `10:${minute()}`, + "12:34", + `${generator.integer({ min: 11, max: 23 })}:${minute()}`, + ])("should accept values after config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + + it.each([ + "09:59:59", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) }) }) - it.each([ - "15:00:01", - `${generator.integer({ min: 16, max: 23 })}:${minute()}`, - ])("should reject values after range (%s)", async time => { - const row = { time } + describe("latest only", () => { + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "", + latest: "15:16:17", + }, + } + + it.each([ + "15:16:17", + "15:16", + "15:00", + `${generator.integer({ min: 0, max: 12 })}:${minute()}`, + ])("should accept values before config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + + it.each([ + "15:16:18", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:16:17"], + }) + }) + }) + + describe("range", () => { const table = getTable() table.schema.time.constraints = { presence: true, @@ -122,9 +150,39 @@ describe("validate", () => { latest: "15:00", }, } - const output = await validate({ table, tableId: table._id!, row }) - expect(output.valid).toBe(false) - expect(output.errors).toEqual({ time: ["must be no later than 15:00"] }) + + it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "9:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:00"], + }) + }) }) }) From d1ef9067dcbaaa345c975bb55fba46107818e0b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:47:54 +0200 Subject: [PATCH 26/36] Allow range crossing midnight --- .../src/sdk/app/rows/tests/utils.spec.ts | 20 +++++++++++ packages/server/src/sdk/app/rows/utils.ts | 35 +++++++++++++------ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 7361f7b13c..acd4281256 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -183,6 +183,26 @@ describe("validate", () => { time: ["must be no later than 15:00"], }) }) + + describe("range crossing midnight", () => { + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "15:00", + latest: "10:00", + }, + } + + it.each(["10:00", "15:00", `9:${minute()}`, "16:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + }) }) }) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 708bd6f503..19e8869494 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -227,31 +227,46 @@ function validateTimeOnlyField( res = [`"${fieldName}" is not a valid time`] } else if (constraints) { let castedValue = value - const stringTimeToDateISOString = (value: string) => { + const stringTimeToDate = (value: string) => { const [hour, minute, second] = value.split(":").map((x: string) => +x) let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute) if (!isNaN(second)) { date = date.second(second) } - return date.toISOString() + return date } if (castedValue) { - castedValue = stringTimeToDateISOString(castedValue) + castedValue = stringTimeToDate(castedValue) } let castedConstraints = cloneDeep(constraints) + + let earliest, latest if (castedConstraints.datetime?.earliest) { - castedConstraints.datetime.earliest = stringTimeToDateISOString( - castedConstraints.datetime?.earliest - ) + earliest = stringTimeToDate(castedConstraints.datetime?.earliest) } if (castedConstraints.datetime?.latest) { - castedConstraints.datetime.latest = stringTimeToDateISOString( - castedConstraints.datetime?.latest - ) + latest = stringTimeToDate(castedConstraints.datetime?.latest) } - let jsValidation = validateJs.single(castedValue, castedConstraints) + if (earliest && latest && earliest.isAfter(latest)) { + latest = latest.add(1, "day") + if (earliest.isAfter(castedValue)) { + castedValue = castedValue.add(1, "day") + } + } + + if (earliest || latest) { + castedConstraints.datetime = { + earliest: earliest?.toISOString() || "", + latest: latest?.toISOString() || "", + } + } + + let jsValidation = validateJs.single( + castedValue?.toISOString(), + castedConstraints + ) jsValidation = jsValidation?.map((m: string) => m ?.replace( From b8400294d54b1baef60de5e95a835918f21a8656 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:53:17 +0200 Subject: [PATCH 27/36] Add extra tests --- .../server/src/sdk/app/rows/tests/utils.spec.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index acd4281256..4c44791135 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -194,7 +194,7 @@ describe("validate", () => { }, } - it.each(["10:00", "15:00", `9:${minute()}`, "16:34"])( + it.each(["10:00", "15:00", `9:${minute()}`, "16:34", "00:00"])( "should accept values in range (%s)", async time => { const row = { time } @@ -202,6 +202,18 @@ describe("validate", () => { expect(output.valid).toBe(true) } ) + + it.each(["10:01", "14:59:59", `12:${minute()}`])( + "should reject values out range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 10:00"], + }) + } + ) }) }) }) From 43acea931ac4b3846604a40569d125b2ad7155af Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 15:23:02 +0200 Subject: [PATCH 28/36] Ensure iso time config still work --- .../src/sdk/app/rows/tests/utils.spec.ts | 90 +++++++++++++++++++ packages/server/src/sdk/app/rows/utils.ts | 21 ++++- 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 4c44791135..55cdf9ea20 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -1,3 +1,4 @@ +import dayjs from "dayjs" import { FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -241,5 +242,94 @@ describe("validate", () => { expect(output.errors).toEqual({ time: ["can't be blank"] }) }) }) + + describe("range", () => { + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + + it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "9:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:00"], + }) + }) + + describe("datetime ISO configs", () => { + const table = getTable() + + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: dayjs().hour(10).minute(0).second(0).toISOString(), + latest: dayjs().hour(15).minute(0).second(0).toISOString(), + }, + } + + it.each(["10:00", "15:00", `12:${minute()}`])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "09:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:00"], + }) + }) + }) + }) }) }) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 19e8869494..23e1ab3e6b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -242,11 +242,24 @@ function validateTimeOnlyField( let castedConstraints = cloneDeep(constraints) let earliest, latest + let easliestTimeString: string, latestTimeString: string if (castedConstraints.datetime?.earliest) { - earliest = stringTimeToDate(castedConstraints.datetime?.earliest) + easliestTimeString = castedConstraints.datetime.earliest + if (dayjs(castedConstraints.datetime.earliest).isValid()) { + easliestTimeString = dayjs(castedConstraints.datetime.earliest).format( + "HH:mm" + ) + } + earliest = stringTimeToDate(easliestTimeString) } if (castedConstraints.datetime?.latest) { - latest = stringTimeToDate(castedConstraints.datetime?.latest) + latestTimeString = castedConstraints.datetime.latest + if (dayjs(castedConstraints.datetime.latest).isValid()) { + latestTimeString = dayjs(castedConstraints.datetime.latest).format( + "HH:mm" + ) + } + latest = stringTimeToDate(latestTimeString) } if (earliest && latest && earliest.isAfter(latest)) { @@ -271,11 +284,11 @@ function validateTimeOnlyField( m ?.replace( castedConstraints.datetime?.earliest || "", - constraints.datetime?.earliest || "" + easliestTimeString || "" ) .replace( castedConstraints.datetime?.latest || "", - constraints.datetime?.latest || "" + latestTimeString || "" ) ) if (jsValidation) { From a6d2f82e7b185656e661d7a7821b3965d00b118f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 15:16:52 +0100 Subject: [PATCH 29/36] Fix tests. --- packages/server/src/api/controllers/view/exporters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index 3b5f951dca..9cf114f4e5 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -57,5 +57,5 @@ export function isFormat(format: any): format is Format { } export function parseCsvExport(value: string) { - return JSON.parse(value?.replace(/'/g, '"')) as T + return JSON.parse(value) as T } From 7d256d235afe75c179056616d142392a96101637 Mon Sep 17 00:00:00 2001 From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com> Date: Thu, 23 May 2024 15:31:11 +0100 Subject: [PATCH 30/36] Enum columns unexpectedly breaking fix (#13756) * Added enum to SQL_MISC_TYPE_MAP to correctly map to FieldType.OPTIONS * improve enum values extraction for multiple single-select support * Tests to ensure enums are typed correctly MySQL and Postgres * Fixed linting issue * Ran prettier --- .../server/src/integration-test/mysql.spec.ts | 36 +++++++++++++++++++ .../src/integration-test/postgres.spec.ts | 31 ++++++++++++++++ packages/server/src/integrations/postgres.ts | 10 +++--- .../server/src/integrations/utils/utils.ts | 1 + 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index b4eb1035d6..8cf4fb8212 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -281,4 +281,40 @@ describe("mysql integrations", () => { ]) }) }) + + describe("POST /api/datasources/:datasourceId/schema", () => { + let tableName: string + + beforeEach(async () => { + tableName = uniqueTableName() + }) + + afterEach(async () => { + await rawQuery(rawDatasource, `DROP TABLE IF EXISTS \`${tableName}\``) + }) + + it("recognises enum columns as options", async () => { + const enumColumnName = "status" + + const createTableQuery = ` + CREATE TABLE \`${tableName}\` ( + \`order_id\` INT AUTO_INCREMENT PRIMARY KEY, + \`customer_name\` VARCHAR(100) NOT NULL, + \`${enumColumnName}\` ENUM('pending', 'processing', 'shipped', 'delivered', 'cancelled') + ); + ` + + await rawQuery(rawDatasource, createTableQuery) + + const response = await makeRequest( + "post", + `/api/datasources/${datasource._id}/schema` + ) + + const table = response.body.datasource.entities[tableName] + + expect(table).toBeDefined() + expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) + }) + }) }) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index ec4cb90a86..ccf63d0820 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1122,6 +1122,37 @@ describe("postgres integrations", () => { [tableName]: "Table contains invalid columns.", }) }) + + it("recognises enum columns as options", async () => { + const tableName = `orders_${generator + .guid() + .replaceAll("-", "") + .substring(0, 6)}` + const enumColumnName = "status" + + await rawQuery( + rawDatasource, + ` + CREATE TYPE order_status AS ENUM ('pending', 'processing', 'shipped', 'delivered', 'cancelled'); + + CREATE TABLE ${tableName} ( + order_id SERIAL PRIMARY KEY, + customer_name VARCHAR(100) NOT NULL, + ${enumColumnName} order_status + ); + ` + ) + + const response = await makeRequest( + "post", + `/api/datasources/${datasource._id}/schema` + ) + + const table = response.body.datasource.entities[tableName] + + expect(table).toBeDefined() + expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) + }) }) describe("Integration compatibility with postgres search_path", () => { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index e810986757..584f0ef15d 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -329,14 +329,12 @@ class PostgresIntegration extends Sql implements DatasourcePlus { // Fetch enum values const enumsResponse = await this.client.query(this.ENUM_VALUES()) + // output array, allows for more than 1 single-select to be used at a time const enumValues = enumsResponse.rows?.reduce((acc, row) => { - if (!acc[row.typname]) { - return { - [row.typname]: [row.enumlabel], - } + return { + ...acc, + [row.typname]: [...(acc[row.typname] || []), row.enumlabel], } - acc[row.typname].push(row.enumlabel) - return acc }, {}) for (let column of columnsResponse.rows) { diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 8f3aba8907..5bc90bc295 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -102,6 +102,7 @@ const SQL_OPTIONS_TYPE_MAP: Record = { const SQL_MISC_TYPE_MAP: Record = { json: FieldType.JSON, bigint: FieldType.BIGINT, + enum: FieldType.OPTIONS, } const SQL_TYPE_MAP: Record = { From 1004b052baae179513d32dfab57e1a3a64ea1ed3 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 23 May 2024 14:39:52 +0000 Subject: [PATCH 31/36] Bump version to 2.27.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a943000cea..db0a1d59fa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.2", + "version": "2.27.3", "npmClient": "yarn", "packages": [ "packages/*", From 325819ebae9fea081f2bd46a94952d7b51fab58a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 16:13:07 +0100 Subject: [PATCH 32/36] Fix tests (take 3). --- .../src/integrations/tests/sqlAlias.spec.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 0de4d0a151..f907e1ab8e 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -77,7 +77,7 @@ describe("Captures of real examples", () => { "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 + left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where COALESCE("b"."taskname" = $2, FALSE) order by "a"."productname" asc nulls first limit $3`), }) }) @@ -137,12 +137,12 @@ describe("Captures of real examples", () => { "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname" - from (select * from "tasks" as "a" where not "a"."completed" = $1 + from (select * from "tasks" as "a" where COALESCE("a"."completed" != $1, TRUE) order by "a"."taskname" asc nulls first limit $2) as "a" left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" left join "products" as "b" on "b"."productid" = "d"."productid" left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" - where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`), + where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE) order by "a"."taskname" asc nulls first limit $6`), }) }) }) @@ -154,7 +154,7 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`), + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), }) }) @@ -164,7 +164,7 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`), + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), }) }) }) @@ -175,8 +175,9 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) expect(query).toEqual({ bindings: ["ddd", ""], - sql: multiline(`delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 - returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), + sql: multiline(`delete from "compositetable" as "a" + where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE) + returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), }) }) }) @@ -197,7 +198,7 @@ describe("Captures of real examples", () => { returningQuery = input }, queryJson) expect(returningQuery).toEqual({ - sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]", + sql: "select * from (select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]", bindings: [1, "Test", 22], }) }) From ef60893df1677d58c450f70dcf7532f45ee305be Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 May 2024 17:02:08 +0100 Subject: [PATCH 33/36] Fix tests (take 4). --- packages/server/src/integrations/tests/sql.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 9b84409e92..302b61fc74 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -189,7 +189,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit], - sql: `select * from (select * from (select * from "test" where (LOWER("test"."age") LIKE :1 AND LOWER("test"."age") LIKE :2) and (LOWER("test"."name") LIKE :3 AND LOWER("test"."name") LIKE :4)) where rownum <= :5) "test"`, + sql: `select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4)) where rownum <= :5) "test"`, }) query = new Sql(SqlClient.ORACLE, limit)._query( From 0db08498d9c1a38b02485ec987561dfdd53ea3dc Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 23 May 2024 17:44:23 +0100 Subject: [PATCH 34/36] Fix for grid arrow event handling when creating a column --- .../DataTable/modals/CreateEditColumn.svelte | 479 +++++++++--------- .../grid/overlays/KeyboardManager.svelte | 1 + 2 files changed, 242 insertions(+), 238 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 8583dbcab7..477ca1caa9 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -517,258 +517,261 @@ }) - - {#if mounted} - { - if ( - !uneditable && - !(linkEditDisabled && editableColumn.type === LINK_TYPE) - ) { - editableColumn.name = e.target.value - } - }} - disabled={uneditable || - (linkEditDisabled && editableColumn.type === LINK_TYPE)} - error={errors?.name} - /> - {/if} - - {:else if editableColumn.type === FieldType.OPTIONS} - - {:else if editableColumn.type === FieldType.LONGFORM} -
-
- - - - -
- - + + {#if mounted} + { + if ( + !uneditable && + !(linkEditDisabled && editableColumn.type === LINK_TYPE) + ) { + editableColumn.name = e.target.value + } + }} + disabled={uneditable || + (linkEditDisabled && editableColumn.type === LINK_TYPE)} + error={errors?.name} /> -
- {:else if editableColumn.type === FieldType.ARRAY} - - {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn} -
-
- -
-
- -
-
- -
-
- -
-
- -
-
- {#if !editableColumn.timeOnly} - {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} -
-
- - - - -
- -
- {/if} - {/if} - {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn} -
-
- -
-
- -
-
- -
-
- -
-
- -
-
- {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} - field.name} + getOptionValue={field => field.fieldId} + getOptionIcon={field => field.icon} + isOptionEnabled={option => { + if (option.type === AUTO_TYPE) { + return availableAutoColumnKeys?.length > 0 + } + return true + }} /> - {:else if editableColumn.type === FORMULA_TYPE} - {#if !table.sql} + + {#if editableColumn.type === FieldType.STRING} + + {:else if editableColumn.type === FieldType.OPTIONS} + + {:else if editableColumn.type === FieldType.LONGFORM} +
+
+ + + + +
+ + +
+ {:else if editableColumn.type === FieldType.ARRAY} + + {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
- +
- (editableColumn.subtype = e.detail)} - options={Object.entries(autoColumnOptions)} - getOptionLabel={option => option[1].name} - getOptionValue={option => option[0]} - disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn} - error={errors?.subtype} - /> - {/if} - {#if canBeRequired || canBeDisplay} -
- {#if canBeRequired} - +
+
+ +
+
+ +
+
+ {#if !editableColumn.timeOnly} + {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} +
+
+ + + + +
+ +
+ {/if} + {/if} -
- {/if} - - -
- {#if !uneditable && originalName != null} - - {/if} - - + {/if} + + +
+ Date: Thu, 23 May 2024 17:45:01 +0100 Subject: [PATCH 35/36] Fix for tooltips in the Component tree in the builder --- .../builder/src/components/common/NavItem.svelte | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index 5cc6db65a0..5a04907570 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -1,5 +1,5 @@ - - - {#if mounted} - { - if ( - !uneditable && - !(linkEditDisabled && editableColumn.type === LINK_TYPE) - ) { - editableColumn.name = e.target.value - } - }} - disabled={uneditable || - (linkEditDisabled && editableColumn.type === LINK_TYPE)} - error={errors?.name} - /> - {/if} - { + if ( + !uneditable && + !(linkEditDisabled && editableColumn.type === LINK_TYPE) + ) { + editableColumn.name = e.target.value } - return true }} + disabled={uneditable || + (linkEditDisabled && editableColumn.type === LINK_TYPE)} + error={errors?.name} /> + {/if} + - {:else if editableColumn.type === FieldType.OPTIONS} - - {:else if editableColumn.type === FieldType.LONGFORM} -
-
- - - - -
+ {#if editableColumn.type === FieldType.STRING} + + {:else if editableColumn.type === FieldType.OPTIONS} + + {:else if editableColumn.type === FieldType.LONGFORM} +
+
+ + + + +
- +
+ {:else if editableColumn.type === FieldType.ARRAY} + + {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn} +
+
+ +
+
+
- {:else if editableColumn.type === FieldType.ARRAY} - - {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn} -
-
- -
-
- -
-
+
-
-
- -
-
- -
+
+
+
- {#if !editableColumn.timeOnly} - {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} -
-
- - - - -
- +
+ +
+
+ {#if !editableColumn.timeOnly} + {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} +
+
+ + + +
- {/if} - - {/if} - {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn} -
-
- -
-
- -
-
- -
-
- -
-
- -
-
- {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} - - {:else if editableColumn.type === FORMULA_TYPE} - {#if !table.sql} -
-
- -
-
- (editableColumn.subtype = e.detail)} - options={Object.entries(autoColumnOptions)} - getOptionLabel={option => option[1].name} - getOptionValue={option => option[0]} - disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn} - error={errors?.subtype} - /> - {/if} - - {#if canBeRequired || canBeDisplay} -
- {#if canBeRequired} - {/if} +
+ {/if} + + {/if} + {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn} +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} + + {:else if editableColumn.type === FORMULA_TYPE} + {#if !table.sql} +
+
+ +
+
+ (editableColumn.subtype = e.detail)} + options={Object.entries(autoColumnOptions)} + getOptionLabel={option => option[1].name} + getOptionValue={option => option[0]} + disabled={!availableAutoColumnKeys?.length || editableColumn.autocolumn} + error={errors?.subtype} + /> + {/if} + + {#if canBeRequired || canBeDisplay} +
+ {#if canBeRequired} + {/if} - -
- +
+ {/if} + + +
+ {#if !uneditable && originalName != null} + + {/if} + + +