From 2cc329994a16ff3010259837782bdcf49aace95c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 May 2024 17:18:39 +0200 Subject: [PATCH 01/18] Don't allow chaning time only to datetime --- .../DataTable/modals/CreateEditColumn.svelte | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e88c28a9d9..26d0357b5f 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -604,27 +604,29 @@ - {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} -
-
- - - - + {#if !editableColumn.timeOnly} + {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} +
+
+ + + + +
+
- -
+ {/if} + {/if} - {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
From b3bea1e839b091efbe4beac0e50faa981c5e1716 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 22 May 2024 17:18:58 +0200 Subject: [PATCH 02/18] Block timeOnly changes in the api --- packages/server/src/sdk/app/tables/external/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 98e6e561c8..53302085fc 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -73,6 +73,16 @@ function validate(table: Table, oldTable?: Table) { `Column "${key}" has subtype "${column.subtype}" - this is not supported.` ) } + + if (column.type === FieldType.DATETIME) { + const oldColumn = oldTable?.schema[key] as typeof column + + if (column.timeOnly !== oldColumn.timeOnly) { + throw new Error( + `Column "${key}" can not change from time to datetime or viceversa.` + ) + } + } } } From 028afd9cca04808615b672f9979f62af469b13d5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 11:34:58 +0200 Subject: [PATCH 03/18] Fix checks --- packages/server/src/sdk/app/tables/external/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 53302085fc..842b6b5648 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -77,7 +77,7 @@ function validate(table: Table, oldTable?: Table) { if (column.type === FieldType.DATETIME) { const oldColumn = oldTable?.schema[key] as typeof column - if (column.timeOnly !== oldColumn.timeOnly) { + if (oldColumn && column.timeOnly !== oldColumn.timeOnly) { throw new Error( `Column "${key}" can not change from time to datetime or viceversa.` ) From f6146c4974b82e96d1ab3242d4ddb202de76015f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 11:33:41 +0200 Subject: [PATCH 04/18] Validate time only fields --- packages/server/src/sdk/app/rows/utils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 777ebff655..5456b81858 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -205,6 +205,10 @@ export async function validate({ } catch (err) { errors[fieldName] = [`Contains invalid JSON`] } + } else if (type === FieldType.DATETIME && column.timeOnly) { + if (row[fieldName] && !row[fieldName].match(/^(\d+)(:[0-5]\d){1,2}$/)) { + errors[fieldName] = [`${fieldName} is not a valid time`] + } } else { res = validateJs.single(row[fieldName], constraints) } From 2626c1a721ad40f3ae2bc35745e02b80b8b887ab Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 11:33:50 +0200 Subject: [PATCH 05/18] Send right time --- packages/bbui/src/helpers.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js index 1db4a773ba..dd94d12f7f 100644 --- a/packages/bbui/src/helpers.js +++ b/packages/bbui/src/helpers.js @@ -168,7 +168,11 @@ export const stringifyDate = ( // Ensure we use the correct offset for the date const referenceDate = value.toDate() const offset = referenceDate.getTimezoneOffset() * 60000 - return new Date(value.valueOf() - offset).toISOString().slice(0, -1) + const date = new Date(value.valueOf() - offset) + if (timeOnly) { + return date.toISOString().slice(11, 19) + } + return date.toISOString().slice(0, -1) } // For date-only fields, construct a manual timestamp string without a time From 9b3f467a52f917c320a25c53e7cde3038fae5f63 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 11:51:02 +0200 Subject: [PATCH 06/18] Time-date only on constraint settings --- .../DataTable/modals/CreateEditColumn.svelte | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e88c28a9d9..b16e57e866 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -586,13 +586,17 @@ bind:constraints={editableColumn.constraints} bind:optionColors={editableColumn.optionColors} /> - {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn} + {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
- +
@@ -601,7 +605,11 @@
- +
{#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly} From ac92aaeab3f8f5ad8a78f38abd9ef4cf8f2b03aa Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:00:50 +0200 Subject: [PATCH 07/18] Extract HH:mm --- packages/bbui/src/helpers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/helpers.js b/packages/bbui/src/helpers.js index dd94d12f7f..0f912a7161 100644 --- a/packages/bbui/src/helpers.js +++ b/packages/bbui/src/helpers.js @@ -170,7 +170,8 @@ export const stringifyDate = ( const offset = referenceDate.getTimezoneOffset() * 60000 const date = new Date(value.valueOf() - offset) if (timeOnly) { - return date.toISOString().slice(11, 19) + // Extract HH:mm + return date.toISOString().slice(11, 16) } return date.toISOString().slice(0, -1) } From 402426a5f3b06ceb7cc78948135fc0fc0767351d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:07:45 +0200 Subject: [PATCH 08/18] Validate time only field constrains --- packages/server/src/sdk/app/rows/utils.ts | 60 +++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 5456b81858..78f499f895 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,8 +1,10 @@ -import cloneDeep from "lodash/cloneDeep" import validateJs from "validate.js" +import dayjs from "dayjs" +import cloneDeep from "lodash/fp/cloneDeep" import { Datasource, DatasourcePlusQueryResponse, + FieldConstraints, FieldType, QueryJson, Row, @@ -206,9 +208,7 @@ export async function validate({ errors[fieldName] = [`Contains invalid JSON`] } } else if (type === FieldType.DATETIME && column.timeOnly) { - if (row[fieldName] && !row[fieldName].match(/^(\d+)(:[0-5]\d){1,2}$/)) { - errors[fieldName] = [`${fieldName} is not a valid time`] - } + res = validateTimeOnlyField(fieldName, row[fieldName], constraints) } else { res = validateJs.single(row[fieldName], constraints) } @@ -216,3 +216,55 @@ export async function validate({ } return { valid: Object.keys(errors).length === 0, errors } } + +function validateTimeOnlyField( + fieldName: string, + value: any, + constraints: FieldConstraints | undefined +) { + let res + if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) { + res = [`${fieldName} is not a valid time`] + } + if (constraints) { + let castedValue = value + const stringTimeToDateISOString = (value: string) => { + const [hour, minute] = value.split(":").map((x: string) => +x) + return dayjs().hour(hour).minute(minute).toISOString() + } + + if (castedValue) { + castedValue = stringTimeToDateISOString(castedValue) + } + let castedConstraints = cloneDeep(constraints) + if (castedConstraints.datetime?.earliest) { + castedConstraints.datetime.earliest = stringTimeToDateISOString( + castedConstraints.datetime?.earliest + ) + } + if (castedConstraints.datetime?.latest) { + castedConstraints.datetime.latest = stringTimeToDateISOString( + castedConstraints.datetime?.latest + ) + } + + let jsValidation = validateJs.single(castedValue, castedConstraints) + jsValidation = jsValidation?.map((m: string) => + m + ?.replace( + castedConstraints.datetime?.earliest || "", + constraints.datetime?.earliest || "" + ) + .replace( + castedConstraints.datetime?.latest || "", + constraints.datetime?.latest || "" + ) + ) + if (jsValidation) { + res ??= [] + res.push(...jsValidation) + } + } + + return res +} From e169454490ba580302fdad8b75470dc1ad8bb0bb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:51:49 +0200 Subject: [PATCH 09/18] Move utils to backend-core --- packages/backend-core/src/docIds/ids.ts | 8 ++++++++ packages/server/src/db/utils.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts index 9627b2b94c..a828c1b91e 100644 --- a/packages/backend-core/src/docIds/ids.ts +++ b/packages/backend-core/src/docIds/ids.ts @@ -18,6 +18,14 @@ export const generateAppID = (tenantId?: string | null) => { return `${id}${newid()}` } +/** + * Generates a new table ID. + * @returns The new table ID which the table doc can be stored under. + */ +export function generateTableID() { + return `${DocumentType.TABLE}${SEPARATOR}${newid()}` +} + /** * Gets a new row ID for the specified table. * @param tableId The table which the row is being created for. diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index ce8d0accbb..e42f30c723 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -77,7 +77,7 @@ export function getTableParams(tableId?: Optional, otherProps = {}) { * @returns The new table ID which the table doc can be stored under. */ export function generateTableID() { - return `${DocumentType.TABLE}${SEPARATOR}${newid()}` + return dbCore.generateTableID() } /** From bed18615b54df076bbae18cbef147621a97c7d06 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 13:57:42 +0200 Subject: [PATCH 10/18] Add basic tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/server/src/sdk/app/rows/tests/utils.spec.ts diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts new file mode 100644 index 0000000000..38c2c64d6b --- /dev/null +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -0,0 +1,48 @@ +import { + FieldType, + INTERNAL_TABLE_SOURCE_ID, + Table, + TableSourceType, +} from "@budibase/types" +import { generateTableID } from "../../../../db/utils" +import { validate } from "../utils" + +describe("validate", () => { + describe("time only", () => { + const getTable = (): Table => ({ + type: "table", + _id: generateTableID(), + name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + time: { + name: "time", + type: FieldType.DATETIME, + timeOnly: true, + }, + }, + }) + + it("should accept empty fields", async () => { + const row = {} + const table = getTable() + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + expect(output.errors).toEqual({}) + }) + + describe("required", () => { + it("should reject empty fields", async () => { + const row = {} + const table = getTable() + table.schema.time.constraints = { + presence: true, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ["can't be blank"] }) + }) + }) + }) +}) From fb06254964acf4c1a67a034a0afe8e456998a4b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:00:15 +0200 Subject: [PATCH 11/18] Extra tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 38c2c64d6b..e7a9fa10eb 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -6,6 +6,7 @@ import { } from "@budibase/types" import { generateTableID } from "../../../../db/utils" import { validate } from "../utils" +import { generator } from "@budibase/backend-core/tests" describe("validate", () => { describe("time only", () => { @@ -24,7 +25,7 @@ describe("validate", () => { }, }) - it("should accept empty fields", async () => { + it("should accept empty values", async () => { const row = {} const table = getTable() const output = await validate({ table, tableId: table._id!, row }) @@ -32,8 +33,26 @@ describe("validate", () => { expect(output.errors).toEqual({}) }) + it("should accept valid times with HH:mm format", async () => { + const row = { + time: `${generator.hour()}:${generator.minute()}`, + } + const table = getTable() + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + + it("should accept valid times with HH:mm:ss format", async () => { + const row = { + time: `${generator.hour()}:${generator.minute()}:${generator.second()}`, + } + const table = getTable() + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + describe("required", () => { - it("should reject empty fields", async () => { + it("should reject empty values", async () => { const row = {} const table = getTable() table.schema.time.constraints = { @@ -43,6 +62,17 @@ describe("validate", () => { expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) + + it.each([undefined, null])("should reject %s values", async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ["can't be blank"] }) + }) }) }) }) From d58c144dce7d76215e6f0edbeea7bae945ea5569 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:27:31 +0200 Subject: [PATCH 12/18] Add extra tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 81 ++++++++++++++++++- packages/server/src/sdk/app/rows/utils.ts | 13 +-- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index e7a9fa10eb..bed2a7fbfb 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -9,6 +9,10 @@ import { validate } from "../utils" import { generator } from "@budibase/backend-core/tests" describe("validate", () => { + const hour = () => generator.hour().toString().padStart(2, "0") + const minute = () => generator.minute().toString().padStart(2, "0") + const second = minute + describe("time only", () => { const getTable = (): Table => ({ type: "table", @@ -35,7 +39,7 @@ describe("validate", () => { it("should accept valid times with HH:mm format", async () => { const row = { - time: `${generator.hour()}:${generator.minute()}`, + time: `${hour()}:${minute()}`, } const table = getTable() const output = await validate({ table, tableId: table._id!, row }) @@ -44,13 +48,86 @@ describe("validate", () => { it("should accept valid times with HH:mm:ss format", async () => { const row = { - time: `${generator.hour()}:${generator.minute()}:${generator.second()}`, + time: `${hour()}:${minute()}:${second()}`, } const table = getTable() const output = await validate({ table, tableId: table._id!, row }) expect(output.valid).toBe(true) }) + it.each([ + ["ISO datetimes", generator.date().toISOString()], + ["random values", generator.word()], + ])("should reject %s", async (_, time) => { + const row = { + time, + } + const table = getTable() + table.schema.time.constraints = { + presence: true, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ['"time" is not a valid time'] }) + }) + + describe("time constraints", () => { + it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "9:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ time: ["must be no later than 15:00"] }) + }) + }) + describe("required", () => { it("should reject empty values", async () => { const row = {} diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 78f499f895..708bd6f503 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -224,13 +224,16 @@ function validateTimeOnlyField( ) { let res if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) { - res = [`${fieldName} is not a valid time`] - } - if (constraints) { + res = [`"${fieldName}" is not a valid time`] + } else if (constraints) { let castedValue = value const stringTimeToDateISOString = (value: string) => { - const [hour, minute] = value.split(":").map((x: string) => +x) - return dayjs().hour(hour).minute(minute).toISOString() + const [hour, minute, second] = value.split(":").map((x: string) => +x) + let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute) + if (!isNaN(second)) { + date = date.second(second) + } + return date.toISOString() } if (castedValue) { From 5b80e4fb6ecbb6ff51001ff739cd7d726788730b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:35:33 +0200 Subject: [PATCH 13/18] Add more tests --- .../src/sdk/app/rows/tests/utils.spec.ts | 128 +++++++++++++----- 1 file changed, 93 insertions(+), 35 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index bed2a7fbfb..7361f7b13c 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -72,48 +72,76 @@ describe("validate", () => { }) describe("time constraints", () => { - it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( - "should accept values in range (%s)", - async time => { - const row = { time } - const table = getTable() - table.schema.time.constraints = { - presence: true, - datetime: { - earliest: "10:00", - latest: "15:00", - }, - } - const output = await validate({ table, tableId: table._id!, row }) - expect(output.valid).toBe(true) - } - ) - - it.each([ - "9:59:50", - `${generator.integer({ min: 0, max: 9 })}:${minute()}`, - ])("should reject values before range (%s)", async time => { - const row = { time } + describe("earliest only", () => { const table = getTable() table.schema.time.constraints = { presence: true, datetime: { earliest: "10:00", - latest: "15:00", + latest: "", }, } - const output = await validate({ table, tableId: table._id!, row }) - expect(output.valid).toBe(false) - expect(output.errors).toEqual({ - time: ["must be no earlier than 10:00"], + + it.each([ + "10:00", + "15:00", + `10:${minute()}`, + "12:34", + `${generator.integer({ min: 11, max: 23 })}:${minute()}`, + ])("should accept values after config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + + it.each([ + "09:59:59", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) }) }) - it.each([ - "15:00:01", - `${generator.integer({ min: 16, max: 23 })}:${minute()}`, - ])("should reject values after range (%s)", async time => { - const row = { time } + describe("latest only", () => { + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "", + latest: "15:16:17", + }, + } + + it.each([ + "15:16:17", + "15:16", + "15:00", + `${generator.integer({ min: 0, max: 12 })}:${minute()}`, + ])("should accept values before config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + }) + + it.each([ + "15:16:18", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after config value (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:16:17"], + }) + }) + }) + + describe("range", () => { const table = getTable() table.schema.time.constraints = { presence: true, @@ -122,9 +150,39 @@ describe("validate", () => { latest: "15:00", }, } - const output = await validate({ table, tableId: table._id!, row }) - expect(output.valid).toBe(false) - expect(output.errors).toEqual({ time: ["must be no later than 15:00"] }) + + it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "9:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:00"], + }) + }) }) }) From d1ef9067dcbaaa345c975bb55fba46107818e0b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:47:54 +0200 Subject: [PATCH 14/18] Allow range crossing midnight --- .../src/sdk/app/rows/tests/utils.spec.ts | 20 +++++++++++ packages/server/src/sdk/app/rows/utils.ts | 35 +++++++++++++------ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 7361f7b13c..acd4281256 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -183,6 +183,26 @@ describe("validate", () => { time: ["must be no later than 15:00"], }) }) + + describe("range crossing midnight", () => { + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "15:00", + latest: "10:00", + }, + } + + it.each(["10:00", "15:00", `9:${minute()}`, "16:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + }) }) }) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 708bd6f503..19e8869494 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -227,31 +227,46 @@ function validateTimeOnlyField( res = [`"${fieldName}" is not a valid time`] } else if (constraints) { let castedValue = value - const stringTimeToDateISOString = (value: string) => { + const stringTimeToDate = (value: string) => { const [hour, minute, second] = value.split(":").map((x: string) => +x) let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute) if (!isNaN(second)) { date = date.second(second) } - return date.toISOString() + return date } if (castedValue) { - castedValue = stringTimeToDateISOString(castedValue) + castedValue = stringTimeToDate(castedValue) } let castedConstraints = cloneDeep(constraints) + + let earliest, latest if (castedConstraints.datetime?.earliest) { - castedConstraints.datetime.earliest = stringTimeToDateISOString( - castedConstraints.datetime?.earliest - ) + earliest = stringTimeToDate(castedConstraints.datetime?.earliest) } if (castedConstraints.datetime?.latest) { - castedConstraints.datetime.latest = stringTimeToDateISOString( - castedConstraints.datetime?.latest - ) + latest = stringTimeToDate(castedConstraints.datetime?.latest) } - let jsValidation = validateJs.single(castedValue, castedConstraints) + if (earliest && latest && earliest.isAfter(latest)) { + latest = latest.add(1, "day") + if (earliest.isAfter(castedValue)) { + castedValue = castedValue.add(1, "day") + } + } + + if (earliest || latest) { + castedConstraints.datetime = { + earliest: earliest?.toISOString() || "", + latest: latest?.toISOString() || "", + } + } + + let jsValidation = validateJs.single( + castedValue?.toISOString(), + castedConstraints + ) jsValidation = jsValidation?.map((m: string) => m ?.replace( From b8400294d54b1baef60de5e95a835918f21a8656 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 14:53:17 +0200 Subject: [PATCH 15/18] Add extra tests --- .../server/src/sdk/app/rows/tests/utils.spec.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index acd4281256..4c44791135 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -194,7 +194,7 @@ describe("validate", () => { }, } - it.each(["10:00", "15:00", `9:${minute()}`, "16:34"])( + it.each(["10:00", "15:00", `9:${minute()}`, "16:34", "00:00"])( "should accept values in range (%s)", async time => { const row = { time } @@ -202,6 +202,18 @@ describe("validate", () => { expect(output.valid).toBe(true) } ) + + it.each(["10:01", "14:59:59", `12:${minute()}`])( + "should reject values out range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 10:00"], + }) + } + ) }) }) }) From 43acea931ac4b3846604a40569d125b2ad7155af Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 23 May 2024 15:23:02 +0200 Subject: [PATCH 16/18] Ensure iso time config still work --- .../src/sdk/app/rows/tests/utils.spec.ts | 90 +++++++++++++++++++ packages/server/src/sdk/app/rows/utils.ts | 21 ++++- 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 4c44791135..55cdf9ea20 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -1,3 +1,4 @@ +import dayjs from "dayjs" import { FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -241,5 +242,94 @@ describe("validate", () => { expect(output.errors).toEqual({ time: ["can't be blank"] }) }) }) + + describe("range", () => { + const table = getTable() + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: "10:00", + latest: "15:00", + }, + } + + it.each(["10:00", "15:00", `10:${minute()}`, "12:34"])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "9:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:00"], + }) + }) + + describe("datetime ISO configs", () => { + const table = getTable() + + table.schema.time.constraints = { + presence: true, + datetime: { + earliest: dayjs().hour(10).minute(0).second(0).toISOString(), + latest: dayjs().hour(15).minute(0).second(0).toISOString(), + }, + } + + it.each(["10:00", "15:00", `12:${minute()}`])( + "should accept values in range (%s)", + async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(true) + } + ) + + it.each([ + "09:59:50", + `${generator.integer({ min: 0, max: 9 })}:${minute()}`, + ])("should reject values before range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no earlier than 10:00"], + }) + }) + + it.each([ + "15:00:01", + `${generator.integer({ min: 16, max: 23 })}:${minute()}`, + ])("should reject values after range (%s)", async time => { + const row = { time } + const output = await validate({ table, tableId: table._id!, row }) + expect(output.valid).toBe(false) + expect(output.errors).toEqual({ + time: ["must be no later than 15:00"], + }) + }) + }) + }) }) }) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 19e8869494..23e1ab3e6b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -242,11 +242,24 @@ function validateTimeOnlyField( let castedConstraints = cloneDeep(constraints) let earliest, latest + let easliestTimeString: string, latestTimeString: string if (castedConstraints.datetime?.earliest) { - earliest = stringTimeToDate(castedConstraints.datetime?.earliest) + easliestTimeString = castedConstraints.datetime.earliest + if (dayjs(castedConstraints.datetime.earliest).isValid()) { + easliestTimeString = dayjs(castedConstraints.datetime.earliest).format( + "HH:mm" + ) + } + earliest = stringTimeToDate(easliestTimeString) } if (castedConstraints.datetime?.latest) { - latest = stringTimeToDate(castedConstraints.datetime?.latest) + latestTimeString = castedConstraints.datetime.latest + if (dayjs(castedConstraints.datetime.latest).isValid()) { + latestTimeString = dayjs(castedConstraints.datetime.latest).format( + "HH:mm" + ) + } + latest = stringTimeToDate(latestTimeString) } if (earliest && latest && earliest.isAfter(latest)) { @@ -271,11 +284,11 @@ function validateTimeOnlyField( m ?.replace( castedConstraints.datetime?.earliest || "", - constraints.datetime?.earliest || "" + easliestTimeString || "" ) .replace( castedConstraints.datetime?.latest || "", - constraints.datetime?.latest || "" + latestTimeString || "" ) ) if (jsValidation) { From 7d256d235afe75c179056616d142392a96101637 Mon Sep 17 00:00:00 2001 From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com> Date: Thu, 23 May 2024 15:31:11 +0100 Subject: [PATCH 17/18] Enum columns unexpectedly breaking fix (#13756) * Added enum to SQL_MISC_TYPE_MAP to correctly map to FieldType.OPTIONS * improve enum values extraction for multiple single-select support * Tests to ensure enums are typed correctly MySQL and Postgres * Fixed linting issue * Ran prettier --- .../server/src/integration-test/mysql.spec.ts | 36 +++++++++++++++++++ .../src/integration-test/postgres.spec.ts | 31 ++++++++++++++++ packages/server/src/integrations/postgres.ts | 10 +++--- .../server/src/integrations/utils/utils.ts | 1 + 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index b4eb1035d6..8cf4fb8212 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -281,4 +281,40 @@ describe("mysql integrations", () => { ]) }) }) + + describe("POST /api/datasources/:datasourceId/schema", () => { + let tableName: string + + beforeEach(async () => { + tableName = uniqueTableName() + }) + + afterEach(async () => { + await rawQuery(rawDatasource, `DROP TABLE IF EXISTS \`${tableName}\``) + }) + + it("recognises enum columns as options", async () => { + const enumColumnName = "status" + + const createTableQuery = ` + CREATE TABLE \`${tableName}\` ( + \`order_id\` INT AUTO_INCREMENT PRIMARY KEY, + \`customer_name\` VARCHAR(100) NOT NULL, + \`${enumColumnName}\` ENUM('pending', 'processing', 'shipped', 'delivered', 'cancelled') + ); + ` + + await rawQuery(rawDatasource, createTableQuery) + + const response = await makeRequest( + "post", + `/api/datasources/${datasource._id}/schema` + ) + + const table = response.body.datasource.entities[tableName] + + expect(table).toBeDefined() + expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) + }) + }) }) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index ec4cb90a86..ccf63d0820 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1122,6 +1122,37 @@ describe("postgres integrations", () => { [tableName]: "Table contains invalid columns.", }) }) + + it("recognises enum columns as options", async () => { + const tableName = `orders_${generator + .guid() + .replaceAll("-", "") + .substring(0, 6)}` + const enumColumnName = "status" + + await rawQuery( + rawDatasource, + ` + CREATE TYPE order_status AS ENUM ('pending', 'processing', 'shipped', 'delivered', 'cancelled'); + + CREATE TABLE ${tableName} ( + order_id SERIAL PRIMARY KEY, + customer_name VARCHAR(100) NOT NULL, + ${enumColumnName} order_status + ); + ` + ) + + const response = await makeRequest( + "post", + `/api/datasources/${datasource._id}/schema` + ) + + const table = response.body.datasource.entities[tableName] + + expect(table).toBeDefined() + expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) + }) }) describe("Integration compatibility with postgres search_path", () => { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index e810986757..584f0ef15d 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -329,14 +329,12 @@ class PostgresIntegration extends Sql implements DatasourcePlus { // Fetch enum values const enumsResponse = await this.client.query(this.ENUM_VALUES()) + // output array, allows for more than 1 single-select to be used at a time const enumValues = enumsResponse.rows?.reduce((acc, row) => { - if (!acc[row.typname]) { - return { - [row.typname]: [row.enumlabel], - } + return { + ...acc, + [row.typname]: [...(acc[row.typname] || []), row.enumlabel], } - acc[row.typname].push(row.enumlabel) - return acc }, {}) for (let column of columnsResponse.rows) { diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 8f3aba8907..5bc90bc295 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -102,6 +102,7 @@ const SQL_OPTIONS_TYPE_MAP: Record = { const SQL_MISC_TYPE_MAP: Record = { json: FieldType.JSON, bigint: FieldType.BIGINT, + enum: FieldType.OPTIONS, } const SQL_TYPE_MAP: Record = { From 1004b052baae179513d32dfab57e1a3a64ea1ed3 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 23 May 2024 14:39:52 +0000 Subject: [PATCH 18/18] Bump version to 2.27.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a943000cea..db0a1d59fa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.27.2", + "version": "2.27.3", "npmClient": "yarn", "packages": [ "packages/*",