From 253fa0def8ce95e11cb38edb613c5270e09d7704 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 16:20:44 +0100 Subject: [PATCH 01/40] In progress: bigint tests. --- .../src/api/routes/tests/search.spec.ts | 97 ++++++++++++++++++- .../src/sdk/app/tables/external/index.ts | 6 ++ .../server/src/sdk/app/tables/internal/sqs.ts | 2 +- 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5b71ec9044..39ba0b589d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -18,14 +18,15 @@ import _ from "lodash" jest.unmock("mssql") describe.each([ - ["internal", undefined], + // ["internal", undefined], ["internal-sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" + const isInternal = name === "internal" const config = setup.getConfig() let envCleanup: (() => void) | undefined @@ -550,4 +551,90 @@ describe.each([ ])) }) }) + + describe("bigints", () => { + const SMALL = "1" + const MEDIUM = "10000000" + + // Our bigints are int64s in most datasources. + const BIG = "9223372036854775807" + + beforeAll(async () => { + await createTable({ + num: { name: "num", type: FieldType.BIGINT }, + }) + await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { num: SMALL } }).toContainExactly([ + { num: SMALL }, + ])) + + it("successfully finds a big value", () => + expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { num: "2" } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + { num: MEDIUM }, + { num: BIG }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, + ])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + { num: SMALL }, + ])) + + it("successfully finds all rows", () => + expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { num: [2] } }).toFindNothing()) + }) + + // Range searches against bigints don't seem to work at all in Lucene, and I + // couldn't figure out why. Given that we're replacing Lucene with SQS, + // we've decided not to spend time on it. + !isInternal && + describe("range", () => { + it.only("successfully finds a row", () => + expectQuery({ + range: { num: { low: SMALL, high: "5" } }, + }).toContainExactly([{ num: SMALL }])) + + it("successfully finds multiple rows", () => + expectQuery({ + range: { num: { low: SMALL, high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + + it("successfully finds a row with a high bound", () => + expectQuery({ + range: { num: { low: MEDIUM, high: BIG } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + + it("successfully finds no rows", () => + expectQuery({ + range: { num: { low: "5", high: "5" } }, + }).toFindNothing()) + }) + }) }) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 65cd4a07c1..bc8430c72c 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -52,6 +52,12 @@ export async function save( !oldTable && (tableToSave.primary == null || tableToSave.primary.length === 0) ) { + if (tableToSave.schema.id) { + throw new Error( + "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column." + ) + } + tableToSave.primary = ["id"] tableToSave.schema.id = { type: FieldType.NUMBER, diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 79d9be2348..14313973c1 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -31,7 +31,7 @@ const FieldTypeMap: Record = { [FieldType.ATTACHMENT_SINGLE]: SQLiteType.BLOB, [FieldType.ARRAY]: SQLiteType.BLOB, [FieldType.LINK]: SQLiteType.BLOB, - [FieldType.BIGINT]: SQLiteType.REAL, + [FieldType.BIGINT]: SQLiteType.TEXT, // TODO: consider the difference between multi-user and single user types (subtyping) [FieldType.BB_REFERENCE]: SQLiteType.TEXT, } From 03b18234635445da730dc3f01bfbf7bb95c3de3b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:28:21 +0100 Subject: [PATCH 02/40] Range tests passing. --- .../src/api/routes/tests/search.spec.ts | 50 ++++++++++++++-- packages/server/src/integrations/base/sql.ts | 57 +++++++++++++++---- .../server/src/sdk/app/rows/search/sqs.ts | 2 +- packages/types/src/sdk/search.ts | 11 ++-- 4 files changed, 97 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 39ba0b589d..698ea0c10b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -18,12 +18,12 @@ import _ from "lodash" jest.unmock("mssql") describe.each([ - // ["internal", undefined], + ["internal", undefined], ["internal-sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" const isInternal = name === "internal" @@ -337,6 +337,20 @@ describe.each([ expectQuery({ range: { age: { low: 5, high: 9 } }, }).toFindNothing()) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a low value", () => + expectQuery({ + range: { age: { low: 5 } }, + }).toContainExactly([{ age: 10 }])) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a high value", () => + expectQuery({ + range: { age: { high: 5 } }, + }).toContainExactly([{ age: 1 }])) }) describe("sort", () => { @@ -441,6 +455,20 @@ describe.each([ expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_9TH } }, }).toFindNothing()) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a low value", () => + expectQuery({ + range: { dob: { low: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_10TH }])) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a high value", () => + expectQuery({ + range: { dob: { high: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_1ST }])) }) describe("sort", () => { @@ -616,7 +644,7 @@ describe.each([ // we've decided not to spend time on it. !isInternal && describe("range", () => { - it.only("successfully finds a row", () => + it("successfully finds a row", () => expectQuery({ range: { num: { low: SMALL, high: "5" } }, }).toContainExactly([{ num: SMALL }])) @@ -635,6 +663,16 @@ describe.each([ expectQuery({ range: { num: { low: "5", high: "5" } }, }).toFindNothing()) + + it("can search using just a low value", () => + expectQuery({ + range: { num: { low: MEDIUM } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + + it("can search using just a high value", () => + expectQuery({ + range: { num: { high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) }) }) }) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 259abec106..a3454d7a56 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -146,7 +146,7 @@ class InternalBuilder { addFilters( query: Knex.QueryBuilder, filters: SearchFilters | undefined, - tableName: string, + table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { function getTableName(name: string) { @@ -161,7 +161,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableName(tableName)}.${updatedKey}`, value) + fn(`${getTableName(table.name)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") @@ -276,6 +276,9 @@ class InternalBuilder { } if (filters.range) { iterate(filters.range, (key, value) => { + const fieldName = key.split(".")[1] + const field = table.schema[fieldName] + const isEmptyObject = (val: any) => { return ( val && @@ -293,16 +296,46 @@ class InternalBuilder { highValid = isValidFilter(value.high) if (lowValid && highValid) { // Use a between operator if we have 2 valid range values - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) + if ( + field.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [value.low, value.high] + ) + } else { + const fnc = allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [value.low, value.high]) + } } else if (lowValid) { // Use just a single greater than operator if we only have a low - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">", value.low) + if ( + field.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, + [value.low] + ) + } else { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", value.low) + } } else if (highValid) { // Use just a single less than operator if we only have a high - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<", value.high) + if ( + field.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, + [value.high] + ) + } else { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", value.high) + } } }) } @@ -532,7 +565,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(query, filters, tableName, { + query = this.addFilters(query, filters, json.meta?.table!, { aliases: tableAliases, }) // add sorting to pre-query @@ -553,7 +586,7 @@ class InternalBuilder { endpoint.schema, tableAliases ) - return this.addFilters(query, filters, tableName, { + return this.addFilters(query, filters, json.meta?.table!, { relationship: true, aliases: tableAliases, }) @@ -563,7 +596,7 @@ class InternalBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) - query = this.addFilters(query, filters, endpoint.entityId, { + query = this.addFilters(query, filters, json.meta?.table!, { aliases: tableAliases, }) // mysql can't use returning @@ -577,7 +610,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, endpoint.entityId, { + query = this.addFilters(query, filters, json.meta?.table!, { aliases: tableAliases, }) // mysql can't use returning diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 7abd7d9e72..4517739d26 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -185,6 +185,6 @@ export async function search( } } catch (err: any) { const msg = typeof err === "string" ? err : err.message - throw new Error(`Unable to search by SQL - ${msg}`) + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 9325f09eed..288618647f 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -13,10 +13,13 @@ export interface SearchFilters { [key: string]: string } range?: { - [key: string]: { - high: number | string - low: number | string - } + [key: string]: + | { + high: number | string + low: number | string + } + | { high: number | string } + | { low: number | string } } equal?: { [key: string]: any From b01b260e396c5597c4cbe201bd3d92ec71124558 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:36:51 +0100 Subject: [PATCH 03/40] Fix some of the broken tests. --- packages/server/src/integrations/base/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 65709d8a82..e157590c5e 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -175,7 +175,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableName(table)}.${updatedKey}`, value) + fn(`${getTableAlias(table.name)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") From 38dc7ae39142e718aebdb767c1120ee8f49abc8e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:39:05 +0100 Subject: [PATCH 04/40] Remove extraneous ? and ! operators. --- packages/server/src/integrations/base/sql.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index e157590c5e..4b85f1cea2 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -392,7 +392,7 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort, paginate } = json - const table = json.meta?.table + const table = json.meta.table const tableName = getTableName(table) const aliases = json.tableAliases const aliased = @@ -580,7 +580,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(query, filters, json.meta?.table!, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // add sorting to pre-query @@ -601,7 +601,7 @@ class InternalBuilder { endpoint.schema, tableAliases ) - return this.addFilters(query, filters, json.meta?.table!, { + return this.addFilters(query, filters, json.meta.table, { relationship: true, aliases: tableAliases, }) @@ -611,7 +611,7 @@ class InternalBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) - query = this.addFilters(query, filters, json.meta?.table!, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // mysql can't use returning @@ -625,7 +625,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, json.meta?.table!, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // mysql can't use returning @@ -717,7 +717,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // when creating if an ID has been inserted need to make sure // the id filter is enriched with it before trying to retrieve the row checkLookupKeys(id: any, json: QueryJson) { - if (!id || !json.meta?.table || !json.meta.table.primary) { + if (!id || !json.meta.table || !json.meta.table.primary) { return json } const primaryKey = json.meta.table.primary?.[0] From affa546159c48ddf3e7f7ff309fff5aadef9813d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 16 Apr 2024 17:45:06 +0100 Subject: [PATCH 05/40] Fix tests. --- packages/server/src/integrations/base/sql.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 4b85f1cea2..7ee22b5933 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -163,6 +163,9 @@ class InternalBuilder { table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { + const tableName = + this.client === SqlClient.SQL_LITE ? table._id! : table.name + function getTableAlias(name: string) { const alias = opts.aliases?.[name] return alias || name @@ -175,7 +178,7 @@ class InternalBuilder { const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { - fn(`${getTableAlias(table.name)}.${updatedKey}`, value) + fn(`${getTableAlias(tableName)}.${updatedKey}`, value) } if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") From 8161dfc0a909c1ef99590ef6d359d52af23bcf1b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:16:13 +0200 Subject: [PATCH 06/40] Create SWITCHABLE_TYPES in shared-core --- packages/shared-core/src/constants/fields.ts | 33 ++++++++++++++++++++ packages/shared-core/src/constants/index.ts | 1 + 2 files changed, 34 insertions(+) create mode 100644 packages/shared-core/src/constants/fields.ts diff --git a/packages/shared-core/src/constants/fields.ts b/packages/shared-core/src/constants/fields.ts new file mode 100644 index 0000000000..89504de473 --- /dev/null +++ b/packages/shared-core/src/constants/fields.ts @@ -0,0 +1,33 @@ +import { FieldType } from "@budibase/types" + +type SwitchableTypes = Partial<{ + [K in FieldType]: [K, ...FieldType[]] +}> + +export const SWITCHABLE_TYPES: Partial = { + [FieldType.STRING]: [ + FieldType.STRING, + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ], + [FieldType.OPTIONS]: [ + FieldType.OPTIONS, + FieldType.STRING, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ], + [FieldType.LONGFORM]: [ + FieldType.LONGFORM, + FieldType.STRING, + FieldType.OPTIONS, + FieldType.BARCODEQR, + ], + [FieldType.BARCODEQR]: [ + FieldType.BARCODEQR, + FieldType.STRING, + FieldType.OPTIONS, + FieldType.LONGFORM, + ], + [FieldType.NUMBER]: [FieldType.NUMBER, FieldType.BOOLEAN], +} diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 922f0d4387..afb7e659e1 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,4 +1,5 @@ export * from "./api" +export * from "./fields" export const OperatorOptions = { Equals: { From 3e32ce4d242b375190fbed56f60802827e6593fd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:21:37 +0200 Subject: [PATCH 07/40] Use shared-core --- .../DataTable/modals/CreateEditColumn.svelte | 22 +++++-------------- .../builder/src/constants/backend/index.js | 20 ----------------- packages/builder/src/stores/builder/tables.js | 4 ++-- packages/shared-core/src/constants/fields.ts | 2 +- 4 files changed, 9 insertions(+), 39 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 92501bec3b..16a26b60c1 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -13,6 +13,7 @@ Layout, AbsTooltip, } from "@budibase/bbui" + import { SWITCHABLE_TYPES } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -20,11 +21,6 @@ import { FIELDS, RelationshipType, - ALLOWABLE_STRING_OPTIONS, - ALLOWABLE_NUMBER_OPTIONS, - ALLOWABLE_STRING_TYPES, - ALLOWABLE_NUMBER_TYPES, - SWITCHABLE_TYPES, PrettyRelationshipDefinitions, DB_TYPE_EXTERNAL, } from "constants/backend" @@ -175,7 +171,7 @@ $: typeEnabled = !originalName || (originalName && - SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && + SWITCHABLE_TYPES[editableColumn.type] && !editableColumn?.autocolumn) const fieldDefinitions = Object.values(FIELDS).reduce( @@ -367,16 +363,10 @@ } function getAllowedTypes() { - if ( - originalName && - ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1 - ) { - return ALLOWABLE_STRING_OPTIONS - } else if ( - originalName && - ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1 - ) { - return ALLOWABLE_NUMBER_OPTIONS + if (originalName) { + return ( + SWITCHABLE_TYPES[editableColumn.type] || [editableColumn.type] + ).map(f => FIELDS[f.toUpperCase()]) } const isUsers = diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ffa6ef36c6..84975d93e2 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -202,26 +202,6 @@ export const PrettyRelationshipDefinitions = { ONE: "One row", } -export const ALLOWABLE_STRING_OPTIONS = [ - FIELDS.STRING, - FIELDS.OPTIONS, - FIELDS.LONGFORM, - FIELDS.BARCODEQR, -] -export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map( - opt => opt.type -) - -export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN] -export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map( - opt => opt.type -) - -export const SWITCHABLE_TYPES = [ - ...ALLOWABLE_STRING_TYPES, - ...ALLOWABLE_NUMBER_TYPES, -] - export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" export const BUDIBASE_DATASOURCE_TYPE = "budibase" diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 0163281480..8a246a8ac2 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -1,8 +1,8 @@ import { FieldType } from "@budibase/types" +import { SWITCHABLE_TYPES } from "@budibase/shared-core" import { get, writable, derived } from "svelte/store" import { cloneDeep } from "lodash/fp" import { API } from "api" -import { SWITCHABLE_TYPES } from "constants/backend" export function createTablesStore() { const store = writable({ @@ -64,7 +64,7 @@ export function createTablesStore() { if ( oldField != null && oldField?.type !== field.type && - SWITCHABLE_TYPES.indexOf(oldField?.type) === -1 + SWITCHABLE_TYPES[oldField?.type] ) { updatedTable.schema[key] = oldField } diff --git a/packages/shared-core/src/constants/fields.ts b/packages/shared-core/src/constants/fields.ts index 89504de473..5acf07d863 100644 --- a/packages/shared-core/src/constants/fields.ts +++ b/packages/shared-core/src/constants/fields.ts @@ -4,7 +4,7 @@ type SwitchableTypes = Partial<{ [K in FieldType]: [K, ...FieldType[]] }> -export const SWITCHABLE_TYPES: Partial = { +export const SWITCHABLE_TYPES: SwitchableTypes = { [FieldType.STRING]: [ FieldType.STRING, FieldType.OPTIONS, From 4aba988ca97682c337f6a3e8576636cf23922602 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:34:06 +0200 Subject: [PATCH 08/40] Keep schema by default --- packages/server/src/integrations/utils.ts | 41 +++++++++++++---------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index d5f6d191e1..f0f96c82a5 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -284,8 +284,8 @@ export function isIsoDateString(str: string) { * @param column The column to check, to see if it is a valid relationship. * @param tableIds The IDs of the tables which currently exist. */ -export function shouldCopyRelationship( - column: { type: string; tableId?: string }, +function shouldCopyRelationship( + column: { type: FieldType.LINK; tableId?: string }, tableIds: string[] ) { return ( @@ -303,28 +303,18 @@ export function shouldCopyRelationship( * @param column The column to check for options or boolean type. * @param fetchedColumn The fetched column to check for the type in the external database. */ -export function shouldCopySpecialColumn( +function shouldCopySpecialColumn( column: { type: string }, fetchedColumn: { type: string } | undefined ) { const isFormula = column.type === FieldType.FORMULA - const specialTypes = [ - FieldType.OPTIONS, - FieldType.LONGFORM, - FieldType.ARRAY, - FieldType.FORMULA, - FieldType.BB_REFERENCE, - ] // column has been deleted, remove - formulas will never exist, always copy if (!isFormula && column && !fetchedColumn) { return false } const fetchedIsNumber = !fetchedColumn || fetchedColumn.type === FieldType.NUMBER - return ( - specialTypes.indexOf(column.type as FieldType) !== -1 || - (fetchedIsNumber && column.type === FieldType.BOOLEAN) - ) + return fetchedIsNumber && column.type === FieldType.BOOLEAN } /** @@ -357,12 +347,29 @@ function copyExistingPropsOver( continue } const column = existingTableSchema[key] + if ( - shouldCopyRelationship(column, tableIds) || - shouldCopySpecialColumn(column, table.schema[key]) + column.type === FieldType.LINK && + !shouldCopyRelationship(column, tableIds) ) { - table.schema[key] = existingTableSchema[key] + continue } + + const specialTypes = [ + FieldType.OPTIONS, + FieldType.LONGFORM, + FieldType.ARRAY, + FieldType.FORMULA, + FieldType.BB_REFERENCE, + ] + if ( + specialTypes.includes(column.type) && + !shouldCopySpecialColumn(column, table.schema[key]) + ) { + continue + } + + table.schema[key] = existingTableSchema[key] } } return table From b34227039900ad430f732dee8dff7f0754019c37 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 14:43:52 +0200 Subject: [PATCH 09/40] Lint --- .../backend/DataTable/modals/CreateEditColumn.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 16a26b60c1..8716a00660 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -13,7 +13,7 @@ Layout, AbsTooltip, } from "@budibase/bbui" - import { SWITCHABLE_TYPES } from "@budibase/shared-core" + import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -29,7 +29,6 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" - import { ValidColumnNameRegex } from "@budibase/shared-core" import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" import { RowUtils } from "@budibase/frontend-core" From 92c307105c2527377ddb8e878bcaa32312411b6b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:20:58 +0200 Subject: [PATCH 10/40] Skip test --- packages/server/src/integration-test/mysql.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 7e54b53b15..fc9bb1aafa 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -235,7 +235,8 @@ describe("mysql integrations", () => { describe("POST /api/tables/", () => { const emitDatasourceUpdateMock = jest.fn() - it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { + // TODO: This is not actually required, will fix after cleaning the `_add` logic + xit("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { const addColumnToTable: TableRequest = { type: "table", sourceType: TableSourceType.EXTERNAL, From a6132c28042681bac22198ff6fc4900804ec6d68 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:27:05 +0200 Subject: [PATCH 11/40] Lint --- packages/server/src/integration-test/mysql.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index fc9bb1aafa..59340eb202 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -236,7 +236,7 @@ describe("mysql integrations", () => { const emitDatasourceUpdateMock = jest.fn() // TODO: This is not actually required, will fix after cleaning the `_add` logic - xit("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { + it.skip("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { const addColumnToTable: TableRequest = { type: "table", sourceType: TableSourceType.EXTERNAL, From 764235469cecfec95b0543dd34b318b0e5fd38e5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:32:45 +0200 Subject: [PATCH 12/40] Lint --- packages/server/src/integration-test/mysql.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 59340eb202..964bc8fce7 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -236,6 +236,7 @@ describe("mysql integrations", () => { const emitDatasourceUpdateMock = jest.fn() // TODO: This is not actually required, will fix after cleaning the `_add` logic + // eslint-disable-next-line jest/no-disabled-tests it.skip("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { const addColumnToTable: TableRequest = { type: "table", From e1c092d65d498769f8e368f6e84fe4bde4f32417 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:36:52 +0200 Subject: [PATCH 13/40] Refetch when type changed --- packages/server/src/integrations/utils.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index f0f96c82a5..165fffc652 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -7,7 +7,7 @@ import { } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" -import { helpers } from "@budibase/shared-core" +import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core" import env from "../environment" import { Knex } from "knex" @@ -348,6 +348,16 @@ function copyExistingPropsOver( } const column = existingTableSchema[key] + // If the db column type changed to a non-compatible one, we want to re-fetch it + if ( + table.schema[key].type !== existingTableSchema[key].type && + !SWITCHABLE_TYPES[existingTableSchema[key].type]?.includes( + table.schema[key].type + ) + ) { + continue + } + if ( column.type === FieldType.LINK && !shouldCopyRelationship(column, tableIds) From 8afb0e3c91d4d029b2b53f7e69c481d6a96a3e66 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 16:50:19 +0200 Subject: [PATCH 14/40] Fix tests --- packages/server/src/integrations/utils.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 165fffc652..b8bd1db897 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -304,8 +304,8 @@ function shouldCopyRelationship( * @param fetchedColumn The fetched column to check for the type in the external database. */ function shouldCopySpecialColumn( - column: { type: string }, - fetchedColumn: { type: string } | undefined + column: { type: FieldType }, + fetchedColumn: { type: FieldType } | undefined ) { const isFormula = column.type === FieldType.FORMULA // column has been deleted, remove - formulas will never exist, always copy @@ -348,12 +348,13 @@ function copyExistingPropsOver( } const column = existingTableSchema[key] + const existingColumnType = column?.type + const updatedColumnType = table.schema[key]?.type + // If the db column type changed to a non-compatible one, we want to re-fetch it if ( - table.schema[key].type !== existingTableSchema[key].type && - !SWITCHABLE_TYPES[existingTableSchema[key].type]?.includes( - table.schema[key].type - ) + updatedColumnType !== existingColumnType && + !SWITCHABLE_TYPES[existingColumnType]?.includes(updatedColumnType) ) { continue } From 54f2c825fc7d2cf69ce20b79f76087782e266440 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 12:03:46 +0200 Subject: [PATCH 15/40] Remove _add --- packages/builder/src/stores/builder/tables.js | 6 ------ .../server/src/api/controllers/table/external.ts | 3 +-- packages/server/src/api/controllers/table/index.ts | 1 - packages/server/src/sdk/app/tables/external/index.ts | 12 ++---------- packages/types/src/documents/app/table/table.ts | 3 +-- packages/types/src/sdk/search.ts | 4 ---- 6 files changed, 4 insertions(+), 25 deletions(-) diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 8a246a8ac2..c2aab5634b 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -148,12 +148,6 @@ export function createTablesStore() { if (indexes) { draft.indexes = indexes } - // Add object to indicate if column is being added - if (draft.schema[field.name] === undefined) { - draft._add = { - name: field.name, - } - } draft.schema = { ...draft.schema, [field.name]: cloneDeep(field), diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 7c036bec9d..e526af4ecb 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -31,7 +31,6 @@ export async function save( renaming?: RenameColumn ) { const inputs = ctx.request.body - const adding = inputs?._add // can't do this right now delete inputs.rows const tableId = ctx.request.body._id @@ -44,7 +43,7 @@ export async function save( const { datasource, table } = await sdk.tables.external.save( datasourceId!, inputs, - { tableId, renaming, adding } + { tableId, renaming } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) return table diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index f799113333..cae3a3fe11 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -79,7 +79,6 @@ export async function save(ctx: UserCtx) { const api = pickApi({ table }) // do not pass _rename or _add if saving to CouchDB if (api === internal) { - delete ctx.request.body._add delete ctx.request.body._rename } let savedTable = await api.save(ctx, renaming) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 65cd4a07c1..f3f2f070be 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -3,7 +3,6 @@ import { Operation, RelationshipType, RenameColumn, - AddColumn, Table, TableRequest, ViewV2, @@ -33,7 +32,7 @@ import * as viewSdk from "../../views" export async function save( datasourceId: string, update: Table, - opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } + opts?: { tableId?: string; renaming?: RenameColumn } ) { let tableToSave: TableRequest = { ...update, @@ -179,14 +178,7 @@ export async function save( // remove the rename prop delete tableToSave._rename - // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column - if (opts?.adding) { - datasource.entities[tableToSave.name] = ( - await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name]) - ).tables[tableToSave.name] - } else { - datasource.entities[tableToSave.name] = tableToSave - } + datasource.entities[tableToSave.name] = tableToSave // store it into couch now for budibase reference await db.put(populateExternalTableSchemas(datasource)) diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index b284e9a840..f0e6079aef 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -1,6 +1,6 @@ import { Document } from "../../document" import { View, ViewV2 } from "../view" -import { AddColumn, RenameColumn } from "../../../sdk" +import { RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" @@ -30,6 +30,5 @@ export interface Table extends Document { export interface TableRequest extends Table { _rename?: RenameColumn - _add?: AddColumn created?: boolean } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 0b93fb9215..b9aedede09 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -61,10 +61,6 @@ export interface RenameColumn { updated: string } -export interface AddColumn { - name: string -} - export interface RelationshipsJson { through?: string from?: string From 88fc133f45464556f8c971b7a56e44193a457d12 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 12:04:33 +0200 Subject: [PATCH 16/40] Move responsability --- packages/server/src/api/controllers/table/index.ts | 4 ---- packages/server/src/api/controllers/table/internal.ts | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index cae3a3fe11..63ce00c5ef 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -77,10 +77,6 @@ export async function save(ctx: UserCtx) { const renaming = ctx.request.body._rename const api = pickApi({ table }) - // do not pass _rename or _add if saving to CouchDB - if (api === internal) { - delete ctx.request.body._rename - } let savedTable = await api.save(ctx, renaming) if (!table._id) { savedTable = sdk.tables.enrichViewSchemas(savedTable) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index eb5e4b6c41..a06cc4dee3 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -16,7 +16,7 @@ export async function save( ctx: UserCtx, renaming?: RenameColumn ) { - const { rows, ...rest } = ctx.request.body + const { _rename, rows, ...rest } = ctx.request.body let tableToSave: Table = { _id: generateTableID(), ...rest, From 33c40a897a2297be1d513633bb419b82963a8f17 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 12:38:46 +0200 Subject: [PATCH 17/40] Fix test --- packages/server/src/api/routes/tests/table.spec.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 7639b840dc..77e05b8e07 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -219,9 +219,6 @@ describe.each([ it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { - _add: { - name: "NEW_COLUMN", - }, ...basicTable(), } @@ -235,7 +232,6 @@ describe.each([ updatedAt: expect.stringMatching(ISO_REGEX_PATTERN), views: {}, } - delete expectedResponse._add expect(response).toEqual(expectedResponse) }) }) From de47f44959bf9705197fcc1983e0be7c059b46b7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 13:04:16 +0200 Subject: [PATCH 18/40] Remove _add references --- packages/server/src/integration-test/mysql.spec.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 964bc8fce7..04a7b548a6 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -255,9 +255,6 @@ describe("mysql integrations", () => { name: "new_column", }, }, - _add: { - name: "new_column", - }, } jest @@ -273,25 +270,18 @@ describe("mysql integrations", () => { type: FieldType.NUMBER, name: "id", autocolumn: true, - constraints: { - presence: false, - }, externalType: "int unsigned", }, new_column: { type: FieldType.NUMBER, name: "new_column", autocolumn: false, - constraints: { - presence: false, - }, externalType: "float(8,2)", }, }, created: true, _id: `${datasource._id}__${addColumnToTable.name}`, } - delete expectedTable._add expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1) const emittedDatasource: Datasource = From da399970abcc94917ffa5e6d02e1965090b8b7db Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 16 Apr 2024 17:02:58 +0200 Subject: [PATCH 19/40] Lint --- .eslintrc.json | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 525072dc6c..624c2b8f26 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -42,7 +42,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-budibase-imports": "error" } }, @@ -59,7 +67,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-test-com": "error", "local-rules/email-domain-example-com": "error", "no-console": "warn", @@ -89,7 +105,8 @@ { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", - "destructuredArrayIgnorePattern": "^_" + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true } ], "import/no-relative-packages": "error", From 2da0daafe51239d56ace0fa169d0550685e366ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 10:15:07 +0200 Subject: [PATCH 20/40] Clean --- packages/server/src/sdk/app/datasources/datasources.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 336a94636b..84e1601152 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -348,8 +348,7 @@ const preSaveAction: Partial> = { * Make sure all datasource entities have a display name selected */ export function setDefaultDisplayColumns(datasource: Datasource) { - // - for (let entity of Object.values(datasource.entities || {})) { + for (const entity of Object.values(datasource.entities || {})) { if (entity.primaryDisplay) { continue } From 21898afb29df6d5cec7e652cdfd0dfb76186507e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 10:15:42 +0200 Subject: [PATCH 21/40] Persist externalType changes --- packages/server/src/integrations/utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b8bd1db897..39f2bd3cb0 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -380,7 +380,12 @@ function copyExistingPropsOver( continue } - table.schema[key] = existingTableSchema[key] + table.schema[key] = { + ...existingTableSchema[key], + externalType: + existingTableSchema[key].externalType || + table.schema[key].externalType, + } } } return table From 828d78f2afa63fc1a5932c8ce733bd9d835bad76 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 14:47:56 +0100 Subject: [PATCH 22/40] Fixing build after recent type updates. --- .../src/integrations/tests/sqlAlias.spec.ts | 3 ++- packages/shared-core/src/filters.ts | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index f4edab8dad..fda2a091fa 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -117,7 +117,8 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] - const rangeValue = Object.values(filters?.range!)[0] + const rangeValue: { high?: string | number; low?: string | number } = + Object.values(filters?.range!)[0] const equalValue = Object.values(filters?.equal!)[0] expect(query).toEqual({ diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6010f064bf..0554e0c1e4 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -218,14 +218,16 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", } } - if ((operator as any) === "rangeLow" && value != null && value !== "") { - query.range[field].low = value - } else if ( - (operator as any) === "rangeHigh" && - value != null && - value !== "" - ) { - query.range[field].high = value + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } } } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { From b3ff97df7b32567adb1fc2f0491a9325f1241b0c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:13:28 +0200 Subject: [PATCH 23/40] Fix typo --- .../backend/DataTable/modals/CreateEditColumn.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 8716a00660..3034b40ae6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -56,8 +56,8 @@ let primaryDisplay let indexes = [...($tables.selected.indexes || [])] let isCreating = undefined - let relationshipPart1 = PrettyRelationshipDefinitions.Many - let relationshipPart2 = PrettyRelationshipDefinitions.One + let relationshipPart1 = PrettyRelationshipDefinitions.MANY + let relationshipPart2 = PrettyRelationshipDefinitions.ONE let relationshipTableIdPrimary = null let relationshipTableIdSecondary = null let table = $tables.selected From f7a1b4cb12185c3ac8553ac68ef2714971be635e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:13:46 +0200 Subject: [PATCH 24/40] Fix switching types --- .../backend/DataTable/modals/CreateEditColumn.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 3034b40ae6..e56f28dca6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -170,7 +170,7 @@ $: typeEnabled = !originalName || (originalName && - SWITCHABLE_TYPES[editableColumn.type] && + SWITCHABLE_TYPES[field.type] && !editableColumn?.autocolumn) const fieldDefinitions = Object.values(FIELDS).reduce( @@ -363,9 +363,9 @@ function getAllowedTypes() { if (originalName) { - return ( - SWITCHABLE_TYPES[editableColumn.type] || [editableColumn.type] - ).map(f => FIELDS[f.toUpperCase()]) + return (SWITCHABLE_TYPES[field.type] || [editableColumn.type]).map( + f => FIELDS[f.toUpperCase()] + ) } const isUsers = From ae137ca67711050eddbd86a17c0eb2439a8311ca Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:34:08 +0200 Subject: [PATCH 25/40] Fix update --- packages/builder/src/stores/builder/tables.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index c2aab5634b..88b26929ad 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -64,7 +64,7 @@ export function createTablesStore() { if ( oldField != null && oldField?.type !== field.type && - SWITCHABLE_TYPES[oldField?.type] + !SWITCHABLE_TYPES[oldField?.type]?.includes(field.type) ) { updatedTable.schema[key] = oldField } From 7c3c82013c0b27e4ecd3d08e5b81e72ea2fa2df1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 16:43:13 +0200 Subject: [PATCH 26/40] Sort types --- .../backend/DataTable/modals/CreateEditColumn.svelte | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e56f28dca6..ff9830c92a 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -363,9 +363,14 @@ function getAllowedTypes() { if (originalName) { - return (SWITCHABLE_TYPES[field.type] || [editableColumn.type]).map( - f => FIELDS[f.toUpperCase()] - ) + const possibleTypes = ( + SWITCHABLE_TYPES[field.type] || [editableColumn.type] + ).map(t => t.toLowerCase()) + return Object.entries(FIELDS) + .filter(([fieldType]) => + possibleTypes.includes(fieldType.toLowerCase()) + ) + .map(([_, fieldDefinition]) => fieldDefinition) } const isUsers = From 1bc7072a720bd44001353033f75e419a873bda5a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 17:18:57 +0200 Subject: [PATCH 27/40] Fix type swap on refetch schema --- packages/server/src/integrations/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 39f2bd3cb0..cc832d8062 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -354,7 +354,7 @@ function copyExistingPropsOver( // If the db column type changed to a non-compatible one, we want to re-fetch it if ( updatedColumnType !== existingColumnType && - !SWITCHABLE_TYPES[existingColumnType]?.includes(updatedColumnType) + !SWITCHABLE_TYPES[updatedColumnType]?.includes(existingColumnType) ) { continue } From a33c2599b51b793d13ba25a614a94d18b5d07898 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 16:25:27 +0100 Subject: [PATCH 28/40] Adding error to catch scenario that caused tests to fail - fixing cases of table metadata not aligning with entityId --- .../api/controllers/row/ExternalRequest.ts | 28 +++++++++++++------ .../server/src/integrations/base/query.ts | 12 ++++++++ packages/types/src/sdk/datasources.ts | 8 ++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 3dd3f9b8e7..132028b86f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -374,38 +374,44 @@ export class ExternalRequest { ) { continue } - let tableId: string | undefined, + let relatedTableId: string | undefined, lookupField: string | undefined, fieldName: string | undefined if (isManyToMany(field)) { - tableId = field.through + relatedTableId = field.through lookupField = primaryKey fieldName = field.throughTo || primaryKey } else if (isManyToOne(field)) { - tableId = field.tableId + relatedTableId = field.tableId lookupField = field.foreignKey fieldName = field.fieldName } - if (!tableId || !lookupField || !fieldName) { + if (!relatedTableId || !lookupField || !fieldName) { throw new Error( "Unable to lookup relationships - undefined column properties." ) } - const { tableName: relatedTableName } = breakExternalTableId(tableId) + const { tableName: relatedTableName } = + breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] if (!lookupField || !row[lookupField]) { continue } + const endpoint = getEndpoint(relatedTableId, Operation.READ) + const relatedTable = this.tables[endpoint.entityId] + if (!relatedTable) { + throw new Error("unable to find related table") + } const response = await getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.READ), + endpoint: endpoint, filters: { equal: { [fieldName]: row[lookupField], }, }, meta: { - table, + table: relatedTable, }, }) // this is the response from knex if no rows found @@ -414,7 +420,11 @@ export class ExternalRequest { const storeTo = isManyToMany(field) ? field.throughFrom || linkPrimaryKey : fieldName - related[storeTo] = { rows, isMany: isManyToMany(field), tableId } + related[storeTo] = { + rows, + isMany: isManyToMany(field), + tableId: relatedTableId, + } } return related } @@ -484,7 +494,7 @@ export class ExternalRequest { body, filters: buildFilters(id, {}, linkTable), meta: { - table, + table: linkTable, }, }) ) diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 03e6028e32..371592bece 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -2,6 +2,7 @@ import { QueryJson, Datasource, DatasourcePlusQueryResponse, + RowOperations, } from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" @@ -10,6 +11,17 @@ export async function makeExternalQuery( datasource: Datasource, json: QueryJson ): Promise { + const entityId = json.endpoint.entityId, + tableName = json.meta.table.name, + tableId = json.meta.table._id + // case found during testing - make sure this doesn't happen again + if ( + RowOperations.includes(json.endpoint.operation) && + entityId !== tableId && + entityId !== tableName + ) { + throw new Error("Entity ID and table metadata do not align") + } datasource = await sdk.datasources.enrich(datasource) const Integration = await getIntegration(datasource.source) // query is the opinionated function diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index e1a012d81e..77e4877dfa 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -14,6 +14,14 @@ export enum Operation { DELETE_TABLE = "DELETE_TABLE", } +export const RowOperations = [ + Operation.CREATE, + Operation.READ, + Operation.UPDATE, + Operation.DELETE, + Operation.BULK_CREATE, +] + export enum SortDirection { ASCENDING = "ASCENDING", DESCENDING = "DESCENDING", From 30077418eb267a66a2ff5d8c7594dec88ff73219 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 16:31:45 +0100 Subject: [PATCH 29/40] Fixing linting issue. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 132028b86f..be6ac885df 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -447,7 +447,6 @@ export class ExternalRequest { // if we're creating (in a through table) need to wipe the existing ones first const promises = [] const related = await this.lookupRelations(mainTableId, row) - const table = this.getTable(mainTableId)! for (let relationship of relationships) { const { key, tableId, isUpdate, id, ...rest } = relationship const body: { [key: string]: any } = processObjectSync(rest, row, {}) From 60ed4d8443938ee6a72c8205413105db0ea74b14 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 17:12:26 +0100 Subject: [PATCH 30/40] Updating how between/less than/greater than are handled for sqlite. --- packages/server/src/integrations/base/sql.ts | 59 +++----------- .../server/src/integrations/tests/sql.spec.ts | 24 ++++-- .../server/src/integrations/utils/index.ts | 2 + .../src/integrations/utils/sqlStatements.ts | 80 +++++++++++++++++++ .../src/integrations/{ => utils}/utils.ts | 6 +- 5 files changed, 112 insertions(+), 59 deletions(-) create mode 100644 packages/server/src/integrations/utils/index.ts create mode 100644 packages/server/src/integrations/utils/sqlStatements.ts rename packages/server/src/integrations/{ => utils}/utils.ts (98%) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 7ee22b5933..4c975cbef7 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -6,6 +6,7 @@ import { SqlClient, isValidFilter, getNativeSql, + SqlStatements, } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import { @@ -163,6 +164,13 @@ class InternalBuilder { table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { + if (!filters) { + return query + } + filters = parseFilters(filters) + // if all or specified in filters, then everything is an or + const allOr = filters.allOr + const sqlStatements = new SqlStatements(this.client, table, { allOr }) const tableName = this.client === SqlClient.SQL_LITE ? table._id! : table.name @@ -261,12 +269,6 @@ class InternalBuilder { } } - if (!filters) { - return query - } - filters = parseFilters(filters) - // if all or specified in filters, then everything is an or - const allOr = filters.allOr if (filters.oneOf) { iterate(filters.oneOf, (key, array) => { const fnc = allOr ? "orWhereIn" : "whereIn" @@ -293,9 +295,6 @@ class InternalBuilder { } if (filters.range) { iterate(filters.range, (key, value) => { - const fieldName = key.split(".")[1] - const field = table.schema[fieldName] - const isEmptyObject = (val: any) => { return ( val && @@ -312,47 +311,11 @@ class InternalBuilder { const lowValid = isValidFilter(value.low), highValid = isValidFilter(value.high) if (lowValid && highValid) { - // Use a between operator if we have 2 valid range values - if ( - field.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [value.low, value.high] - ) - } else { - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) - } + query = sqlStatements.between(query, key, value.low, value.high) } else if (lowValid) { - // Use just a single greater than operator if we only have a low - if ( - field.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, - [value.low] - ) - } else { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", value.low) - } + query = sqlStatements.lower(query, key, value.low) } else if (highValid) { - // Use just a single less than operator if we only have a high - if ( - field.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, - [value.high] - ) - } else { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", value.high) - } + query = sqlStatements.higher(query, key, value.high) } }) } diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 4ee544cc5e..5de9cc4fbc 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -1,11 +1,11 @@ import { SqlClient } from "../utils" import Sql from "../base/sql" import { + FieldType, Operation, QueryJson, - TableSourceType, Table, - FieldType, + TableSourceType, } from "@budibase/types" const TABLE_NAME = "test" @@ -13,7 +13,12 @@ const TABLE: Table = { type: "table", sourceType: TableSourceType.EXTERNAL, sourceId: "SOURCE_ID", - schema: {}, + schema: { + id: { + name: "id", + type: FieldType.NUMBER, + }, + }, name: TABLE_NAME, primary: ["id"], } @@ -73,7 +78,7 @@ function generateUpdateJson({ meta?: any }): QueryJson { if (!meta.table) { - meta.table = table + meta.table = TABLE } return { endpoint: endpoint(table, "UPDATE"), @@ -158,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) { }, ], extra: { idFilter: {} }, + meta: { + table: TABLE, + }, } } @@ -341,7 +349,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -360,7 +368,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -594,7 +602,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["2000-01-01 00:00:00", 500], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -613,7 +621,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["2010-01-01 00:00:00", 500], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`, }) }) diff --git a/packages/server/src/integrations/utils/index.ts b/packages/server/src/integrations/utils/index.ts new file mode 100644 index 0000000000..a9c2019ba2 --- /dev/null +++ b/packages/server/src/integrations/utils/index.ts @@ -0,0 +1,2 @@ +export * from "./utils" +export { SqlStatements } from "./sqlStatements" diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts new file mode 100644 index 0000000000..b05196f56f --- /dev/null +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -0,0 +1,80 @@ +import { FieldType, Table, FieldSchema } from "@budibase/types" +import { SqlClient } from "./utils" +import { Knex } from "knex" + +export class SqlStatements { + client: string + table: Table + allOr: boolean | undefined + constructor( + client: string, + table: Table, + { allOr }: { allOr?: boolean } = {} + ) { + this.client = client + this.table = table + this.allOr = allOr + } + + getField(key: string): FieldSchema | undefined { + const fieldName = key.split(".")[1] + return this.table.schema[fieldName] + } + + between( + query: Knex.QueryBuilder, + key: string, + low: number | string, + high: number | string + ) { + // Use a between operator if we have 2 valid range values + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [low, high] + ) + } else { + const fnc = this.allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [low, high]) + } + return query + } + + lower(query: Knex.QueryBuilder, key: string, low: number | string) { + // Use just a single greater than operator if we only have a low + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + low, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", low) + } + return query + } + + higher(query: Knex.QueryBuilder, key: string, high: number | string) { + const field = this.getField(key) + // Use just a single less than operator if we only have a high + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + high, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", high) + } + return query + } +} diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils/utils.ts similarity index 98% rename from packages/server/src/integrations/utils.ts rename to packages/server/src/integrations/utils/utils.ts index d5f6d191e1..8f2bed04a2 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -5,10 +5,10 @@ import { FieldType, TableSourceType, } from "@budibase/types" -import { DocumentType, SEPARATOR } from "../db/utils" -import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" +import { DocumentType, SEPARATOR } from "../../db/utils" +import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants" import { helpers } from "@budibase/shared-core" -import env from "../environment" +import env from "../../environment" import { Knex } from "knex" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` From 58d4b2b56e79d2f09b6e96ce4efd6f3cf53d481d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 17:13:08 +0100 Subject: [PATCH 31/40] renaming some sqlstatement generation to lte/gte. --- packages/server/src/integrations/base/sql.ts | 4 ++-- packages/server/src/integrations/utils/sqlStatements.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 4c975cbef7..a8d746add9 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -313,9 +313,9 @@ class InternalBuilder { if (lowValid && highValid) { query = sqlStatements.between(query, key, value.low, value.high) } else if (lowValid) { - query = sqlStatements.lower(query, key, value.low) + query = sqlStatements.lte(query, key, value.low) } else if (highValid) { - query = sqlStatements.higher(query, key, value.high) + query = sqlStatements.gte(query, key, value.high) } }) } diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts index b05196f56f..7a5482830b 100644 --- a/packages/server/src/integrations/utils/sqlStatements.ts +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -44,7 +44,7 @@ export class SqlStatements { return query } - lower(query: Knex.QueryBuilder, key: string, low: number | string) { + lte(query: Knex.QueryBuilder, key: string, low: number | string) { // Use just a single greater than operator if we only have a low const field = this.getField(key) if ( @@ -61,7 +61,7 @@ export class SqlStatements { return query } - higher(query: Knex.QueryBuilder, key: string, high: number | string) { + gte(query: Knex.QueryBuilder, key: string, high: number | string) { const field = this.getField(key) // Use just a single less than operator if we only have a high if ( From e90e2b214e413a832902cfc55b45e5f269f66c9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 17 Apr 2024 17:36:19 +0100 Subject: [PATCH 32/40] Making sure meta.table is always available. --- .../routes/tests/queries/generic-sql.spec.ts | 18 ++++++++++++++++-- packages/server/src/sdk/app/rows/utils.ts | 6 ++++++ .../src/tests/utilities/api/datasource.ts | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 7790f909e7..e8a38dcfaa 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -4,6 +4,7 @@ import { Query, QueryPreview, SourceName, + TableSourceType, } from "@budibase/types" import * as setup from "../utilities" import { @@ -740,12 +741,25 @@ describe.each( }) describe("query through datasource", () => { - it("should be able to query a pg datasource", async () => { + it("should be able to query the datasource", async () => { + const entityId = "test_table" + await config.api.datasource.update({ + ...datasource, + entities: { + [entityId]: { + name: entityId, + schema: {}, + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + }, + }, + }) const res = await config.api.datasource.query({ endpoint: { datasourceId: datasource._id!, operation: Operation.READ, - entityId: "test_table", + entityId, }, resource: { fields: ["id", "name"], diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d307b17947..a9df4f89cd 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -52,6 +52,12 @@ export async function getDatasourceAndQuery( ): Promise { const datasourceId = json.endpoint.datasourceId const datasource = await sdk.datasources.get(datasourceId) + const table = datasource.entities?.[json.endpoint.entityId] + if (!json.meta && table) { + json.meta = { + table, + } + } return makeExternalQuery(datasource, json) } diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 0362a25940..6ac624f0db 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -61,7 +61,7 @@ export class DatasourceAPI extends TestAPI { } query = async ( - query: Omit, + query: Omit & Partial>, expectations?: Expectations ) => { return await this._post(`/api/datasources/query`, { From 17c6b4ab68770a5118229b435f5e80eb564d21cf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 19:51:22 +0200 Subject: [PATCH 33/40] Remove skipped test --- .../server/src/integration-test/mysql.spec.ts | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 04a7b548a6..69093c8668 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -233,64 +233,6 @@ describe("mysql integrations", () => { }) describe("POST /api/tables/", () => { - const emitDatasourceUpdateMock = jest.fn() - - // TODO: This is not actually required, will fix after cleaning the `_add` logic - // eslint-disable-next-line jest/no-disabled-tests - it.skip("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { - const addColumnToTable: TableRequest = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - name: uniqueTableName(), - sourceId: datasource._id!, - primary: ["id"], - schema: { - id: { - type: FieldType.AUTO, - name: "id", - autocolumn: true, - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - }, - }, - } - - jest - .spyOn(builderSocket!, "emitDatasourceUpdate") - .mockImplementation(emitDatasourceUpdateMock) - - await makeRequest("post", "/api/tables/", addColumnToTable) - - const expectedTable: TableRequest = { - ...addColumnToTable, - schema: { - id: { - type: FieldType.NUMBER, - name: "id", - autocolumn: true, - externalType: "int unsigned", - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - autocolumn: false, - externalType: "float(8,2)", - }, - }, - created: true, - _id: `${datasource._id}__${addColumnToTable.name}`, - } - - expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1) - const emittedDatasource: Datasource = - emitDatasourceUpdateMock.mock.calls[0][1] - expect(emittedDatasource.entities![expectedTable.name]).toEqual( - expectedTable - ) - }) - it("will rename a column", async () => { await makeRequest("post", "/api/tables/", primaryMySqlTable) From 5e094dd3bca9225b0263af8c831836642924cd23 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 17 Apr 2024 20:06:54 +0200 Subject: [PATCH 34/40] Lint --- packages/server/src/integration-test/mysql.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 69093c8668..b4eb1035d6 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -16,7 +16,6 @@ import { getDatasource, rawQuery, } from "../integrations/tests/utils" -import { builderSocket } from "../websockets" import { generator } from "@budibase/backend-core/tests" // @ts-ignore fetch.mockSearch() From c36ca27292375c5a85f991aeae3306224d802216 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 12 Apr 2024 13:48:50 +0200 Subject: [PATCH 35/40] Use field types --- .../DataTable/modals/CreateEditColumn.svelte | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ff9830c92a..e8e4180c3d 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -34,15 +34,15 @@ import { RowUtils } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" - const AUTO_TYPE = FIELDS.AUTO.type - const FORMULA_TYPE = FIELDS.FORMULA.type - const LINK_TYPE = FIELDS.LINK.type - const STRING_TYPE = FIELDS.STRING.type - const NUMBER_TYPE = FIELDS.NUMBER.type - const JSON_TYPE = FIELDS.JSON.type - const DATE_TYPE = FIELDS.DATETIME.type - const USER_TYPE = FIELDS.USER.subtype - const USERS_TYPE = FIELDS.USERS.subtype + const AUTO_TYPE = FieldType.AUTO + const FORMULA_TYPE = FieldType.FORMULA + const LINK_TYPE = FieldType.LINK + const STRING_TYPE = FieldType.STRING + const NUMBER_TYPE = FieldType.NUMBER + const JSON_TYPE = FieldType.JSON + const DATE_TYPE = FieldType.DATETIME + const USER_TYPE = FieldSubtype.USER + const USERS_TYPE = FieldSubtype.USERS const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] From e327ebd613ce25736c375ed60b18eccc33a2e521 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:06:34 +0200 Subject: [PATCH 36/40] Do not display relationship selector if autocolumn links --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index e8e4180c3d..d271462f3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -626,7 +626,7 @@ /> - {:else if editableColumn.type === FieldType.LINK} + {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} Date: Thu, 18 Apr 2024 08:26:29 +0000 Subject: [PATCH 37/40] Bump version to 2.23.7 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a2be7be7b4..19f6486f22 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.6", + "version": "2.23.7", "npmClient": "yarn", "packages": [ "packages/*", From e90d40c0be5aa16dd893b9528e6bedb7ecec6cc9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:49:22 +0200 Subject: [PATCH 38/40] Add all scripts as nx cache inputs --- nx.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nx.json b/nx.json index 618395ec90..8ba8798946 100644 --- a/nx.json +++ b/nx.json @@ -9,10 +9,7 @@ }, "targetDefaults": { "build": { - "inputs": [ - "{workspaceRoot}/scripts/build.js", - "{workspaceRoot}/lerna.json" - ] + "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } } } From 8e2fb778233a9b8a3d7373655f80db2cd0968c81 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:53:23 +0200 Subject: [PATCH 39/40] Prevent dev:docker running for oss --- package.json | 2 +- scripts/devDocker.sh | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100755 scripts/devDocker.sh diff --git a/package.json b/package.json index e520b7c2cf..e60a086e17 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "dev:camunda": "./scripts/deploy-camunda.sh", "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "./scripts/devDocker.sh", "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh new file mode 100755 index 0000000000..3d454bafb5 --- /dev/null +++ b/scripts/devDocker.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Check if the pro submodule is loaded +if [ ! -d "./packages/pro/src" ]; then + echo "Submodule is not loaded. This is only allowed with loaded submodules." + exit 1 +fi + +yarn build --scope @budibase/server --scope @budibase/worker +docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 + + From ccbbb2e1ee278a556543c32a4812c02dce06a5f4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 18 Apr 2024 10:53:58 +0200 Subject: [PATCH 40/40] Add message --- scripts/devDocker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh index 3d454bafb5..5e01e5813a 100755 --- a/scripts/devDocker.sh +++ b/scripts/devDocker.sh @@ -2,7 +2,7 @@ # Check if the pro submodule is loaded if [ ! -d "./packages/pro/src" ]; then - echo "Submodule is not loaded. This is only allowed with loaded submodules." + echo "[ERROR] Submodule is not loaded. This is only allowed with loaded submodules." exit 1 fi