From 9733ba5f95eb49c55be6dbbc36020b74d6d0e505 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Aug 2024 18:04:07 +0100 Subject: [PATCH 1/6] Allowing incorrectly setup column schemas to still function as part of search - requires further investigation as to how this happens, but search should still work. --- .../src/api/controllers/row/utils/basic.ts | 3 +- .../src/api/routes/tests/search.spec.ts | 29 +++++++++++++++++++ .../server/src/integrations/utils/utils.ts | 7 +++-- .../src/sdk/app/rows/search/internal/sqs.ts | 5 ++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 883ba5a806..f28f650422 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -71,8 +71,7 @@ export function basicProcessing({ }): Row { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) - for (let field of Object.values(table.schema)) { - const fieldName = field.name + for (let fieldName of Object.keys(table.schema)) { let value = extractFieldValue({ row, tableName: table.name, diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d8c2a4a257..6967f7bd6e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2353,6 +2353,35 @@ describe.each([ }) }) + describe("Invalid column definitions", () => { + beforeAll(async () => { + // need to create an invalid table - means ignoring typescript + table = await createTable({ + // @ts-ignore + invalid: { + type: FieldType.STRING, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + }) + await createRows([ + { name: "foo", invalid: "id1" }, + { name: "bar", invalid: "id2" }, + ]) + }) + + it("can get rows with all table data", async () => { + await expectSearch({ + query: {}, + }).toContain([ + { name: "foo", invalid: "id1" }, + { name: "bar", invalid: "id2" }, + ]) + }) + }) + describe.each(["data_name_test", "name_data_test", "name_test_data_"])( "special (%s) case", column => { diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 84742626c1..6d0e4b838f 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -15,6 +15,7 @@ import { helpers, utils } from "@budibase/shared-core" import { pipeline } from "stream/promises" import tmp from "tmp" import fs from "fs" +import { merge } from "lodash" type PrimitiveTypes = | FieldType.STRING @@ -291,10 +292,12 @@ function copyExistingPropsOver( const fetchedColumnDefinition: FieldSchema | undefined = table.schema[key] table.schema[key] = { - ...existingTableSchema[key], + // merge the properties - anything missing will be filled in, old definition preferred + ...merge(fetchedColumnDefinition, existingTableSchema[key]), + // always take externalType and autocolumn from the fetched definition externalType: existingTableSchema[key].externalType || - table.schema[key]?.externalType, + fetchedColumnDefinition?.externalType, autocolumn: fetchedColumnDefinition?.autocolumn, } as FieldSchema // check constraints which can be fetched from the DB (they could be updated) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 66ec905c61..4bfa8f8fa5 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -73,13 +73,14 @@ function buildInternalFieldList( fieldList = fieldList.concat( PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`) ) - for (let col of Object.values(table.schema)) { + for (let key of Object.keys(table.schema)) { + const col = table.schema[key] const isRelationship = col.type === FieldType.LINK if (!opts?.relationships && isRelationship) { continue } if (!isRelationship) { - fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`) + fieldList.push(`${table._id}.${mapToUserColumn(key)}`) } else { const linkCol = col as RelationshipFieldMetadata const relatedTable = tables.find(table => table._id === linkCol.tableId) From efafb3e3c22b59e4127d49f2befe6ef278857ebe Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 8 Aug 2024 18:55:41 +0100 Subject: [PATCH 2/6] Getting composite keys working, fixing p2 issue and adding test case for it. --- packages/backend-core/src/sql/sqlTable.ts | 21 ++++++++--- .../api/controllers/row/ExternalRequest.ts | 4 +- .../src/api/routes/tests/search.spec.ts | 37 +++++++++++++++++++ packages/server/src/utilities/schema.ts | 3 +- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 02acc8af85..55a6bfcaeb 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -28,16 +28,25 @@ function generateSchema( oldTable: null | Table = null, renamed?: RenameColumn ) { - let primaryKey = table && table.primary ? table.primary[0] : null + let primaryKeys = table && table.primary ? table.primary : [] const columns = Object.values(table.schema) // all columns in a junction table will be meta let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta) let isJunction = metaCols.length === columns.length + let columnTypeSet: string[] = [] + // can't change primary once its set for now - if (primaryKey && !oldTable && !isJunction) { - schema.increments(primaryKey).primary() - } else if (!oldTable && isJunction) { - schema.primary(metaCols.map(col => col.name)) + if (!oldTable) { + // junction tables are special - we have an expected format + if (isJunction) { + schema.primary(metaCols.map(col => col.name)) + } else if (primaryKeys.length === 1) { + schema.increments(primaryKeys[0]).primary() + // note that we've set its type + columnTypeSet.push(primaryKeys[0]) + } else { + schema.primary(primaryKeys) + } } // check if any columns need added @@ -49,7 +58,7 @@ function generateSchema( const oldColumn = oldTable ? oldTable.schema[key] : null if ( (oldColumn && oldColumn.type) || - (primaryKey === key && !isJunction) || + columnTypeSet.includes(key) || renamed?.updated === key ) { continue diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 6538e7347a..14dd909edd 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -1,5 +1,6 @@ import dayjs from "dayjs" import { + ArrayOperator, AutoFieldSubType, AutoReason, Datasource, @@ -196,11 +197,12 @@ export class ExternalRequest { // need to map over the filters and make sure the _id field isn't present let prefix = 1 for (const operator of Object.values(filters)) { + const isArrayOperator = Object.values(ArrayOperator).includes(operator) for (const field of Object.keys(operator || {})) { if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { const parts = breakRowIdField(operator[field]) - if (primary.length > 1) { + if (primary.length > 1 && isArrayOperator) { operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = { id: primary, values: parts[0], diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6967f7bd6e..d0cc8848ae 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -39,6 +39,7 @@ import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" +import { generateRowIdField } from "../../../integrations/utils" describe.each([ ["in-memory", undefined], @@ -2648,6 +2649,42 @@ describe.each([ }) }) + !isInternal && + describe("search by composite key", () => { + beforeAll(async () => { + table = await config.api.table.save( + tableForDatasource(datasource, { + schema: { + idColumn1: { + name: "idColumn1", + type: FieldType.NUMBER, + }, + idColumn2: { + name: "idColumn2", + type: FieldType.NUMBER, + }, + }, + primary: ["idColumn1", "idColumn2"], + }) + ) + await createRows([{ idColumn1: 1, idColumn2: 2 }]) + }) + + it("can filter by the row ID with limit 1", async () => { + await expectSearch({ + query: { + equal: { _id: generateRowIdField([1, 2]) }, + }, + limit: 1, + }).toContain([ + { + idColumn1: 1, + idColumn2: 2, + }, + ]) + }) + }) + isSql && describe("pagination edge case with relationships", () => { let mainRows: Row[] = [] diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index b398285710..b1fbd7577a 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -146,7 +146,8 @@ export function parse(rows: Rows, table: Table): Rows { return rows.map(row => { const parsedRow: Row = {} - Object.entries(row).forEach(([columnName, columnData]) => { + Object.keys(row).forEach(columnName => { + const columnData = row[columnName] const schema = table.schema if (!(columnName in schema)) { // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case From 904f0dc9fd506c41db49e22359c88cdd1b972cea Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 9 Aug 2024 13:36:22 +0100 Subject: [PATCH 3/6] Adding jest-extended to allow use of oneOf expect, fixing for updated fetching. --- packages/server/package.json | 1 + .../src/api/routes/tests/datasource.spec.ts | 17 +- packages/server/src/tests/jestSetup.ts | 2 + yarn.lock | 236 ++++++++++-------- 4 files changed, 148 insertions(+), 108 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index b835477489..df0ece7bb6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -144,6 +144,7 @@ "copyfiles": "2.4.1", "docker-compose": "0.23.17", "jest": "29.7.0", + "jest-extended": "^4.0.2", "jest-openapi": "0.14.2", "nock": "13.5.4", "nodemon": "2.0.15", diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 237133e639..3d98acdf0c 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -26,6 +26,14 @@ import { tableForDatasource } from "../../../tests/utilities/structures" import nock from "nock" import { Knex } from "knex" +function fetchedSchemaProps() { + return { + externalType: expect.toBeOneOf([expect.any(String), undefined, null]), + constraints: expect.toBeOneOf([expect.anything(), undefined, null]), + autocolumn: expect.toBeOneOf([expect.anything(), undefined, null]), + } +} + describe("/datasources", () => { const config = setup.getConfig() let datasource: Datasource @@ -380,21 +388,22 @@ describe("/datasources", () => { persisted?.entities && Object.entries(persisted.entities).reduce>( (acc, [tableName, table]) => { - acc[tableName] = { + acc[tableName] = expect.objectContaining({ ...table, primaryDisplay: expect.not.stringMatching( new RegExp(`^${table.primaryDisplay || ""}$`) ), schema: Object.entries(table.schema).reduce( (acc, [fieldName, field]) => { - acc[fieldName] = expect.objectContaining({ + acc[fieldName] = { ...field, - }) + ...fetchedSchemaProps(), + } return acc }, {} ), - } + }) return acc }, {} diff --git a/packages/server/src/tests/jestSetup.ts b/packages/server/src/tests/jestSetup.ts index bc6384e4cd..60cf96cb51 100644 --- a/packages/server/src/tests/jestSetup.ts +++ b/packages/server/src/tests/jestSetup.ts @@ -1,8 +1,10 @@ import env from "../environment" +import * as matchers from "jest-extended" import { env as coreEnv, timers } from "@budibase/backend-core" import { testContainerUtils } from "@budibase/backend-core/tests" import nock from "nock" +expect.extend(matchers) if (!process.env.CI) { // set a longer timeout in dev for debugging 100 seconds jest.setTimeout(100 * 1000) diff --git a/yarn.lock b/yarn.lock index 0195f19a2a..d672fb7643 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2045,44 +2045,6 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.29.24": - version "0.0.0" - dependencies: - "@budibase/nano" "10.1.5" - "@budibase/pouchdb-replication-stream" "1.2.11" - "@budibase/shared-core" "0.0.0" - "@budibase/types" "0.0.0" - aws-cloudfront-sign "3.0.2" - aws-sdk "2.1030.0" - bcrypt "5.1.0" - bcryptjs "2.4.3" - bull "4.10.1" - correlation-id "4.0.0" - dd-trace "5.2.0" - dotenv "16.0.1" - ioredis "5.3.2" - joi "17.6.0" - jsonwebtoken "9.0.2" - knex "2.4.2" - koa-passport "^6.0.0" - koa-pino-logger "4.0.0" - lodash "4.17.21" - node-fetch "2.6.7" - passport-google-oauth "2.0.0" - passport-local "1.0.0" - passport-oauth2-refresh "^2.1.0" - pino "8.11.0" - pino-http "8.3.3" - posthog-node "1.3.0" - pouchdb "7.3.0" - pouchdb-find "7.2.2" - redlock "4.2.0" - rotating-file-stream "3.1.0" - sanitize-s3-objectkey "0.0.1" - semver "^7.5.4" - tar-fs "2.1.1" - uuid "^8.3.2" - "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2125,44 +2087,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@npm:@budibase/pro@latest": - version "2.29.24" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.24.tgz#2dbd4c6c0f757aab7e17c413c6d6e4520086f9ac" - integrity sha512-m1v24UD6O21Vbrfsuo5kC5oeg7FzjWO2w8TQMw1VvPKmdIqqclaKDPTPytxwllTMkapMDRNzM5cQzqnQ3yHf6A== - dependencies: - "@budibase/backend-core" "2.29.24" - "@budibase/shared-core" "2.29.24" - "@budibase/string-templates" "2.29.24" - "@budibase/types" "2.29.24" - "@koa/router" "8.0.8" - bull "4.10.1" - joi "17.6.0" - jsonwebtoken "9.0.2" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "2.6.7" - scim-patch "^0.8.1" - scim2-parse-filter "^0.2.8" - -"@budibase/shared-core@2.29.24": - version "0.0.0" - dependencies: - "@budibase/types" "0.0.0" - cron-validate "1.4.5" - -"@budibase/string-templates@2.29.24": - version "0.0.0" - dependencies: - "@budibase/handlebars-helpers" "^0.13.2" - dayjs "^1.10.8" - handlebars "^4.7.8" - lodash.clonedeep "^4.5.0" - -"@budibase/types@2.29.24": - version "0.0.0" - dependencies: - scim-patch "^0.8.1" - "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -7343,7 +7267,30 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0: +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11344,7 +11291,7 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.15.0: +follow-redirects@^1.14.0, follow-redirects@^1.14.4, follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== @@ -12432,7 +12379,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13429,11 +13381,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -13614,7 +13561,7 @@ jest-config@^29.7.0: slash "^3.0.0" strip-json-comments "^3.1.1" -"jest-diff@>=29.4.3 < 30", jest-diff@^29.4.1, jest-diff@^29.7.0: +"jest-diff@>=29.4.3 < 30", jest-diff@^29.0.0, jest-diff@^29.4.1, jest-diff@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a" integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw== @@ -13673,12 +13620,20 @@ jest-environment-node@^29.7.0: jest-mock "^29.7.0" jest-util "^29.7.0" +jest-extended@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/jest-extended/-/jest-extended-4.0.2.tgz#d23b52e687cedf66694e6b2d77f65e211e99e021" + integrity sha512-FH7aaPgtGYHc9mRjriS0ZEHYM5/W69tLrFTIdzm+yJgeoCmmrSB/luSfMSqWP9O29QWHPEmJ4qmU6EwsZideog== + dependencies: + jest-diff "^29.0.0" + jest-get-type "^29.0.0" + jest-get-type@^26.3.0: version "26.3.0" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0" integrity sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig== -jest-get-type@^29.6.3: +jest-get-type@^29.0.0, jest-get-type@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1" integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw== @@ -15984,7 +15939,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16178,13 +16133,25 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17337,7 +17304,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18606,7 +18581,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19668,6 +19643,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19740,13 +19720,33 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3, semver@~2.3.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4, semver@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21311,7 +21311,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21321,6 +21321,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21804,14 +21812,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22578,10 +22578,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22591,6 +22614,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From a9acc7f87b50c7b3a2ca3b7d890fba595c86ba7e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 9 Aug 2024 13:39:47 +0100 Subject: [PATCH 4/6] Adding expect function to allow undefined. --- .../server/src/api/routes/tests/datasource.spec.ts | 14 ++++---------- .../api/routes/tests/utilities/TestFunctions.ts | 4 ++++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 3d98acdf0c..7545253181 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -1,5 +1,5 @@ import * as setup from "./utilities" -import { checkBuilderEndpoint } from "./utilities/TestFunctions" +import { checkBuilderEndpoint, allowUndefined } from "./utilities/TestFunctions" import { getCachedVariable } from "../../../threads/utils" import { context, events } from "@budibase/backend-core" import sdk from "../../../sdk" @@ -26,14 +26,6 @@ import { tableForDatasource } from "../../../tests/utilities/structures" import nock from "nock" import { Knex } from "knex" -function fetchedSchemaProps() { - return { - externalType: expect.toBeOneOf([expect.any(String), undefined, null]), - constraints: expect.toBeOneOf([expect.anything(), undefined, null]), - autocolumn: expect.toBeOneOf([expect.anything(), undefined, null]), - } -} - describe("/datasources", () => { const config = setup.getConfig() let datasource: Datasource @@ -397,7 +389,9 @@ describe("/datasources", () => { (acc, [fieldName, field]) => { acc[fieldName] = { ...field, - ...fetchedSchemaProps(), + externalType: allowUndefined(expect.any(String)), + constraints: allowUndefined(expect.any(Object)), + autocolumn: allowUndefined(expect.any(Boolean)), } return acc }, diff --git a/packages/server/src/api/routes/tests/utilities/TestFunctions.ts b/packages/server/src/api/routes/tests/utilities/TestFunctions.ts index a9a8e7051b..6fa9e054b9 100644 --- a/packages/server/src/api/routes/tests/utilities/TestFunctions.ts +++ b/packages/server/src/api/routes/tests/utilities/TestFunctions.ts @@ -184,3 +184,7 @@ export const runInProd = async (func: any) => { env._set("NODE_ENV", nodeEnv) env._set("JEST_WORKER_ID", workerId) } + +export function allowUndefined(expectation: jest.Expect) { + return expect.toBeOneOf([expectation, undefined, null]) +} From 3e1a0a60b4b5b5b66eb458e6b41896b2e0b859e1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 9 Aug 2024 14:35:13 +0100 Subject: [PATCH 5/6] Checking the correct operation - also typeguarding the check. --- packages/backend-core/src/sql/sqlTable.ts | 7 ++++++- packages/server/src/api/controllers/row/ExternalRequest.ts | 7 +++---- packages/server/src/sdk/app/rows/utils.ts | 6 ++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 55a6bfcaeb..35d7978449 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -70,7 +70,12 @@ function generateSchema( case FieldType.LONGFORM: case FieldType.BARCODEQR: case FieldType.BB_REFERENCE_SINGLE: - schema.text(key) + // primary key strings have to have a length in some DBs + if (primaryKeys.includes(key)) { + schema.string(key, 255) + } else { + schema.text(key) + } break case FieldType.NUMBER: // if meta is specified then this is a junction table entry diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 14dd909edd..5ee2d0fe2b 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -1,6 +1,5 @@ import dayjs from "dayjs" import { - ArrayOperator, AutoFieldSubType, AutoReason, Datasource, @@ -196,13 +195,13 @@ export class ExternalRequest { if (filters) { // need to map over the filters and make sure the _id field isn't present let prefix = 1 - for (const operator of Object.values(filters)) { - const isArrayOperator = Object.values(ArrayOperator).includes(operator) + for (const [operatorType, operator] of Object.entries(filters)) { + const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType) for (const field of Object.keys(operator || {})) { if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { const parts = breakRowIdField(operator[field]) - if (primary.length > 1 && isArrayOperator) { + if (primary.length > 1 && isArrayOp) { operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = { id: primary, values: parts[0], diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index e463397ad9..0cae39f5a9 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -12,6 +12,7 @@ import { Table, TableSchema, SqlClient, + ArrayOperator, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" @@ -311,3 +312,8 @@ function validateTimeOnlyField( return res } + +// type-guard check +export function isArrayFilter(operator: any): operator is ArrayOperator { + return Object.values(ArrayOperator).includes(operator) +} From 28a10bf908f7bf20c81b8f93d14c6fd337ea1e18 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 9 Aug 2024 14:54:47 +0100 Subject: [PATCH 6/6] Another test fix (yay for this one) --- packages/server/src/integrations/utils/utils.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 6d0e4b838f..43302de36f 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -15,7 +15,7 @@ import { helpers, utils } from "@budibase/shared-core" import { pipeline } from "stream/promises" import tmp from "tmp" import fs from "fs" -import { merge } from "lodash" +import { merge, cloneDeep } from "lodash" type PrimitiveTypes = | FieldType.STRING @@ -293,7 +293,11 @@ function copyExistingPropsOver( table.schema[key] table.schema[key] = { // merge the properties - anything missing will be filled in, old definition preferred - ...merge(fetchedColumnDefinition, existingTableSchema[key]), + // have to clone due to the way merge works + ...merge( + cloneDeep(fetchedColumnDefinition), + existingTableSchema[key] + ), // always take externalType and autocolumn from the fetched definition externalType: existingTableSchema[key].externalType ||