From 81801582e9d49705a3d78c02113aac8b1019f925 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 17:43:42 +0200 Subject: [PATCH 1/9] Add type guards --- packages/shared-core/src/helpers/schema.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/shared-core/src/helpers/schema.ts b/packages/shared-core/src/helpers/schema.ts index 985e068be0..ad4c237247 100644 --- a/packages/shared-core/src/helpers/schema.ts +++ b/packages/shared-core/src/helpers/schema.ts @@ -6,7 +6,10 @@ import { export function isDeprecatedSingleUserColumn( schema: Pick -) { +): schema is { + type: FieldType.BB_REFERENCE + subtype: BBReferenceFieldSubType.USER +} { const result = schema.type === FieldType.BB_REFERENCE && schema.subtype === BBReferenceFieldSubType.USER && From efb02a2c44ae0b352181b086d96269d11ee41593 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 17:44:52 +0200 Subject: [PATCH 2/9] Improve message --- .../src/utilities/rowProcessor/bbReferenceProcessor.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index d69fe73052..3786079883 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -14,7 +14,11 @@ export async function processInputBBReference( subtype: BBReferenceFieldSubType.USER ): Promise { if (value && Array.isArray(value)) { - throw "BB_REFERENCE_SINGLE cannot be an array" + throw new InvalidBBRefError( + JSON.stringify(value), + BBReferenceFieldSubType.USER, + "BB_REFERENCE_SINGLE cannot be an array" + ) } let id = typeof value === "string" ? value : value?._id From 092e75ed6c99de5e29be7c9d67e3023dcb6185b7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 17:48:10 +0200 Subject: [PATCH 3/9] Treat deprecatedSingleUserColumn as single on inputting --- .../utilities/rowProcessor/bbReferenceProcessor.ts | 12 +++++++----- packages/server/src/utilities/rowProcessor/index.ts | 10 +++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 3786079883..874113f6f1 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -14,11 +14,13 @@ export async function processInputBBReference( subtype: BBReferenceFieldSubType.USER ): Promise { if (value && Array.isArray(value)) { - throw new InvalidBBRefError( - JSON.stringify(value), - BBReferenceFieldSubType.USER, - "BB_REFERENCE_SINGLE cannot be an array" - ) + if (value.length > 1) { + throw new InvalidBBRefError( + JSON.stringify(value), + BBReferenceFieldSubType.USER + ) + } + value = value[0] } let id = typeof value === "string" ? value : value?._id diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index e7bc725285..a3f3ebc8f6 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -18,6 +18,7 @@ import { processOutputBBReferences, } from "./bbReferenceProcessor" import { isExternalTableID } from "../../integrations/utils" +import { helpers } from "@budibase/shared-core" export * from "./utils" export * from "./attachments" @@ -162,10 +163,13 @@ export async function inputProcessing( if (attachment?.url) { delete clonedRow[key].url } - } else if (field.type === FieldType.BB_REFERENCE && value) { - clonedRow[key] = await processInputBBReferences(value, field.subtype) - } else if (field.type === FieldType.BB_REFERENCE_SINGLE && value) { + } else if ( + field.type === FieldType.BB_REFERENCE_SINGLE || + helpers.schema.isDeprecatedSingleUserColumn(field) + ) { clonedRow[key] = await processInputBBReference(value, field.subtype) + } else if (field.type === FieldType.BB_REFERENCE) { + clonedRow[key] = await processInputBBReferences(value, field.subtype) } } From 6c1b3a0d2d8e1e67562c904d7b21e973cf5a103c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 18:08:41 +0200 Subject: [PATCH 4/9] Fix tests --- packages/server/src/sdk/app/rows/search.ts | 2 +- packages/server/src/sdk/app/rows/search/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 2d194ab196..69ded8e76f 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -80,7 +80,7 @@ export async function search( const table = await sdk.tables.getTable(options.tableId) options = searchInputMapping(table, options, { - isSql: !!table.sql || !!env.SQS_SEARCH_ENABLE, + isSql: !!table.sql, }) if (isExternalTable) { diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 62f5af2b70..bae45298eb 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -73,7 +73,7 @@ function userColumnMapping( let wrapper = (s: string) => s if (isDeprecatedSingleUserColumn && filterValue && isSql) { - // Decreated single users are stored as stringified arrays of a single value + // Deprecated single users are stored as stringified arrays of a single value wrapper = (s: string) => JSON.stringify([s]) } From 74ac68d72d81d4c045b44663e0314112dca497af Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 18:22:55 +0200 Subject: [PATCH 5/9] Fix tests --- packages/server/src/utilities/rowProcessor/index.ts | 7 ++++--- .../utilities/rowProcessor/tests/inputProcessing.spec.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index a3f3ebc8f6..59237be5f3 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -164,11 +164,12 @@ export async function inputProcessing( delete clonedRow[key].url } } else if ( - field.type === FieldType.BB_REFERENCE_SINGLE || - helpers.schema.isDeprecatedSingleUserColumn(field) + value && + (field.type === FieldType.BB_REFERENCE_SINGLE || + helpers.schema.isDeprecatedSingleUserColumn(field)) ) { clonedRow[key] = await processInputBBReference(value, field.subtype) - } else if (field.type === FieldType.BB_REFERENCE) { + } else if (value && field.type === FieldType.BB_REFERENCE) { clonedRow[key] = await processInputBBReferences(value, field.subtype) } } diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index b1928b696b..81094583e2 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -102,7 +102,7 @@ describe("rowProcessor - inputProcessing", () => { name: "user", constraints: { presence: true, - type: "string", + type: "array", }, }, }, @@ -154,7 +154,7 @@ describe("rowProcessor - inputProcessing", () => { name: "user", constraints: { presence: false, - type: "string", + type: "array", }, }, }, @@ -196,7 +196,7 @@ describe("rowProcessor - inputProcessing", () => { name: "user", constraints: { presence: false, - type: "string", + type: "array", }, }, }, From 25d86d179dc08e083bfcd74ed0d41b4c096c5b02 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 15 May 2024 10:56:55 +0200 Subject: [PATCH 6/9] Handle inputProcessing on bulk row import --- .../api/controllers/row/ExternalRequest.ts | 19 +++++++++++++++++-- .../src/api/controllers/table/external.ts | 13 +++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index be6ac885df..bd92413851 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -265,7 +265,10 @@ export class ExternalRequest { } } - inputProcessing(row: Row | undefined, table: Table) { + inputProcessing( + row: T, + table: Table + ): { row: T; manyRelationships: ManyRelationship[] } { if (!row) { return { row, manyRelationships: [] } } @@ -346,7 +349,7 @@ export class ExternalRequest { // we return the relationships that may need to be created in the through table // we do this so that if the ID is generated by the DB it can be inserted // after the fact - return { row: newRow, manyRelationships } + return { row: newRow as T, manyRelationships } } /** @@ -598,6 +601,18 @@ export class ExternalRequest { // clean up row on ingress using schema const processed = this.inputProcessing(row, table) row = processed.row + let manyRelationships = processed.manyRelationships + + if (!row && rows) { + manyRelationships = [] + for (let i = 0; i < rows.length; i++) { + const processed = this.inputProcessing(rows[i], table) + rows[i] = processed.row + if (processed.manyRelationships.length) { + manyRelationships.push(...processed.manyRelationships) + } + } + } if ( operation === Operation.DELETE && (filters == null || Object.keys(filters).length === 0) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index e526af4ecb..bd674d7d38 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -15,6 +15,7 @@ import { } from "@budibase/types" import sdk from "../../../sdk" import { builderSocket } from "../../../websockets" +import { inputProcessing } from "../../../utilities/rowProcessor" function getDatasourceId(table: Table) { if (!table) { @@ -80,7 +81,7 @@ export async function destroy(ctx: UserCtx) { export async function bulkImport( ctx: UserCtx ) { - const table = await sdk.tables.getTable(ctx.params.tableId) + let table = await sdk.tables.getTable(ctx.params.tableId) const { rows } = ctx.request.body const schema = table.schema @@ -88,7 +89,15 @@ export async function bulkImport( ctx.throw(400, "Provided data import information is invalid.") } - const parsedRows = parse(rows, schema) + const parsedRows = [] + for (const row of parse(rows, schema)) { + const processed = await inputProcessing(ctx.user?._id, table, row, { + noAutoRelationships: true, + }) + parsedRows.push(processed.row) + table = processed.table + } + await handleRequest(Operation.BULK_CREATE, table._id!, { rows: parsedRows, }) From 062df0ff8b0711416bb82c26708ec512fc7501d4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 15 May 2024 11:01:36 +0200 Subject: [PATCH 7/9] Cleanup --- packages/server/src/sdk/app/rows/search.ts | 4 +--- .../server/src/sdk/app/rows/search/utils.ts | 22 +++++-------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 69ded8e76f..e347a8657d 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -79,9 +79,7 @@ export async function search( } const table = await sdk.tables.getTable(options.tableId) - options = searchInputMapping(table, options, { - isSql: !!table.sql, - }) + options = searchInputMapping(table, options) if (isExternalTable) { return external.search(options, table) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index bae45298eb..cf4a4e61c6 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -52,8 +52,7 @@ function findColumnInQueries( function userColumnMapping( column: string, options: RowSearchParams, - isDeprecatedSingleUserColumn: boolean = false, - isSql: boolean = false + isDeprecatedSingleUserColumn: boolean = false ) { findColumnInQueries(column, options, (filterValue: any): any => { const isArray = Array.isArray(filterValue), @@ -71,33 +70,23 @@ function userColumnMapping( } } - let wrapper = (s: string) => s - if (isDeprecatedSingleUserColumn && filterValue && isSql) { - // Deprecated single users are stored as stringified arrays of a single value - wrapper = (s: string) => JSON.stringify([s]) - } - if (isArray) { return filterValue.map(el => { if (typeof el === "string") { - return wrapper(processString(el)) + return processString(el) } else { return el } }) } else { - return wrapper(processString(filterValue)) + return processString(filterValue) } }) } // maps through the search parameters to check if any of the inputs are invalid // based on the table schema, converts them to something that is valid. -export function searchInputMapping( - table: Table, - options: RowSearchParams, - datasourceOptions: { isSql?: boolean } = {} -) { +export function searchInputMapping(table: Table, options: RowSearchParams) { if (!table?.schema) { return options } @@ -119,8 +108,7 @@ export function searchInputMapping( userColumnMapping( key, options, - helpers.schema.isDeprecatedSingleUserColumn(column), - datasourceOptions.isSql + helpers.schema.isDeprecatedSingleUserColumn(column) ) break } From 8e0fb2f74b3c93aa367c011ca3bd967db9dd4778 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 15 May 2024 11:30:18 +0200 Subject: [PATCH 8/9] Lint --- packages/server/src/sdk/app/rows/search/utils.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index cf4a4e61c6..797383eff0 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -11,7 +11,7 @@ import { RowSearchParams, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" -import { helpers, utils } from "@budibase/shared-core" +import { utils } from "@budibase/shared-core" export async function paginatedSearch( query: SearchFilters, @@ -49,11 +49,7 @@ function findColumnInQueries( } } -function userColumnMapping( - column: string, - options: RowSearchParams, - isDeprecatedSingleUserColumn: boolean = false -) { +function userColumnMapping(column: string, options: RowSearchParams) { findColumnInQueries(column, options, (filterValue: any): any => { const isArray = Array.isArray(filterValue), isString = typeof filterValue === "string" @@ -105,11 +101,7 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { break } case FieldType.BB_REFERENCE: { - userColumnMapping( - key, - options, - helpers.schema.isDeprecatedSingleUserColumn(column) - ) + userColumnMapping(key, options) break } } From cfddf020c3c6a516b9eb6566060129ccc1fea605 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 15 May 2024 12:18:07 +0200 Subject: [PATCH 9/9] Don't allow migrating autocolumns --- .../frontend-core/src/components/grid/cells/HeaderCell.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index bb38c5094f..530595fe40 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -386,7 +386,7 @@ > Hide column - {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === TableNames.USERS} + {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === TableNames.USERS && !column.schema.autocolumn} Migrate to user column