From 3e4b0e8cd663c6de21e9de8d86de90025bee8aec Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 Apr 2024 14:03:42 +0200 Subject: [PATCH 01/18] Handle migrations --- .../server/src/api/routes/tests/table.spec.ts | 20 ++++++++-------- .../server/src/sdk/app/tables/migration.ts | 23 +++++++++++-------- .../types/src/documents/app/table/schema.ts | 12 ---------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 9aabffd8b9..2aa44ce64f 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -496,7 +496,7 @@ describe.each([ oldColumn: table.schema["user relationship"], newColumn: { name: "user column", - type: FieldType.BB_REFERENCE, + type: FieldType.BB_REFERENCE_SINGLE, subtype: BBReferenceFieldSubType.USER, }, }) @@ -515,7 +515,7 @@ describe.each([ expect(migratedRow["user column"]).toBeDefined() expect(migratedRow["user relationship"]).not.toBeDefined() expect(row["user relationship"][0]._id).toEqual( - migratedRow["user column"][0]._id + migratedRow["user column"]._id ) } }) @@ -562,7 +562,7 @@ describe.each([ newColumn: { name: "user column", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }) @@ -614,7 +614,7 @@ describe.each([ newColumn: { name: "user column", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }) @@ -669,7 +669,7 @@ describe.each([ newColumn: { name: "user column", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }) @@ -728,7 +728,7 @@ describe.each([ newColumn: { name: "", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }, { status: 400 } @@ -743,7 +743,7 @@ describe.each([ newColumn: { name: "_id", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }, { status: 400 } @@ -758,7 +758,7 @@ describe.each([ newColumn: { name: "num", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }, { status: 400 } @@ -772,12 +772,12 @@ describe.each([ oldColumn: { name: "not a column", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, newColumn: { name: "new column", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, + subtype: BBReferenceFieldSubType.USER, }, }, { status: 400 } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index ae47897289..732773f164 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -4,7 +4,6 @@ import { FieldSchema, BBReferenceFieldSubType, InternalTable, - isBBReferenceField, isRelationshipField, LinkDocument, LinkInfo, @@ -12,6 +11,8 @@ import { RelationshipType, Row, Table, + FieldType, + BBReferenceSingleFieldMetadata, } from "@budibase/types" import sdk from "../../../sdk" import { isExternalTableID } from "../../../integrations/utils" @@ -75,11 +76,14 @@ function getColumnMigrator( throw new BadRequestError(`Column "${oldColumn.name}" does not exist`) } - if (!isBBReferenceField(newColumn)) { + if ( + newColumn.type !== FieldType.BB_REFERENCE_SINGLE && + newColumn.type !== FieldType.BB_REFERENCE + ) { throw new BadRequestError(`Column "${newColumn.name}" is not a user column`) } - if (newColumn.subtype !== "user" && newColumn.subtype !== "users") { + if (newColumn.subtype !== BBReferenceFieldSubType.USER) { throw new BadRequestError(`Column "${newColumn.name}" is not a user column`) } @@ -96,7 +100,7 @@ function getColumnMigrator( } if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) { - if (newColumn.subtype !== BBReferenceFieldSubType.USER) { + if (newColumn.type !== FieldType.BB_REFERENCE_SINGLE) { throw new BadRequestError( `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column` ) @@ -107,22 +111,23 @@ function getColumnMigrator( oldColumn.relationshipType === RelationshipType.MANY_TO_MANY || oldColumn.relationshipType === RelationshipType.MANY_TO_ONE ) { - if (newColumn.subtype !== BBReferenceFieldSubType.USERS) { + if (newColumn.type !== FieldType.BB_REFERENCE) { throw new BadRequestError( `Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column` ) } + return new MultiUserColumnMigrator(table, oldColumn, newColumn) } throw new BadRequestError(`Unknown migration type`) } -abstract class UserColumnMigrator implements ColumnMigrator { +abstract class UserColumnMigrator implements ColumnMigrator { constructor( protected table: Table, protected oldColumn: RelationshipFieldMetadata, - protected newColumn: BBReferenceFieldMetadata + protected newColumn: T ) {} abstract updateRow(row: Row, linkInfo: LinkInfo): void @@ -192,7 +197,7 @@ abstract class UserColumnMigrator implements ColumnMigrator { } } -class SingleUserColumnMigrator extends UserColumnMigrator { +class SingleUserColumnMigrator extends UserColumnMigrator { updateRow(row: Row, linkInfo: LinkInfo): void { row[this.newColumn.name] = dbCore.getGlobalIDFromUserMetadataID( linkInfo.rowId @@ -200,7 +205,7 @@ class SingleUserColumnMigrator extends UserColumnMigrator { } } -class MultiUserColumnMigrator extends UserColumnMigrator { +class MultiUserColumnMigrator extends UserColumnMigrator { updateRow(row: Row, linkInfo: LinkInfo): void { if (!row[this.newColumn.name]) { row[this.newColumn.name] = [] diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 03d57aced8..38424b26b6 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -214,15 +214,3 @@ export function isManyToOne( ): field is ManyToOneRelationshipFieldMetadata { return field.relationshipType === RelationshipType.MANY_TO_ONE } - -export function isBBReferenceField( - field: FieldSchema -): field is BBReferenceFieldMetadata { - return field.type === FieldType.BB_REFERENCE -} - -export function isAttachmentField( - field: FieldSchema -): field is AttachmentFieldMetadata { - return field.type === FieldType.ATTACHMENTS -} From d39c9bd3dd3817d2289ebbec30863ab854a6ccd9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 Apr 2024 15:34:39 +0200 Subject: [PATCH 02/18] Fix user migration --- .../components/grid/controls/MigrationModal.svelte | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index f0cdd7deab..5613598a26 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -33,10 +33,10 @@ } const migrateUserColumn = async () => { - let subtype = BBReferenceFieldSubType.USERS - if (column.schema.relationshipType === RelationshipType.ONE_TO_MANY) { - subtype = BBReferenceFieldSubType.USER - } + const type = + column.schema.relationshipType === RelationshipType.ONE_TO_MANY + ? FieldType.BB_REFERENCE_SINGLE + : FieldType.BB_REFERENCE try { await API.migrateColumn({ @@ -44,8 +44,8 @@ oldColumn: column.schema, newColumn: { name: newColumnName, - type: FieldType.BB_REFERENCE, - subtype, + type, + subtype: BBReferenceFieldSubType.USER, }, }) notifications.success("Column migrated") From 569488deeea8cfa59f561e29d45579f21ff39bef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 May 2024 11:50:11 +0100 Subject: [PATCH 03/18] Move migration logic to backend --- .../server/src/api/controllers/table/index.ts | 2 +- .../server/src/api/routes/tests/table.spec.ts | 103 ++++++++---------- .../server/src/sdk/app/tables/migration.ts | 47 ++++++-- packages/types/src/api/web/app/table.ts | 4 +- 4 files changed, 89 insertions(+), 67 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 63ce00c5ef..eae5204c7e 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -180,5 +180,5 @@ export async function migrate(ctx: UserCtx) { } ctx.status = 200 - ctx.body = { message: `Column ${oldColumn.name} migrated.` } + ctx.body = { message: `Column ${oldColumn} migrated.` } } diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 2aa44ce64f..33bd3dba5c 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -493,16 +493,16 @@ describe.each([ ) await config.api.table.migrate(table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "user column", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "user column", }) const migratedTable = await config.api.table.get(table._id!) - expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user column"]).toEqual({ + name: "user column", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }) expect(migratedTable.schema["user relationship"]).not.toBeDefined() const migratedRows = await config.api.row.fetch(table._id!) @@ -558,16 +558,19 @@ describe.each([ ) await config.api.table.migrate(table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "user column", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "user column", }) const migratedTable = await config.api.table.get(table._id!) - expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user column"]).toEqual({ + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: "array", + }, + }) expect(migratedTable.schema["user relationship"]).not.toBeDefined() const migratedRow = await config.api.row.get(table._id!, testRow._id!) @@ -610,16 +613,19 @@ describe.each([ }) await config.api.table.migrate(table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "user column", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "user column", }) const migratedTable = await config.api.table.get(table._id!) - expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user column"]).toEqual({ + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: "array", + }, + }) expect(migratedTable.schema["user relationship"]).not.toBeDefined() const row1Migrated = await config.api.row.get(table._id!, row1._id!) @@ -665,16 +671,19 @@ describe.each([ }) await config.api.table.migrate(table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "user column", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "user column", }) const migratedTable = await config.api.table.get(table._id!) - expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user column"]).toEqual({ + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: "array", + }, + }) expect(migratedTable.schema["user relationship"]).not.toBeDefined() const row1Migrated = await config.api.row.get(table._id!, row1._id!) @@ -690,7 +699,7 @@ describe.each([ ]) }) - describe("unhappy paths", () => { + describe.only("unhappy paths", () => { let table: Table beforeAll(async () => { table = await config.api.table.save( @@ -724,12 +733,8 @@ describe.each([ await config.api.table.migrate( table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "", }, { status: 400 } ) @@ -739,12 +744,8 @@ describe.each([ await config.api.table.migrate( table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "_id", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "_id", }, { status: 400 } ) @@ -754,12 +755,8 @@ describe.each([ await config.api.table.migrate( table._id!, { - oldColumn: table.schema["user relationship"], - newColumn: { - name: "num", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "user relationship", + newColumn: "num", }, { status: 400 } ) @@ -769,16 +766,8 @@ describe.each([ await config.api.table.migrate( table._id!, { - oldColumn: { - name: "not a column", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, - newColumn: { - name: "new column", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: "not a column", + newColumn: "new column", }, { status: 400 } ) diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index 732773f164..d0decf01f6 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -25,25 +25,58 @@ export interface MigrationResult { export async function migrate( table: Table, - oldColumn: FieldSchema, - newColumn: FieldSchema + oldColumnName: string, + newColumnName: string ): Promise { - if (newColumn.name in table.schema) { - throw new BadRequestError(`Column "${newColumn.name}" already exists`) + if (newColumnName in table.schema) { + throw new BadRequestError(`Column "${newColumnName}" already exists`) } - if (newColumn.name === "") { + if (newColumnName === "") { throw new BadRequestError(`Column name cannot be empty`) } - if (dbCore.isInternalColumnName(newColumn.name)) { + if (dbCore.isInternalColumnName(newColumnName)) { throw new BadRequestError(`Column name cannot be a reserved column name`) } + const oldColumn = table.schema[oldColumnName] + + if (!oldColumn) { + throw new BadRequestError( + `Column "${oldColumnName}" does not exist on table "${table.name}"` + ) + } + + if ( + oldColumn.type !== FieldType.LINK || + oldColumn.tableId !== InternalTable.USER_METADATA + ) { + throw new BadRequestError( + `Only user relationship migration columns is currently supported` + ) + } + + const type = + oldColumn.relationshipType === RelationshipType.ONE_TO_MANY + ? FieldType.BB_REFERENCE_SINGLE + : FieldType.BB_REFERENCE + const newColumn: FieldSchema = { + name: newColumnName, + type, + subtype: BBReferenceFieldSubType.USER, + } + + if (newColumn.type === FieldType.BB_REFERENCE) { + newColumn.constraints = { + type: "array", + } + } + table.schema[newColumn.name] = newColumn table = await sdk.tables.saveTable(table) - let migrator = getColumnMigrator(table, oldColumn, newColumn) + const migrator = getColumnMigrator(table, oldColumn, newColumn) try { return await migrator.doMigration() } catch (e) { diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index ffe59ae395..4e2e84baa0 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -31,8 +31,8 @@ export interface BulkImportResponse { } export interface MigrateRequest { - oldColumn: FieldSchema - newColumn: FieldSchema + oldColumn: string + newColumn: string } export interface MigrateResponse { From 95a9641c087103c2486ee132309cca8b22c5ba1a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 May 2024 11:55:15 +0100 Subject: [PATCH 04/18] Remove logic from frontend --- .../grid/controls/MigrationModal.svelte | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index 5613598a26..8ecec03e0e 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -7,11 +7,6 @@ } from "@budibase/bbui" import { getContext } from "svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" - import { - BBReferenceFieldSubType, - FieldType, - RelationshipType, - } from "@budibase/types" const { API, definition, rows } = getContext("grid") @@ -33,20 +28,11 @@ } const migrateUserColumn = async () => { - const type = - column.schema.relationshipType === RelationshipType.ONE_TO_MANY - ? FieldType.BB_REFERENCE_SINGLE - : FieldType.BB_REFERENCE - try { await API.migrateColumn({ tableId: $definition._id, - oldColumn: column.schema, - newColumn: { - name: newColumnName, - type, - subtype: BBReferenceFieldSubType.USER, - }, + oldColumn: column.schema.name, + newColumn: newColumnName, }) notifications.success("Column migrated") } catch (e) { From b36d7868a47a791b7950415894843e39791ded66 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 May 2024 13:39:20 +0100 Subject: [PATCH 05/18] Lint --- packages/server/src/api/routes/tests/table.spec.ts | 2 +- packages/types/src/api/web/app/table.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 33bd3dba5c..cca2f0d908 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -699,7 +699,7 @@ describe.each([ ]) }) - describe.only("unhappy paths", () => { + describe("unhappy paths", () => { let table: Table beforeAll(async () => { table = await config.api.table.save( diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index 4e2e84baa0..c9d63feaab 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -1,5 +1,4 @@ import { - FieldSchema, Row, Table, TableRequest, From 457eebbcb43ec0f7fdf63d6978ed77e9b3abb93b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 3 May 2024 17:29:47 +0200 Subject: [PATCH 06/18] Fix types --- packages/builder/src/constants/backend/index.js | 13 ++----------- packages/frontend-core/src/constants.js | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 491d7f0e99..75f6a053b5 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -165,20 +165,11 @@ export const FIELDS = { BBReferenceFieldSubType.USER ], }, - // Used for display of editing existing columns - DEPRECATED_USER: { - name: "User", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - icon: TypeIconMap[FieldType.BB_REFERENCE_SINGLE][ - BBReferenceFieldSubType.USER - ], - }, USERS: { name: "User List", type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, - icon: TypeIconMap[FieldType.BB_REFERENCE][BBReferenceFieldSubType.USERS], + subtype: BBReferenceFieldSubType.USER, + icon: TypeIconMap[FieldType.BB_REFERENCE][BBReferenceFieldSubType.USER], constraints: { type: "array", }, diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 9915e2e0b2..d9e3509a96 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -132,7 +132,7 @@ export const TypeIconMap = { [FieldType.BIGINT]: "TagBold", [FieldType.AUTO]: "MagicWand", [FieldType.BB_REFERENCE]: { - [BBReferenceFieldSubType.USER]: "User", + [BBReferenceFieldSubType.USER]: "UserGroup", [BBReferenceFieldSubType.USERS]: "UserGroup", }, [FieldType.BB_REFERENCE_SINGLE]: { From 9a6e6ed1158e45e456f7f713bc6d6a3f520c7ebc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 3 May 2024 17:53:59 +0200 Subject: [PATCH 07/18] Fix data between old and new --- .../grid/cells/BBReferenceCell.svelte | 27 ++++++++++++++----- packages/server/src/integrations/base/sql.ts | 4 +-- .../server/src/integrations/base/sqlTable.ts | 15 +---------- .../server/src/sdk/app/rows/search/utils.ts | 10 +------ packages/server/src/utilities/schema.ts | 18 +++---------- packages/shared-core/src/filters.ts | 11 ++------ 6 files changed, 28 insertions(+), 57 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte index ebfb33d884..743b6695bf 100644 --- a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte @@ -1,22 +1,35 @@