From 36346874428a999206f79d5a396dbeaefa15743c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Oct 2023 10:31:36 +0100 Subject: [PATCH 01/56] Create endpoint and controller function for user column migration. --- packages/server/src/api/controllers/table/index.ts | 2 ++ packages/server/src/api/routes/table.ts | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 44f673f284..2780185ed7 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -158,3 +158,5 @@ export async function validateExistingTableImport(ctx: UserCtx) { ctx.status = 422 } } + +export async function migrate(ctx: UserCtx) {} diff --git a/packages/server/src/api/routes/table.ts b/packages/server/src/api/routes/table.ts index 7ffa5acb3e..1aa23257fb 100644 --- a/packages/server/src/api/routes/table.ts +++ b/packages/server/src/api/routes/table.ts @@ -167,4 +167,11 @@ router tableController.bulkImport ) + .post( + "/api/tables/:tableId/migrate", + paramResource("tableId"), + authorized(BUILDER), + tableController.migrate + ) + export default router From 5747f30b5f5cccd7ea5b78a6565e56edfc919400 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Oct 2023 12:04:50 +0100 Subject: [PATCH 02/56] Precondition checks to make sure the migration is from the right column to the right column. --- .../server/src/api/controllers/table/index.ts | 56 ++++++++++++++++++- packages/types/src/api/web/app/table.ts | 11 ++++ .../types/src/documents/app/table/schema.ts | 30 ++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 2780185ed7..36dee4811e 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -11,11 +11,16 @@ import { BulkImportRequest, BulkImportResponse, FetchTablesResponse, + InternalTable, + MigrateRequest, + MigrateResponse, SaveTableRequest, SaveTableResponse, Table, TableResponse, UserCtx, + isBBReferenceField, + isRelationshipField, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -159,4 +164,53 @@ export async function validateExistingTableImport(ctx: UserCtx) { } } -export async function migrate(ctx: UserCtx) {} +function error(ctx: UserCtx, message: string, status = 400) { + ctx.status = status + ctx.body = { message } +} + +export async function migrate(ctx: UserCtx) { + const { tableId, oldColumn, newColumn } = ctx.request.body + + // For now we're only supporting migrations of user relationships to user + // columns in internal tables. In future we may want to support other + // migrations but for now return an error if we aren't migrating a user + // relationship. + if (isExternalTable(tableId)) { + return error(ctx, "External tables cannot be migrated") + } + + const table = await sdk.tables.getTable(tableId) + + if (!(oldColumn.name in table.schema)) { + return error( + ctx, + `Column "${oldColumn.name}" does not exist on table "${table.name}"` + ) + } + + if (newColumn.name in table.schema) { + return error( + ctx, + `Column "${newColumn.name}" already exists on table "${table.name}"` + ) + } + + if (!isBBReferenceField(newColumn)) { + return error(ctx, `Column "${newColumn.name}" is not a user column`) + } + + if (newColumn.subtype !== "user" && newColumn.subtype !== "users") { + return error(ctx, `Column "${newColumn.name}" is not a user column`) + } + + if (!isRelationshipField(oldColumn)) { + return error(ctx, `Column "${oldColumn.name}" is not a user relationship`) + } + + if (oldColumn.tableId !== InternalTable.USER_METADATA) { + return error(ctx, `Column "${oldColumn.name}" is not a user relationship`) + } + + let rows = await sdk.rows.fetch(tableId) +} diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index cb5faaa9ea..61da12d041 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -1,4 +1,5 @@ import { + FieldSchema, Row, Table, TableRequest, @@ -33,3 +34,13 @@ export interface BulkImportRequest { export interface BulkImportResponse { message: string } + +export interface MigrateRequest { + tableId: string + oldColumn: FieldSchema + newColumn: FieldSchema +} + +export interface MigrateResponse { + message: string +} diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index e529a8e8b7..755ccf61e7 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -164,3 +164,33 @@ export type FieldSchema = export interface TableSchema { [key: string]: FieldSchema } + +export function isRelationshipField( + field: FieldSchema +): field is RelationshipFieldMetadata { + return field.type === FieldType.LINK +} + +export function isManyToMany( + field: RelationshipFieldMetadata +): field is ManyToManyRelationshipFieldMetadata { + return field.relationshipType === RelationshipType.MANY_TO_MANY +} + +export function isOneToMany( + field: RelationshipFieldMetadata +): field is OneToManyRelationshipFieldMetadata { + return field.relationshipType === RelationshipType.ONE_TO_MANY +} + +export function isManyToOne( + field: RelationshipFieldMetadata +): field is ManyToOneRelationshipFieldMetadata { + return field.relationshipType === RelationshipType.MANY_TO_ONE +} + +export function isBBReferenceField( + field: FieldSchema +): field is BBReferenceFieldMetadata { + return field.type === FieldType.BB_REFERENCE +} From c25de74e174c9e2fa895f5aa8e2a5cc51f9968bf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Oct 2023 15:14:34 +0100 Subject: [PATCH 03/56] Action Michael's feedback about the structure of this feature. --- .../server/src/api/controllers/table/index.ts | 50 +---------- packages/server/src/sdk/app/tables/index.ts | 12 +++ .../server/src/sdk/app/tables/migration.ts | 84 +++++++++++++++++++ 3 files changed, 97 insertions(+), 49 deletions(-) create mode 100644 packages/server/src/sdk/app/tables/migration.ts diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 36dee4811e..132bae3d81 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -11,7 +11,6 @@ import { BulkImportRequest, BulkImportResponse, FetchTablesResponse, - InternalTable, MigrateRequest, MigrateResponse, SaveTableRequest, @@ -19,8 +18,6 @@ import { Table, TableResponse, UserCtx, - isBBReferenceField, - isRelationshipField, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -164,53 +161,8 @@ export async function validateExistingTableImport(ctx: UserCtx) { } } -function error(ctx: UserCtx, message: string, status = 400) { - ctx.status = status - ctx.body = { message } -} - export async function migrate(ctx: UserCtx) { const { tableId, oldColumn, newColumn } = ctx.request.body - - // For now we're only supporting migrations of user relationships to user - // columns in internal tables. In future we may want to support other - // migrations but for now return an error if we aren't migrating a user - // relationship. - if (isExternalTable(tableId)) { - return error(ctx, "External tables cannot be migrated") - } - const table = await sdk.tables.getTable(tableId) - - if (!(oldColumn.name in table.schema)) { - return error( - ctx, - `Column "${oldColumn.name}" does not exist on table "${table.name}"` - ) - } - - if (newColumn.name in table.schema) { - return error( - ctx, - `Column "${newColumn.name}" already exists on table "${table.name}"` - ) - } - - if (!isBBReferenceField(newColumn)) { - return error(ctx, `Column "${newColumn.name}" is not a user column`) - } - - if (newColumn.subtype !== "user" && newColumn.subtype !== "users") { - return error(ctx, `Column "${newColumn.name}" is not a user column`) - } - - if (!isRelationshipField(oldColumn)) { - return error(ctx, `Column "${oldColumn.name}" is not a user relationship`) - } - - if (oldColumn.tableId !== InternalTable.USER_METADATA) { - return error(ctx, `Column "${oldColumn.name}" is not a user relationship`) - } - - let rows = await sdk.rows.fetch(tableId) + await sdk.tables.migrate(table, oldColumn, newColumn) } diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 64fcde4bff..e4884b2ec4 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -7,6 +7,7 @@ import { } from "../../../integrations/utils" import { Database, + FieldSchema, Table, TableResponse, TableViewsResponse, @@ -14,6 +15,7 @@ import { import datasources from "../datasources" import { populateExternalTableSchemas } from "./validation" import sdk from "../../../sdk" +import { migrate } from "./migration" async function getAllInternalTables(db?: Database): Promise { if (!db) { @@ -84,6 +86,14 @@ async function saveTable(table: Table) { } } +async function addColumn(table: Table, newColumn: FieldSchema) { + if (newColumn.name in table.schema) { + throw `Column "${newColumn.name}" already exists on table "${table.name}"` + } + table.schema[newColumn.name] = newColumn + await saveTable(table) +} + export default { getAllInternalTables, getAllExternalTables, @@ -92,4 +102,6 @@ export default { populateExternalTableSchemas, enrichViewSchemas, saveTable, + addColumn, + migrate, } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts new file mode 100644 index 0000000000..f727b5ea9d --- /dev/null +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -0,0 +1,84 @@ +import { BadRequestError } from "@budibase/backend-core" +import { + BBReferenceFieldMetadata, + FieldSchema, + InternalTable, + RelationshipFieldMetadata, + Table, + isBBReferenceField, + isRelationshipField, +} from "@budibase/types" +import { isExternalTable } from "src/integrations/utils" +import sdk from "../../../sdk" + +export async function migrate( + table: Table, + oldColumn: FieldSchema, + newColumn: FieldSchema +) { + let migrator = getColumnMigrator(table, oldColumn, newColumn) + + await sdk.tables.addColumn(table, newColumn) + + migrator.doMigration() +} + +interface ColumnMigrator { + doMigration(): Promise +} + +function getColumnMigrator( + table: Table, + oldColumn: FieldSchema, + newColumn: FieldSchema +): ColumnMigrator { + // For now we're only supporting migrations of user relationships to user + // columns in internal tables. In future we may want to support other + // migrations but for now return an error if we aren't migrating a user + // relationship. + if (isExternalTable(table._id!)) { + throw new BadRequestError("External tables cannot be migrated") + } + + if (!(oldColumn.name in table.schema)) { + throw new BadRequestError(`Column "${oldColumn.name}" does not exist`) + } + + if (newColumn.name in table.schema) { + throw new BadRequestError(`Column "${newColumn.name}" already exists`) + } + + if (!isBBReferenceField(newColumn)) { + throw new BadRequestError(`Column "${newColumn.name}" is not a user column`) + } + + if (newColumn.subtype !== "user" && newColumn.subtype !== "users") { + throw new BadRequestError(`Column "${newColumn.name}" is not a user column`) + } + + if (!isRelationshipField(oldColumn)) { + throw new BadRequestError( + `Column "${oldColumn.name}" is not a user relationship` + ) + } + + if (oldColumn.tableId !== InternalTable.USER_METADATA) { + throw new BadRequestError( + `Column "${oldColumn.name}" is not a user relationship` + ) + } + + return new UserColumnMigrator(table, oldColumn, newColumn) +} + +class UserColumnMigrator implements ColumnMigrator { + constructor( + private table: Table, + private oldColumn: RelationshipFieldMetadata, + private newColumn: BBReferenceFieldMetadata + ) {} + + async doMigration() { + let rows = await sdk.rows.fetch(this.table._id!) + } +} From 6ae5451fdf46db2dce172047a310b64ebbf596a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Oct 2023 16:56:55 +0100 Subject: [PATCH 04/56] Create failing test. --- .../server/src/api/controllers/table/index.ts | 6 ++- .../server/src/api/routes/tests/table.spec.ts | 44 +++++++++++++++++++ packages/server/src/sdk/app/links/index.ts | 5 +++ packages/server/src/sdk/app/links/links.ts | 32 ++++++++++++++ packages/server/src/sdk/app/rows/search.ts | 6 +-- .../server/src/sdk/app/tables/migration.ts | 3 +- packages/server/src/sdk/index.ts | 2 + .../server/src/tests/utilities/api/table.ts | 22 +++++++++- packages/types/src/api/web/app/table.ts | 1 - 9 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 packages/server/src/sdk/app/links/index.ts create mode 100644 packages/server/src/sdk/app/links/links.ts diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 132bae3d81..75532dbfdf 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -162,7 +162,11 @@ export async function validateExistingTableImport(ctx: UserCtx) { } export async function migrate(ctx: UserCtx) { - const { tableId, oldColumn, newColumn } = ctx.request.body + const { oldColumn, newColumn } = ctx.request.body + let tableId = ctx.params.tableId as string const table = await sdk.tables.getTable(tableId) await sdk.tables.migrate(table, oldColumn, newColumn) + + ctx.status = 200 + ctx.body = { message: `Column ${oldColumn.name} migrated.` } } diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index ded54729b9..eaf3757ea8 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -6,6 +6,8 @@ import { Table, ViewCalculation, AutoFieldSubTypes, + InternalTable, + FieldSubtype, } from "@budibase/types" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" @@ -417,4 +419,46 @@ describe("/tables", () => { }) }) }) + + describe("migrate", () => { + it("should successfully migrate a user relationship to a user column", async () => { + const users = await config.api.row.fetch(InternalTable.USER_METADATA) + const table = await config.api.table.create({ + name: "table", + type: "table", + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.ONE_TO_MANY, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + await config.api.row.save(table._id!, { + "user relationship": users, + }) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USER, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + + const rows = await config.api.row.fetch(table._id!) + expect(rows[0]["user column"]).toBeDefined() + }) + }) }) diff --git a/packages/server/src/sdk/app/links/index.ts b/packages/server/src/sdk/app/links/index.ts new file mode 100644 index 0000000000..6655a76656 --- /dev/null +++ b/packages/server/src/sdk/app/links/index.ts @@ -0,0 +1,5 @@ +import * as links from "./links" + +export default { + ...links, +} diff --git a/packages/server/src/sdk/app/links/links.ts b/packages/server/src/sdk/app/links/links.ts new file mode 100644 index 0000000000..7f3dcbd4ac --- /dev/null +++ b/packages/server/src/sdk/app/links/links.ts @@ -0,0 +1,32 @@ +import { context } from "@budibase/backend-core" +import { isTableId } from "@budibase/backend-core/src/docIds" +import { LinkDocument, LinkDocumentValue } from "@budibase/types" +import { ViewName, getQueryIndex } from "../../../../src/db/utils" + +export async function fetch(tableId: string): Promise { + if (!isTableId(tableId)) { + throw new Error(`Invalid tableId: ${tableId}`) + } + + const db = context.getAppDB() + const params: any = { startkey: [tableId], endkey: [tableId, {}] } + const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows + return linkRows.map(row => row.value as LinkDocumentValue) +} + +export async function fetchWithDocument( + tableId: string +): Promise { + if (!isTableId(tableId)) { + throw new Error(`Invalid tableId: ${tableId}`) + } + + const db = context.getAppDB() + const params: any = { + startkey: [tableId], + endkey: [tableId, {}], + include_docs: true, + } + const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows + return linkRows.map(row => row.doc as LinkDocument) +} diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index ced35db9be..f75bd07437 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,4 +1,4 @@ -import { SearchFilters, SearchParams, Row } from "@budibase/types" +import { SearchFilters, SearchParams } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" @@ -45,7 +45,7 @@ export async function exportRows( return pickApi(options.tableId).exportRows(options) } -export async function fetch(tableId: string): Promise { +export async function fetch(tableId: string) { return pickApi(tableId).fetch(tableId) } @@ -53,6 +53,6 @@ export async function fetchView( tableId: string, viewName: string, params: ViewParams -): Promise { +) { return pickApi(tableId).fetchView(viewName, params) } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index f727b5ea9d..ac7f3a3765 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -8,8 +8,8 @@ import { isBBReferenceField, isRelationshipField, } from "@budibase/types" -import { isExternalTable } from "src/integrations/utils" import sdk from "../../../sdk" +import { isExternalTable } from "../../../../src/integrations/utils" export async function migrate( table: Table, @@ -80,5 +80,6 @@ class UserColumnMigrator implements ColumnMigrator { async doMigration() { let rows = await sdk.rows.fetch(this.table._id!) + let links = await sdk.links.fetchWithDocument(this.table._id!) } } diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 24eb1ebf3c..c3057e3d4f 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -5,6 +5,7 @@ import { default as applications } from "./app/applications" import { default as datasources } from "./app/datasources" import { default as queries } from "./app/queries" import { default as rows } from "./app/rows" +import { default as links } from "./app/links" import { default as users } from "./users" import { default as plugins } from "./plugins" import * as views from "./app/views" @@ -22,6 +23,7 @@ const sdk = { plugins, views, permissions, + links, } // default export for TS diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index 04432a788a..501841f6e7 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -1,4 +1,10 @@ -import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types" +import { + MigrateRequest, + MigrateResponse, + SaveTableRequest, + SaveTableResponse, + Table, +} from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -42,4 +48,18 @@ export class TableAPI extends TestAPI { .expect(expectStatus) return res.body } + + migrate = async ( + tableId: string, + data: MigrateRequest, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .post(`/api/tables/${tableId}/migrate`) + .send(data) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(expectStatus) + return res.body + } } diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index 61da12d041..f4d6720516 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -36,7 +36,6 @@ export interface BulkImportResponse { } export interface MigrateRequest { - tableId: string oldColumn: FieldSchema newColumn: FieldSchema } From 1771b5905a37e544261a2fb8b2deca58786aeabc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Oct 2023 18:02:10 +0100 Subject: [PATCH 05/56] Most of the way to getting my first test passing. --- .../server/src/api/routes/tests/table.spec.ts | 2 + packages/server/src/sdk/app/tables/index.ts | 17 ++++++--- .../server/src/sdk/app/tables/migration.ts | 38 +++++++++++++++++-- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index eaf3757ea8..dcf1704ed0 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -456,9 +456,11 @@ describe("/tables", () => { const migratedTable = await config.api.table.get(table._id!) expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() const rows = await config.api.row.fetch(table._id!) expect(rows[0]["user column"]).toBeDefined() + expect(rows[0]["user relationship"]).not.toBeDefined() }) }) }) diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index e4884b2ec4..fab8c7d198 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -16,6 +16,8 @@ import datasources from "../datasources" import { populateExternalTableSchemas } from "./validation" import sdk from "../../../sdk" import { migrate } from "./migration" +import { DocumentInsertResponse } from "@budibase/nano" +import { cloneDeep } from "lodash" async function getAllInternalTables(db?: Database): Promise { if (!db) { @@ -75,23 +77,28 @@ function enrichViewSchemas(table: Table): TableResponse { } } -async function saveTable(table: Table) { +async function saveTable(table: Table): Promise { const db = context.getAppDB() + let resp: DocumentInsertResponse if (isExternalTable(table._id!)) { const datasource = await sdk.datasources.get(table.sourceId!) datasource.entities![table.name] = table - await db.put(datasource) + resp = await db.put(datasource) } else { - await db.put(table) + resp = await db.put(table) } + + let tableClone = cloneDeep(table) + tableClone._rev = resp.rev + return tableClone } -async function addColumn(table: Table, newColumn: FieldSchema) { +async function addColumn(table: Table, newColumn: FieldSchema): Promise
{ if (newColumn.name in table.schema) { throw `Column "${newColumn.name}" already exists on table "${table.name}"` } table.schema[newColumn.name] = newColumn - await saveTable(table) + return await saveTable(table) } export default { diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index ac7f3a3765..a08b56826d 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -1,15 +1,19 @@ -import { BadRequestError } from "@budibase/backend-core" +import { BadRequestError, context } from "@budibase/backend-core" import { BBReferenceFieldMetadata, FieldSchema, InternalTable, RelationshipFieldMetadata, + Row, Table, isBBReferenceField, isRelationshipField, } from "@budibase/types" import sdk from "../../../sdk" import { isExternalTable } from "../../../../src/integrations/utils" +import { db as dbCore } from "@budibase/backend-core" +import { EventType, updateLinks } from "../../../../src/db/linkedRows" +import { cloneDeep } from "lodash" export async function migrate( table: Table, @@ -17,10 +21,15 @@ export async function migrate( newColumn: FieldSchema ) { let migrator = getColumnMigrator(table, oldColumn, newColumn) + let oldTable = cloneDeep(table) - await sdk.tables.addColumn(table, newColumn) + table = await sdk.tables.addColumn(table, newColumn) - migrator.doMigration() + await migrator.doMigration() + + delete table.schema[oldColumn.name] + await sdk.tables.saveTable(table) + await updateLinks({ eventType: EventType.TABLE_UPDATED, table, oldTable }) } interface ColumnMigrator { @@ -80,6 +89,29 @@ class UserColumnMigrator implements ColumnMigrator { async doMigration() { let rows = await sdk.rows.fetch(this.table._id!) + let rowsById = rows.reduce((acc, row) => { + acc[row._id!] = row + return acc + }, {} as Record) + let links = await sdk.links.fetchWithDocument(this.table._id!) + for (let link of links) { + if (link.doc1.tableId !== this.table._id) { + continue + } + if (link.doc1.fieldName !== this.oldColumn.name) { + continue + } + if (link.doc2.tableId !== InternalTable.USER_METADATA) { + continue + } + + let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId) + let row = rowsById[link.doc1.rowId] + row[this.newColumn.name] = userId + } + + let db = context.getAppDB() + await db.bulkDocs(rows) } } From 77729737bce3cceabcef7290bbf46321939bccd4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Oct 2023 09:47:50 +0100 Subject: [PATCH 06/56] First test passes! --- packages/server/src/sdk/app/rows/external.ts | 2 +- packages/server/src/sdk/app/rows/search.ts | 8 ++++++-- packages/server/src/sdk/app/rows/search/external.ts | 6 ++++++ packages/server/src/sdk/app/rows/search/internal.ts | 9 ++++----- packages/server/src/sdk/app/tables/migration.ts | 4 ++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 8bcf89a3f5..beae02e134 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -1,4 +1,4 @@ -import { IncludeRelationship, Operation, Row } from "@budibase/types" +import { IncludeRelationship, Operation } from "@budibase/types" import { handleRequest } from "../../../api/controllers/row/external" import { breakRowIdField } from "../../../integrations/utils" diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f75bd07437..221aa6486c 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,4 +1,4 @@ -import { SearchFilters, SearchParams } from "@budibase/types" +import { Row, SearchFilters, SearchParams } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" @@ -45,10 +45,14 @@ export async function exportRows( return pickApi(options.tableId).exportRows(options) } -export async function fetch(tableId: string) { +export async function fetch(tableId: string): Promise { return pickApi(tableId).fetch(tableId) } +export async function fetchRaw(tableId: string): Promise { + return pickApi(tableId).fetchRaw(tableId) +} + export async function fetchView( tableId: string, viewName: string, diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index c41efad171..981ae1bf8d 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -186,6 +186,12 @@ export async function fetch(tableId: string): Promise { }) } +export async function fetchRaw(tableId: string): Promise { + return await handleRequest(Operation.READ, tableId, { + includeSqlRelationships: IncludeRelationship.INCLUDE, + }) +} + export async function fetchView(viewName: string) { // there are no views in external datasources, shouldn't ever be called // for now just fetch diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 779ff5f777..58611c8849 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -140,14 +140,13 @@ export async function exportRows( } export async function fetch(tableId: string): Promise { - const db = context.getAppDB() - const table = await sdk.tables.getTable(tableId) - const rows = await getRawTableData(db, tableId) + const rows = await fetchRaw(tableId) return await outputProcessing(table, rows) } -async function getRawTableData(db: Database, tableId: string) { +export async function fetchRaw(tableId: string): Promise { + const db = context.getAppDB() let rows if (tableId === InternalTables.USER_METADATA) { rows = await sdk.users.fetchMetadata() @@ -182,7 +181,7 @@ export async function fetchView( }) } else { const tableId = viewInfo.meta.tableId - const data = await getRawTableData(db, tableId) + const data = await fetchRaw(tableId) response = await inMemoryViews.runView( viewInfo, calculation as string, diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index a08b56826d..cea15114a1 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -28,7 +28,7 @@ export async function migrate( await migrator.doMigration() delete table.schema[oldColumn.name] - await sdk.tables.saveTable(table) + table = await sdk.tables.saveTable(table) await updateLinks({ eventType: EventType.TABLE_UPDATED, table, oldTable }) } @@ -88,7 +88,7 @@ class UserColumnMigrator implements ColumnMigrator { ) {} async doMigration() { - let rows = await sdk.rows.fetch(this.table._id!) + let rows = await sdk.rows.fetchRaw(this.table._id!) let rowsById = rows.reduce((acc, row) => { acc[row._id!] = row return acc From a3ad8780deb6089bc96d1a66d21b0ed9ebf30c2e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Oct 2023 17:28:55 +0100 Subject: [PATCH 07/56] Implement many-to-many user column migrations. --- .../server/src/api/routes/tests/table.spec.ts | 95 +++++++++++++++++-- packages/server/src/sdk/app/rows/search.ts | 2 +- .../server/src/sdk/app/tables/migration.ts | 66 ++++++++++++- .../src/tests/utilities/TestConfiguration.ts | 3 +- .../server/src/tests/utilities/api/index.ts | 1 + 5 files changed, 154 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index dcf1704ed0..907b020246 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -8,6 +8,7 @@ import { AutoFieldSubTypes, InternalTable, FieldSubtype, + Row, } from "@budibase/types" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" @@ -421,8 +422,13 @@ describe("/tables", () => { }) describe("migrate", () => { - it("should successfully migrate a user relationship to a user column", async () => { - const users = await config.api.row.fetch(InternalTable.USER_METADATA) + it("should successfully migrate a one-to-many user relationship to a user column", async () => { + const users = await Promise.all([ + config.createUser({ email: "1@example.com" }), + config.createUser({ email: "2@example.com" }), + config.createUser({ email: "3@example.com" }), + ]) + const table = await config.api.table.create({ name: "table", type: "table", @@ -441,9 +447,11 @@ describe("/tables", () => { }, }) - await config.api.row.save(table._id!, { - "user relationship": users, - }) + const rows = await Promise.all( + users.map(u => + config.api.row.save(table._id!, { "user relationship": [u] }) + ) + ) await config.api.table.migrate(table._id!, { oldColumn: table.schema["user relationship"], @@ -458,9 +466,80 @@ describe("/tables", () => { expect(migratedTable.schema["user column"]).toBeDefined() expect(migratedTable.schema["user relationship"]).not.toBeDefined() - const rows = await config.api.row.fetch(table._id!) - expect(rows[0]["user column"]).toBeDefined() - expect(rows[0]["user relationship"]).not.toBeDefined() + const migratedRows = await config.api.row.fetch(table._id!) + + rows.sort((a, b) => a._id!.localeCompare(b._id!)) + migratedRows.sort((a, b) => a._id!.localeCompare(b._id!)) + + for (const [i, row] of rows.entries()) { + const migratedRow = migratedRows[i] + expect(migratedRow["user column"]).toBeDefined() + expect(migratedRow["user relationship"]).not.toBeDefined() + expect(row["user relationship"][0]._id).toEqual( + migratedRow["user column"][0]._id + ) + } + }) + + it("should successfully migrate a many-to-many user relationship to a users column", async () => { + const users = await Promise.all([ + config.createUser({ email: "1@example.com" }), + config.createUser({ email: "2@example.com" }), + config.createUser({ email: "3@example.com" }), + ]) + + const table = await config.api.table.create({ + name: "table", + type: "table", + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.MANY_TO_MANY, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + const row1 = await config.api.row.save(table._id!, { + "user relationship": [users[0], users[1]], + }) + + const row2 = await config.api.row.save(table._id!, { + "user relationship": [users[2]], + }) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() + + const row1Migrated = (await config.api.row.get(table._id!, row1._id!)) + .body as Row + expect(row1Migrated["user relationship"]).not.toBeDefined() + expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual( + expect.arrayContaining([users[0]._id, users[1]._id]) + ) + + const row2Migrated = (await config.api.row.get(table._id!, row2._id!)) + .body as Row + expect(row2Migrated["user relationship"]).not.toBeDefined() + expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual([ + users[2]._id, + ]) }) }) }) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 221aa6486c..caf9ffea02 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -57,6 +57,6 @@ export async function fetchView( tableId: string, viewName: string, params: ViewParams -) { +): Promise { return pickApi(tableId).fetchView(viewName, params) } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index cea15114a1..e3f817f893 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -2,8 +2,12 @@ import { BadRequestError, context } from "@budibase/backend-core" import { BBReferenceFieldMetadata, FieldSchema, + FieldSubtype, InternalTable, + ManyToManyRelationshipFieldMetadata, + OneToManyRelationshipFieldMetadata, RelationshipFieldMetadata, + RelationshipType, Row, Table, isBBReferenceField, @@ -77,13 +81,30 @@ function getColumnMigrator( ) } - return new UserColumnMigrator(table, oldColumn, newColumn) + if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) { + if (newColumn.subtype !== FieldSubtype.USER) { + throw new BadRequestError( + `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column` + ) + } + return new OneToManyUserColumnMigrator(table, oldColumn, newColumn) + } + if (oldColumn.relationshipType === RelationshipType.MANY_TO_MANY) { + if (newColumn.subtype !== FieldSubtype.USERS) { + throw new BadRequestError( + `Column "${oldColumn.name}" is a many-to-many column but "${newColumn.name}" is not a multi user column` + ) + } + return new ManyToManyUserColumnMigrator(table, oldColumn, newColumn) + } + + throw new BadRequestError(`Unknown migration type`) } -class UserColumnMigrator implements ColumnMigrator { +class OneToManyUserColumnMigrator implements ColumnMigrator { constructor( private table: Table, - private oldColumn: RelationshipFieldMetadata, + private oldColumn: OneToManyRelationshipFieldMetadata, private newColumn: BBReferenceFieldMetadata ) {} @@ -115,3 +136,42 @@ class UserColumnMigrator implements ColumnMigrator { await db.bulkDocs(rows) } } + +class ManyToManyUserColumnMigrator implements ColumnMigrator { + constructor( + private table: Table, + private oldColumn: ManyToManyRelationshipFieldMetadata, + private newColumn: BBReferenceFieldMetadata + ) {} + + async doMigration() { + let rows = await sdk.rows.fetchRaw(this.table._id!) + let rowsById = rows.reduce((acc, row) => { + acc[row._id!] = row + return acc + }, {} as Record) + + let links = await sdk.links.fetchWithDocument(this.table._id!) + for (let link of links) { + if (link.doc1.tableId !== this.table._id) { + continue + } + if (link.doc1.fieldName !== this.oldColumn.name) { + continue + } + if (link.doc2.tableId !== InternalTable.USER_METADATA) { + continue + } + + let userId = dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId) + let row = rowsById[link.doc1.rowId] + if (!row[this.newColumn.name]) { + row[this.newColumn.name] = [] + } + row[this.newColumn.name].push(userId) + } + + let db = context.getAppDB() + await db.bulkDocs(rows) + } +} diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cec8c8aa12..c00772ef63 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -55,6 +55,7 @@ import { RelationshipType, CreateViewRequest, RelationshipFieldMetadata, + User, } from "@budibase/types" import API from "./api" @@ -254,7 +255,7 @@ class TestConfiguration { } catch (err) { existing = { email } } - const user = { + const user: User = { _id: id, ...existing, roles: roles || {}, diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fce8237760..1352874914 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -27,5 +27,6 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) + this.global = new GlobalAPI(config) } } From 2d26597d07d8c4042de04b6a68c8178402e7164e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 09:49:57 +0100 Subject: [PATCH 08/56] Fix tests after merge. --- packages/server/src/sdk/app/tables/index.ts | 2 ++ packages/server/src/sdk/app/tables/migration.ts | 3 ++- packages/server/src/sdk/app/tables/update.ts | 13 ++++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 8542250517..ed71498d44 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -2,10 +2,12 @@ import { populateExternalTableSchemas } from "./validation" import * as getters from "./getters" import * as updates from "./update" import * as utils from "./utils" +import { migrate } from "./migration" export default { populateExternalTableSchemas, ...updates, ...getters, ...utils, + migrate, } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index cea15114a1..76b824959c 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -23,7 +23,8 @@ export async function migrate( let migrator = getColumnMigrator(table, oldColumn, newColumn) let oldTable = cloneDeep(table) - table = await sdk.tables.addColumn(table, newColumn) + table.schema[newColumn.name] = newColumn + table = await sdk.tables.saveTable(table) await migrator.doMigration() diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts index 9bba4a967e..c59f8a8a42 100644 --- a/packages/server/src/sdk/app/tables/update.ts +++ b/packages/server/src/sdk/app/tables/update.ts @@ -3,21 +3,28 @@ import { isExternalTable } from "../../../integrations/utils" import sdk from "../../index" import { context } from "@budibase/backend-core" import { isExternal } from "./utils" +import { DocumentInsertResponse } from "@budibase/nano" import * as external from "./external" import * as internal from "./internal" +import { cloneDeep } from "lodash" export * as external from "./external" export * as internal from "./internal" -export async function saveTable(table: Table) { +export async function saveTable(table: Table): Promise
{ const db = context.getAppDB() + let resp: DocumentInsertResponse if (isExternalTable(table._id!)) { const datasource = await sdk.datasources.get(table.sourceId!) datasource.entities![table.name] = table - await db.put(datasource) + resp = await db.put(datasource) } else { - await db.put(table) + resp = await db.put(table) } + + let tableClone = cloneDeep(table) + tableClone._rev = resp.rev + return tableClone } export async function update(table: Table, renaming?: RenameColumn) { From 9dd16381a7e1c83eb3339f1790289af14b7b8774 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 09:52:17 +0100 Subject: [PATCH 09/56] Merge base branch. --- packages/server/src/tests/utilities/api/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index 1352874914..fce8237760 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -27,6 +27,5 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) - this.global = new GlobalAPI(config) } } From 06f0a8da1ae5ee15f4e583a526e81f8c93eb6fef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 09:59:11 +0100 Subject: [PATCH 10/56] Update pro submodule. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 570d14aa44..56d968bfe6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 +Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9 From 2e0b528331cbc08f9cac96da2dbb19d0548283f5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 09:59:27 +0100 Subject: [PATCH 11/56] Update pro submodule. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 570d14aa44..56d968bfe6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 +Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9 From 9e279be4c334c5e613066a59fe0ab8b42a842e0c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 10:35:41 +0100 Subject: [PATCH 12/56] Put pro back to where it was. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 56d968bfe6..570d14aa44 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9 +Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 From 3fc2ff2c9b1bf30eea4a2cf96c217935a283c0eb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 10:36:05 +0100 Subject: [PATCH 13/56] Put pro back to where it was. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 56d968bfe6..570d14aa44 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 56d968bfe6998e1077d3fce4eb1c9e483d1d6fc9 +Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 From febfab092717f106ef13f9827221e67d74c244b8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 10:48:10 +0100 Subject: [PATCH 14/56] Fix tests/types. --- packages/server/src/sdk/users/tests/utils.spec.ts | 14 +++++++------- .../src/tests/utilities/TestConfiguration.ts | 7 ++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 5c6777df59..f7c9413ebd 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -39,12 +39,12 @@ describe("syncGlobalUsers", () => { expect(metadata).toHaveLength(3) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user1._id), + _id: db.generateUserMetadataID(user1._id!), }) ) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user2._id), + _id: db.generateUserMetadataID(user2._id!), }) ) }) @@ -59,7 +59,7 @@ describe("syncGlobalUsers", () => { expect(metadata).toHaveLength(1) expect(metadata).not.toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user._id), + _id: db.generateUserMetadataID(user._id!), }) ) }) @@ -70,7 +70,7 @@ describe("syncGlobalUsers", () => { const group = await proSdk.groups.save(structures.userGroups.userGroup()) const user1 = await config.createUser({ admin: false, builder: false }) const user2 = await config.createUser({ admin: false, builder: false }) - await proSdk.groups.addUsers(group.id, [user1._id, user2._id]) + await proSdk.groups.addUsers(group.id, [user1._id!, user2._id!]) await config.doInContext(config.appId, async () => { await syncGlobalUsers() @@ -87,12 +87,12 @@ describe("syncGlobalUsers", () => { expect(metadata).toHaveLength(3) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user1._id), + _id: db.generateUserMetadataID(user1._id!), }) ) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user2._id), + _id: db.generateUserMetadataID(user2._id!), }) ) }) @@ -109,7 +109,7 @@ describe("syncGlobalUsers", () => { { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC }, ], }) - await proSdk.groups.addUsers(group.id, [user1._id, user2._id]) + await proSdk.groups.addUsers(group.id, [user1._id!, user2._id!]) await config.doInContext(config.appId, async () => { await syncGlobalUsers() diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index c00772ef63..5f5b211975 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -295,7 +295,7 @@ class TestConfiguration { admin?: boolean roles?: UserRoles } = {} - ) { + ): Promise { let { id, firstName, lastName, email, builder, admin, roles } = user firstName = firstName || this.defaultUserValues.firstName lastName = lastName || this.defaultUserValues.lastName @@ -315,10 +315,7 @@ class TestConfiguration { roles, }) await cache.user.invalidateUser(globalId) - return { - ...resp, - globalId, - } + return resp } async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) { From 5e6ed0fd670f762fb5b4ec4f084697baa1264135 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 11:54:27 +0100 Subject: [PATCH 15/56] Implement many-to-one user column migration. --- .../server/src/api/routes/tests/table.spec.ts | 61 +++++++++++++++++++ .../server/src/sdk/app/tables/migration.ts | 21 ++++--- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 907b020246..420717f7f0 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -510,6 +510,67 @@ describe("/tables", () => { "user relationship": [users[0], users[1]], }) + const row2 = await config.api.row.save(table._id!, { + "user relationship": [users[1], users[2]], + }) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() + + const row1Migrated = (await config.api.row.get(table._id!, row1._id!)) + .body as Row + expect(row1Migrated["user relationship"]).not.toBeDefined() + expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual( + expect.arrayContaining([users[0]._id, users[1]._id]) + ) + + const row2Migrated = (await config.api.row.get(table._id!, row2._id!)) + .body as Row + expect(row2Migrated["user relationship"]).not.toBeDefined() + expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual( + expect.arrayContaining([users[1]._id, users[2]._id]) + ) + }) + + it("should successfully migrate a many-to-one user relationship to a users column", async () => { + const users = await Promise.all([ + config.createUser({ email: "1@example.com" }), + config.createUser({ email: "2@example.com" }), + config.createUser({ email: "3@example.com" }), + ]) + + const table = await config.api.table.create({ + name: "table", + type: "table", + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.MANY_TO_ONE, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + const row1 = await config.api.row.save(table._id!, { + "user relationship": [users[0], users[1]], + }) + const row2 = await config.api.row.save(table._id!, { "user relationship": [users[2]], }) diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index 94b8841136..08c06b28b9 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -5,8 +5,8 @@ import { FieldSubtype, InternalTable, ManyToManyRelationshipFieldMetadata, + ManyToOneRelationshipFieldMetadata, OneToManyRelationshipFieldMetadata, - RelationshipFieldMetadata, RelationshipType, Row, Table, @@ -88,21 +88,24 @@ function getColumnMigrator( `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column` ) } - return new OneToManyUserColumnMigrator(table, oldColumn, newColumn) + return new SingleUserColumnMigrator(table, oldColumn, newColumn) } - if (oldColumn.relationshipType === RelationshipType.MANY_TO_MANY) { + if ( + oldColumn.relationshipType === RelationshipType.MANY_TO_MANY || + oldColumn.relationshipType === RelationshipType.MANY_TO_ONE + ) { if (newColumn.subtype !== FieldSubtype.USERS) { throw new BadRequestError( - `Column "${oldColumn.name}" is a many-to-many column but "${newColumn.name}" is not a multi user column` + `Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column` ) } - return new ManyToManyUserColumnMigrator(table, oldColumn, newColumn) + return new MultiUserColumnMigrator(table, oldColumn, newColumn) } throw new BadRequestError(`Unknown migration type`) } -class OneToManyUserColumnMigrator implements ColumnMigrator { +class SingleUserColumnMigrator implements ColumnMigrator { constructor( private table: Table, private oldColumn: OneToManyRelationshipFieldMetadata, @@ -138,10 +141,12 @@ class OneToManyUserColumnMigrator implements ColumnMigrator { } } -class ManyToManyUserColumnMigrator implements ColumnMigrator { +class MultiUserColumnMigrator implements ColumnMigrator { constructor( private table: Table, - private oldColumn: ManyToManyRelationshipFieldMetadata, + private oldColumn: + | ManyToManyRelationshipFieldMetadata + | ManyToOneRelationshipFieldMetadata, private newColumn: BBReferenceFieldMetadata ) {} From a701933f48ad5f4f141cd85d18f9f86938cbe410 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Oct 2023 17:57:25 +0100 Subject: [PATCH 16/56] Frontend changes for the user column migration work. --- packages/frontend-core/src/api/tables.js | 9 +++++++ .../components/grid/cells/HeaderCell.svelte | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/packages/frontend-core/src/api/tables.js b/packages/frontend-core/src/api/tables.js index a08e35d3d8..34d2371e1a 100644 --- a/packages/frontend-core/src/api/tables.js +++ b/packages/frontend-core/src/api/tables.js @@ -140,4 +140,13 @@ export const buildTableEndpoints = API => ({ }, }) }, + migrateColumn: async ({ tableId, oldColumn, newColumn }) => { + return await API.post({ + url: `/api/tables/${tableId}/migrate`, + body: { + oldColumn, + newColumn, + }, + }) + }, }) diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index d6cbcb582d..01c759a15c 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -10,6 +10,7 @@ export let orderable = true const { + API, reorder, isReordering, isResizing, @@ -114,6 +115,24 @@ open = false } + const migrateUserColumn = async () => { + let subtype = "users" + if (column.schema.relationshipType === "one-to-many") { + subtype = "user" + } + + await API.migrateColumn({ + tableId: $definition._id, + oldColumn: column.schema, + newColumn: { + name: `${column.schema.name} migrated`, + type: "bb_reference", + subtype, + }, + }) + open = false + } + const duplicateColumn = async () => { open = false @@ -262,6 +281,11 @@ > Hide column + {#if column.schema.type === "link" && column.schema.tableId === "ta_users"} + + Migrate to user column + + {/if} {/if} From 2f0a40e9bb8c8e8893517d6dfcca1433ff088173 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Oct 2023 15:18:46 +0100 Subject: [PATCH 17/56] Introduce modal to show warning to users, and toast to show success. --- .../backend/DataTable/TableDataTable.svelte | 6 +++ .../components/grid/cells/HeaderCell.svelte | 42 +++++++++---------- .../grid/controls/MigrationModal.svelte | 42 +++++++++++++++++++ 3 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 packages/frontend-core/src/components/grid/controls/MigrationModal.svelte diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte index 5fee849afb..22deacbe03 100644 --- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte @@ -53,6 +53,11 @@ await datasources.fetch() } } + + const refreshDefinitions = async () => { + await tables.fetch() + await datasources.fetch() + }
@@ -66,6 +71,7 @@ schemaOverrides={isUsersTable ? userSchemaOverrides : null} showAvatars={false} on:updatedatasource={handleGridTableUpdate} + on:refreshdefinitions={refreshDefinitions} > {#if isUsersTable && $store.features.disableUserMetadata} diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index 01c759a15c..9b05f9ef79 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -1,9 +1,17 @@ + + + +
Hide column - {#if column.schema.type === "link" && column.schema.tableId === "ta_users"} - + {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === "ta_users"} + Migrate to user column {/if} diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte new file mode 100644 index 0000000000..b262f93797 --- /dev/null +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -0,0 +1,42 @@ + + + + TODO: copy here + From d3670ddf21a43552aee52602c3a40acd7eea7f38 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Oct 2023 17:22:49 +0100 Subject: [PATCH 18/56] Add an input to allow the user to choose the new column name. --- .../backend/DataTable/TableDataTable.svelte | 1 + .../grid/controls/MigrationModal.svelte | 22 ++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte index 22deacbe03..40bcf2b74e 100644 --- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte @@ -55,6 +55,7 @@ } const refreshDefinitions = async () => { + console.log("woot") await tables.fetch() await datasources.fetch() } diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index b262f93797..e329038217 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -1,11 +1,18 @@
@@ -71,7 +66,6 @@ schemaOverrides={isUsersTable ? userSchemaOverrides : null} showAvatars={false} on:updatedatasource={handleGridTableUpdate} - on:refreshdefinitions={refreshDefinitions} > {#if isUsersTable && $store.features.disableUserMetadata} diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index ee7fd7bfbe..111ff86170 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -45,7 +45,6 @@ notifications.error(`Failed to migrate: ${e.message}`) } await rows.actions.refreshData() - dispatch("refreshdefintions") } From 7acce7b7c0b0a8ca4363263a8d728007c0eecfb8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:01:08 +0100 Subject: [PATCH 29/56] Remove unused dispatch import. --- .../src/components/grid/controls/MigrationModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index 111ff86170..ec2972a03e 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -8,7 +8,7 @@ import { getContext } from "svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" - const { API, dispatch, definition, rows } = getContext("grid") + const { API, definition, rows } = getContext("grid") export let column From 2c5dd99da202a45dfb97a5cb87351862ecc3627e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:37:15 +0100 Subject: [PATCH 30/56] Use FieldSubtype enum instead of raw strings. --- .../src/components/grid/controls/MigrationModal.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index ec2972a03e..0f173da8b9 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -7,6 +7,7 @@ } from "@budibase/bbui" import { getContext } from "svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" + import { FieldSubtype } from "@budibase/types" const { API, definition, rows } = getContext("grid") @@ -25,9 +26,9 @@ } const migrateUserColumn = async () => { - let subtype = "users" + let subtype = FieldSubtype.USERS if (column.schema.relationshipType === "one-to-many") { - subtype = "user" + subtype = FieldSubtype.USER } try { From ef84e96f98c7a38a6145fb47beee4bdbf5237a73 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:38:14 +0100 Subject: [PATCH 31/56] Use RelationshipType enum instead of raw string. --- .../src/components/grid/controls/MigrationModal.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index 0f173da8b9..c6a49c6a48 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -7,7 +7,7 @@ } from "@budibase/bbui" import { getContext } from "svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" - import { FieldSubtype } from "@budibase/types" + import { FieldSubtype, RelationshipType } from "@budibase/types" const { API, definition, rows } = getContext("grid") @@ -27,7 +27,7 @@ const migrateUserColumn = async () => { let subtype = FieldSubtype.USERS - if (column.schema.relationshipType === "one-to-many") { + if (column.schema.relationshipType === RelationshipType.ONE_TO_MANY) { subtype = FieldSubtype.USER } From c5097487e22438a0e4fbc961f2cb422c07329a00 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:38:55 +0100 Subject: [PATCH 32/56] Use FieldType constant instead of raw string. --- .../src/components/grid/controls/MigrationModal.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index c6a49c6a48..73ea32408b 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -7,7 +7,7 @@ } from "@budibase/bbui" import { getContext } from "svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" - import { FieldSubtype, RelationshipType } from "@budibase/types" + import { FieldSubtype, FieldType, RelationshipType } from "@budibase/types" const { API, definition, rows } = getContext("grid") @@ -37,7 +37,7 @@ oldColumn: column.schema, newColumn: { name: newColumnName, - type: "bb_reference", + type: FieldType.BB_REFERENCE, subtype, }, }) From e03b1be9d1b2c7c3e8c3ded8f0b3e304a23446bc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:41:37 +0100 Subject: [PATCH 33/56] Make sure new column name cannot be the same as an existing column name. --- .../src/components/grid/controls/MigrationModal.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte index 73ea32408b..1957c3259f 100644 --- a/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte +++ b/packages/frontend-core/src/components/grid/controls/MigrationModal.svelte @@ -17,8 +17,8 @@ $: error = checkNewColumnName(newColumnName) const checkNewColumnName = newColumnName => { - if (column.schema.name === newColumnName) { - return "New column name can't be the same as the existing column name." + if (newColumnName in $definition.schema) { + return "New column name can't be the same as an existing column name." } if (newColumnName.match(ValidColumnNameRegex) === null) { return "Illegal character; must be alpha-numeric." From 4a00649f7f7c01d1280a08ff35cedd4963779021 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:46:14 +0100 Subject: [PATCH 34/56] Simplify the function signature of processInternalTables --- packages/server/src/sdk/app/tables/getters.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 34cddc8dc7..47da0beb40 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -19,8 +19,8 @@ import { import datasources from "../datasources" import sdk from "../../../sdk" -function processInternalTables(docs: AllDocsResponse): Table[] { - return docs.rows.map(tableDoc => processInternalTable(tableDoc.doc)) +function processInternalTables(tables: Table[]): Table[] { + return tables.map(processInternalTable) } export function processInternalTable(table: Table): Table { @@ -40,7 +40,7 @@ export async function getAllInternalTables(db?: Database): Promise { include_docs: true, }) ) - return processInternalTables(internalTables) + return processInternalTables(internalTables.rows.map(row => row.doc!)) } async function getAllExternalTables(): Promise { @@ -110,7 +110,9 @@ export async function getTables(tableIds: string[]): Promise { const internalTableDocs = await db.allDocs( getMultiIDParams(internalTableIds) ) - tables = tables.concat(processInternalTables(internalTableDocs)) + tables = tables.concat( + processInternalTables(internalTableDocs.rows.map(row => row.doc!)) + ) } return tables } From 6c3b535863878c521bb312606218b89fa2f0f787 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Oct 2023 16:49:29 +0100 Subject: [PATCH 35/56] Simplify try-catch in the migrate function. --- packages/server/src/sdk/app/tables/migration.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index b678b64318..293f9184d6 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -34,10 +34,8 @@ export async function migrate( table = await sdk.tables.saveTable(table) let migrator = getColumnMigrator(table, oldColumn, newColumn) - let result: MigrationResult - try { - result = await migrator.doMigration() + return await migrator.doMigration() } catch (e) { // If the migration fails then we need to roll back the table schema // change. @@ -45,8 +43,6 @@ export async function migrate( await sdk.tables.saveTable(table) throw e } - - return result } interface ColumnMigrator { From 455b26bac95917be3b5d15176248498fd55635e9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 25 Oct 2023 19:00:25 +0100 Subject: [PATCH 36/56] Making sure the source ID is always set when creating a table - the frontend expects this to be set for every table so making the type represent this correctly. --- .../server/src/api/controllers/datasource.ts | 3 +- .../server/src/api/controllers/table/index.ts | 3 +- .../server/src/api/routes/tests/row.spec.ts | 6 +- .../server/src/api/routes/tests/table.spec.ts | 4 ++ .../src/api/routes/tests/viewV2.spec.ts | 2 + packages/server/src/constants/index.ts | 14 ++--- packages/server/src/db/utils.ts | 3 +- .../server/src/integrations/googlesheets.ts | 8 +-- .../src/integrations/microsoftSqlServer.ts | 6 +- packages/server/src/integrations/mysql.ts | 6 +- packages/server/src/integrations/oracle.ts | 6 +- packages/server/src/integrations/postgres.ts | 6 +- .../integrations/tests/googlesheets.spec.ts | 2 + packages/server/src/integrations/utils.ts | 9 ++- .../middleware/tests/trimViewRowInfo.spec.ts | 9 ++- .../src/sdk/app/rows/search/internal.ts | 6 +- .../app/rows/search/tests/external.spec.ts | 62 ++++++++++--------- .../app/rows/search/tests/internal.spec.ts | 9 ++- .../sdk/app/rows/search/tests/utils.spec.ts | 3 + .../src/sdk/app/tables/external/index.ts | 2 +- .../src/sdk/app/tables/external/utils.ts | 4 +- packages/server/src/sdk/app/tables/getters.ts | 10 +-- .../src/sdk/app/tables/tests/tables.spec.ts | 8 ++- .../sdk/app/tables/tests/validation.spec.ts | 41 ++++++++---- .../src/sdk/app/views/tests/views.spec.ts | 2 + .../src/tests/utilities/TestConfiguration.ts | 27 ++++++-- .../server/src/tests/utilities/structures.ts | 2 + .../tests/inputProcessing.spec.ts | 11 +++- .../tests/outputProcessing.spec.ts | 4 ++ packages/server/src/websockets/builder.ts | 2 +- .../types/src/documents/app/table/table.ts | 8 +-- packages/types/src/sdk/datasources.ts | 6 +- 32 files changed, 185 insertions(+), 109 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index b50c2464f0..5d024d51b6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -12,7 +12,6 @@ import { CreateDatasourceResponse, Datasource, DatasourcePlus, - ExternalTable, FetchDatasourceInfoRequest, FetchDatasourceInfoResponse, IntegrationBase, @@ -59,7 +58,7 @@ async function buildSchemaHelper(datasource: Datasource): Promise { const connector = (await getConnector(datasource)) as DatasourcePlus return await connector.buildSchema( datasource._id!, - datasource.entities! as Record + datasource.entities! as Record ) } diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index d2ad63c13e..c814a37ad9 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -24,7 +24,6 @@ import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" import { builderSocket } from "../../../websockets" import { cloneDeep, isEqual } from "lodash" -import { processInternalTable } from "../../../sdk/app/tables/getters" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -50,7 +49,7 @@ export async function fetch(ctx: UserCtx) { return Object.values(entities).map
((entity: Table) => ({ ...entity, type: "external", - sourceId: datasource._id, + sourceId: datasource._id!, sql: isSQL(datasource), })) } else { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 4c2e7a7494..e5a24c27e0 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -21,6 +21,7 @@ import { SortType, StaticQuotaName, Table, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import { expectAnyExternalColsAttributes, @@ -65,6 +66,7 @@ describe.each([ type: "table", primary: ["id"], primaryDisplay: "name", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { id: { type: FieldType.AUTO, @@ -880,6 +882,7 @@ describe.each([ async function userTable(): Promise
{ return { name: `users_${generator.word()}`, + sourceId: INTERNAL_TABLE_SOURCE_ID, type: "table", primary: ["id"], schema: { @@ -1062,6 +1065,7 @@ describe.each([ async function userTable(): Promise
{ return { name: `users_${generator.word()}`, + sourceId: INTERNAL_TABLE_SOURCE_ID, type: "table", primary: ["id"], schema: { @@ -1597,7 +1601,7 @@ describe.each([ const tableConfig = generateTableConfig() if (config.datasource) { - tableConfig.sourceId = config.datasource._id + tableConfig.sourceId = config.datasource._id! if (config.datasource.plus) { tableConfig.type = "external" } diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 420717f7f0..f988b44e0f 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -9,6 +9,7 @@ import { InternalTable, FieldSubtype, Row, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" @@ -432,6 +433,7 @@ describe("/tables", () => { const table = await config.api.table.create({ name: "table", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { "user relationship": { type: FieldType.LINK, @@ -491,6 +493,7 @@ describe("/tables", () => { const table = await config.api.table.create({ name: "table", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { "user relationship": { type: FieldType.LINK, @@ -552,6 +555,7 @@ describe("/tables", () => { const table = await config.api.table.create({ name: "table", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { "user relationship": { type: FieldType.LINK, diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 40060aef48..a1ce5ec10e 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -10,6 +10,7 @@ import { UIFieldMetadata, UpdateViewRequest, ViewV2, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import { generator } from "@budibase/backend-core/tests" import { generateDatasourceID } from "../../../db/utils" @@ -18,6 +19,7 @@ function priceTable(): Table { return { name: "table", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { Price: { type: FieldType.NUMBER, diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index b37a4b36c1..fe69b3c9c8 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -1,5 +1,9 @@ import { objectStore, roles, constants } from "@budibase/backend-core" -import { FieldType as FieldTypes } from "@budibase/types" +import { + FieldType as FieldTypes, + Table, + INTERNAL_TABLE_SOURCE_ID, +} from "@budibase/types" export { FieldType as FieldTypes, RelationshipType, @@ -70,9 +74,10 @@ export enum SortDirection { DESCENDING = "DESCENDING", } -export const USERS_TABLE_SCHEMA = { +export const USERS_TABLE_SCHEMA: Table = { _id: "ta_users", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, views: {}, name: "Users", // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out @@ -87,12 +92,10 @@ export const USERS_TABLE_SCHEMA = { }, presence: true, }, - fieldName: "email", name: "email", }, firstName: { name: "firstName", - fieldName: "firstName", type: FieldTypes.STRING, constraints: { type: FieldTypes.STRING, @@ -101,7 +104,6 @@ export const USERS_TABLE_SCHEMA = { }, lastName: { name: "lastName", - fieldName: "lastName", type: FieldTypes.STRING, constraints: { type: FieldTypes.STRING, @@ -109,7 +111,6 @@ export const USERS_TABLE_SCHEMA = { }, }, roleId: { - fieldName: "roleId", name: "roleId", type: FieldTypes.OPTIONS, constraints: { @@ -119,7 +120,6 @@ export const USERS_TABLE_SCHEMA = { }, }, status: { - fieldName: "status", name: "status", type: FieldTypes.OPTIONS, constraints: { diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 2c07bd8d22..d532d8a8b2 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -5,6 +5,7 @@ import { FieldSchema, RelationshipFieldMetadata, VirtualDocumentType, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import { FieldTypes } from "../constants" export { DocumentType, VirtualDocumentType } from "@budibase/types" @@ -18,7 +19,7 @@ export const enum AppStatus { } export const BudibaseInternalDB = { - _id: "bb_internal", + _id: INTERNAL_TABLE_SOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, name: "Budibase DB", source: "BUDIBASE", diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 57b6682cc8..4433f45863 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -12,7 +12,7 @@ import { Row, SearchFilters, SortJson, - ExternalTable, + Table, TableRequest, Schema, } from "@budibase/types" @@ -262,7 +262,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { id?: string ) { // base table - const table: ExternalTable = { + const table: Table = { name: title, primary: [GOOGLE_SHEETS_PRIMARY_KEY], schema: {}, @@ -283,7 +283,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { // not fully configured yet if (!this.config.auth) { @@ -291,7 +291,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } await this.connect() const sheets = this.client.sheetsByIndex - const tables: Record = {} + const tables: Record = {} let errors: Record = {} await utils.parallelForeach( sheets, diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index ff68026369..b86286756c 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -2,7 +2,7 @@ import { DatasourceFieldType, Integration, Operation, - ExternalTable, + Table, TableSchema, QueryJson, QueryType, @@ -380,7 +380,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { */ async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { await this.connect() let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL) @@ -394,7 +394,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { .map((record: any) => record.TABLE_NAME) .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1) - const tables: Record = {} + const tables: Record = {} for (let tableName of tableNames) { // get the column definition (type) const definition = await this.runSQL( diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 3a954da9bd..fe7eae51be 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -4,7 +4,7 @@ import { QueryType, QueryJson, SqlQuery, - ExternalTable, + Table, TableSchema, DatasourcePlus, DatasourceFeature, @@ -278,9 +278,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { - const tables: { [key: string]: ExternalTable } = {} + const tables: { [key: string]: Table } = {} await this.connect() try { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index b3936320ac..5fde565180 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -5,7 +5,7 @@ import { QueryJson, QueryType, SqlQuery, - ExternalTable, + Table, DatasourcePlus, DatasourceFeature, ConnectionInfo, @@ -263,14 +263,14 @@ class OracleIntegration extends Sql implements DatasourcePlus { */ async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { const columnsResponse = await this.internalQuery({ sql: this.COLUMNS_SQL, }) const oracleTables = this.mapColumns(columnsResponse) - const tables: { [key: string]: ExternalTable } = {} + const tables: { [key: string]: Table } = {} // iterate each table Object.values(oracleTables).forEach(oracleTable => { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 8479cd05d8..38339499b2 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -5,7 +5,7 @@ import { QueryType, QueryJson, SqlQuery, - ExternalTable, + Table, DatasourcePlus, DatasourceFeature, ConnectionInfo, @@ -273,7 +273,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { */ async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { let tableKeys: { [key: string]: string[] } = {} await this.openConnection() @@ -300,7 +300,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { const columnsResponse: { rows: PostgresColumn[] } = await this.client.query(this.COLUMNS_SQL) - const tables: { [key: string]: ExternalTable } = {} + const tables: { [key: string]: Table } = {} for (let column of columnsResponse.rows) { const tableName: string = column.table_name diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 748baddc39..842b681867 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -31,6 +31,7 @@ import { structures } from "@budibase/backend-core/tests" import TestConfiguration from "../../tests/utilities/TestConfiguration" import GoogleSheetsIntegration from "../googlesheets" import { FieldType, Table, TableSchema } from "@budibase/types" +import { generateDatasourceID } from "../../db/utils" describe("Google Sheets Integration", () => { let integration: any, @@ -61,6 +62,7 @@ describe("Google Sheets Integration", () => { function createBasicTable(name: string, columns: string[]): Table { return { name, + sourceId: generateDatasourceID(), schema: { ...columns.reduce((p, c) => { p[c] = { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b37fe9f0ed..9e6c5ca3af 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -4,7 +4,6 @@ import { SearchFilters, Datasource, FieldType, - ExternalTable, } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" import { InvalidColumns, NoEmptyFilterStrings } from "../constants" @@ -301,9 +300,9 @@ function copyExistingPropsOver( * @param entities The old list of tables, if there was any to look for definitions in. */ export function finaliseExternalTables( - tables: Record, - entities: Record -): Record { + tables: Record, + entities: Record +): Record { let finalTables: Record = {} const tableIds = Object.values(tables).map(table => table._id!) for (let [name, table] of Object.entries(tables)) { @@ -316,7 +315,7 @@ export function finaliseExternalTables( } export function checkExternalTables( - tables: Record + tables: Record ): Record { const invalidColumns = Object.values(InvalidColumns) as string[] const errors: Record = {} diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts index bf717d5828..106129f6c9 100644 --- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts +++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts @@ -1,5 +1,11 @@ import { generator } from "@budibase/backend-core/tests" -import { BBRequest, FieldType, Row, Table } from "@budibase/types" +import { + BBRequest, + FieldType, + Row, + Table, + INTERNAL_TABLE_SOURCE_ID, +} from "@budibase/types" import * as utils from "../../db/utils" import trimViewRowInfoMiddleware from "../trimViewRowInfo" @@ -73,6 +79,7 @@ describe("trimViewRowInfo middleware", () => { const table: Table = { _id: tableId, name: generator.word(), + sourceId: INTERNAL_TABLE_SOURCE_ID, type: "table", schema: { name: { diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 58611c8849..1aec8a321e 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -197,11 +197,7 @@ export async function fetchView( try { table = await sdk.tables.getTable(viewInfo.meta.tableId) } catch (err) { - /* istanbul ignore next */ - table = { - name: "", - schema: {}, - } + throw new Error("Unable to retrieve view table.") } rows = await outputProcessing(table, response.rows) } diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index b3bddfbc97..1afdca35fa 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -15,6 +15,7 @@ import { expectAnyExternalColsAttributes, generator, } from "@budibase/backend-core/tests" +import datasource from "../../../../../api/routes/datasource" jest.unmock("mysql2/promise") @@ -23,36 +24,7 @@ jest.setTimeout(30000) describe.skip("external", () => { const config = new TestConfiguration() - let externalDatasource: Datasource - - const tableData: Table = { - name: generator.word(), - type: "external", - primary: ["id"], - schema: { - id: { - name: "id", - type: FieldType.AUTO, - autocolumn: true, - }, - name: { - name: "name", - type: FieldType.STRING, - }, - surname: { - name: "surname", - type: FieldType.STRING, - }, - age: { - name: "age", - type: FieldType.NUMBER, - }, - address: { - name: "address", - type: FieldType.STRING, - }, - }, - } + let externalDatasource: Datasource, tableData: Table beforeAll(async () => { const container = await new GenericContainer("mysql") @@ -84,6 +56,36 @@ describe.skip("external", () => { }, }, }) + + tableData = { + name: generator.word(), + type: "external", + primary: ["id"], + sourceId: externalDatasource._id!, + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + surname: { + name: "surname", + type: FieldType.STRING, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + address: { + name: "address", + type: FieldType.STRING, + }, + }, + } }) describe("search", () => { diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts index b3e98a1149..b3cac2321e 100644 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -1,4 +1,10 @@ -import { FieldType, Row, Table, SearchParams } from "@budibase/types" +import { + FieldType, + Row, + Table, + SearchParams, + INTERNAL_TABLE_SOURCE_ID, +} from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../internal" import { @@ -12,6 +18,7 @@ describe("internal", () => { const tableData: Table = { name: generator.word(), type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { name: "name", diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index d946eea432..428c57be64 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -5,12 +5,14 @@ import { FieldTypeSubtypes, Table, SearchParams, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" const tableId = "ta_a" const tableWithUserCol: Table = { _id: tableId, name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { user: { name: "user", @@ -23,6 +25,7 @@ const tableWithUserCol: Table = { const tableWithUsersCol: Table = { _id: tableId, name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { user: { name: "user", diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 402baada78..f445fcaf08 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -35,10 +35,10 @@ export async function save( opts?: { tableId?: string; renaming?: RenameColumn } ) { let tableToSave: TableRequest = { + ...update, type: "table", _id: buildExternalTableId(datasourceId, update.name), sourceId: datasourceId, - ...update, } const tableId = opts?.tableId || update._id diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index 10c755a7d6..a60667f44f 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -76,12 +76,14 @@ export function generateManyLinkSchema( const primary = table.name + table.primary[0] const relatedPrimary = relatedTable.name + relatedTable.primary[0] const jcTblName = generateJunctionTableName(column, table, relatedTable) + const datasourceId = datasource._id! // first create the new table const junctionTable = { - _id: buildExternalTableId(datasource._id!, jcTblName), + _id: buildExternalTableId(datasourceId, jcTblName), name: jcTblName, primary: [primary, relatedPrimary], constrained: [primary, relatedPrimary], + sourceId: datasourceId, schema: { [primary]: foreignKeyStructure(primary, { toTable: table.name, diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 47da0beb40..af779bcc2b 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,20 +1,16 @@ import { context } from "@budibase/backend-core" -import { - BudibaseInternalDB, - getMultiIDParams, - getTableParams, -} from "../../../db/utils" +import { getMultiIDParams, getTableParams } from "../../../db/utils" import { breakExternalTableId, isExternalTable, isSQL, } from "../../../integrations/utils" import { - AllDocsResponse, Database, Table, TableResponse, TableViewsResponse, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" @@ -27,7 +23,7 @@ export function processInternalTable(table: Table): Table { return { ...table, type: "internal", - sourceId: table.sourceId || BudibaseInternalDB._id, + sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, } } diff --git a/packages/server/src/sdk/app/tables/tests/tables.spec.ts b/packages/server/src/sdk/app/tables/tests/tables.spec.ts index 78ebe59f01..ab45eeb8bb 100644 --- a/packages/server/src/sdk/app/tables/tests/tables.spec.ts +++ b/packages/server/src/sdk/app/tables/tests/tables.spec.ts @@ -1,4 +1,9 @@ -import { FieldType, Table, ViewV2 } from "@budibase/types" +import { + FieldType, + INTERNAL_TABLE_SOURCE_ID, + Table, + ViewV2, +} from "@budibase/types" import { generator } from "@budibase/backend-core/tests" import sdk from "../../.." @@ -13,6 +18,7 @@ describe("table sdk", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/sdk/app/tables/tests/validation.spec.ts b/packages/server/src/sdk/app/tables/tests/validation.spec.ts index 5347eede90..62a9c61dbb 100644 --- a/packages/server/src/sdk/app/tables/tests/validation.spec.ts +++ b/packages/server/src/sdk/app/tables/tests/validation.spec.ts @@ -1,36 +1,50 @@ import { populateExternalTableSchemas } from "../validation" import { cloneDeep } from "lodash/fp" -import { AutoReason, Datasource, Table } from "@budibase/types" +import { + AutoReason, + Datasource, + FieldType, + RelationshipType, + SourceName, + Table, +} from "@budibase/types" import { isEqual } from "lodash" +import { generateDatasourceID } from "../../../../db/utils" -const SCHEMA = { +const datasourceId = generateDatasourceID() + +const SCHEMA: Datasource = { + source: SourceName.POSTGRES, + type: "datasource", + _id: datasourceId, entities: { client: { _id: "tableA", name: "client", primary: ["idC"], primaryDisplay: "Name", + sourceId: datasourceId, schema: { idC: { autocolumn: true, externalType: "int unsigned", name: "idC", - type: "number", + type: FieldType.NUMBER, }, Name: { autocolumn: false, externalType: "varchar(255)", name: "Name", - type: "string", + type: FieldType.STRING, }, project: { fieldName: "idC", foreignKey: "idC", main: true, name: "project", - relationshipType: "many-to-one", + relationshipType: RelationshipType.MANY_TO_ONE, tableId: "tableB", - type: "link", + type: FieldType.LINK, }, }, }, @@ -39,31 +53,32 @@ const SCHEMA = { name: "project", primary: ["idP"], primaryDisplay: "Name", + sourceId: datasourceId, schema: { idC: { externalType: "int unsigned", name: "idC", - type: "number", + type: FieldType.NUMBER, }, idP: { autocolumn: true, externalType: "int unsigned", name: "idProject", - type: "number", + type: FieldType.NUMBER, }, Name: { autocolumn: false, externalType: "varchar(255)", name: "Name", - type: "string", + type: FieldType.STRING, }, client: { fieldName: "idC", foreignKey: "idC", name: "client", - relationshipType: "one-to-many", + relationshipType: RelationshipType.ONE_TO_MANY, tableId: "tableA", - type: "link", + type: FieldType.LINK, }, }, sql: true, @@ -95,12 +110,12 @@ describe("validation and update of external table schemas", () => { function noOtherTableChanges(response: any) { checkOtherColumns( response.entities!.client!, - SCHEMA.entities.client as Table, + SCHEMA.entities!.client, OTHER_CLIENT_COLS ) checkOtherColumns( response.entities!.project!, - SCHEMA.entities.project as Table, + SCHEMA.entities!.project, OTHER_PROJECT_COLS ) } diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts index 8fcc6405ef..2068eda4c3 100644 --- a/packages/server/src/sdk/app/views/tests/views.spec.ts +++ b/packages/server/src/sdk/app/views/tests/views.spec.ts @@ -2,6 +2,7 @@ import _ from "lodash" import { FieldSchema, FieldType, + INTERNAL_TABLE_SOURCE_ID, Table, TableSchema, ViewV2, @@ -14,6 +15,7 @@ describe("table sdk", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 5f5b211975..3d4046b91b 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -56,6 +56,7 @@ import { CreateViewRequest, RelationshipFieldMetadata, User, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" import API from "./api" @@ -68,6 +69,10 @@ type DefaultUserValues = { csrfToken: string } +interface TableToBuild extends Omit { + sourceId?: string +} + class TestConfiguration { server: any request: supertest.SuperTest | undefined @@ -538,10 +543,13 @@ class TestConfiguration { // TABLE async updateTable( - config?: Table, + config?: TableToBuild, { skipReassigning } = { skipReassigning: false } ): Promise
{ config = config || basicTable() + if (!config.sourceId) { + config.sourceId = INTERNAL_TABLE_SOURCE_ID + } const response = await this._req(config, null, controllers.table.save) if (!skipReassigning) { this.table = response @@ -549,13 +557,19 @@ class TestConfiguration { return response } - async createTable(config?: Table, options = { skipReassigning: false }) { + async createTable( + config?: TableToBuild, + options = { skipReassigning: false } + ) { if (config != null && config._id) { delete config._id } config = config || basicTable() + if (!config.sourceId) { + config.sourceId = INTERNAL_TABLE_SOURCE_ID + } if (this.datasource && !config.sourceId) { - config.sourceId = this.datasource._id + config.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID if (this.datasource.plus) { config.type = "external" } @@ -572,12 +586,15 @@ class TestConfiguration { async createLinkedTable( relationshipType = RelationshipType.ONE_TO_MANY, links: any = ["link"], - config?: Table + config?: TableToBuild ) { if (!this.table) { throw "Must have created a table first." } const tableConfig = config || basicTable() + if (!tableConfig.sourceId) { + tableConfig.sourceId = INTERNAL_TABLE_SOURCE_ID + } tableConfig.primaryDisplay = "name" for (let link of links) { tableConfig.schema[link] = { @@ -590,7 +607,7 @@ class TestConfiguration { } if (this.datasource && !tableConfig.sourceId) { - tableConfig.sourceId = this.datasource._id + tableConfig.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID if (this.datasource.plus) { tableConfig.type = "external" } diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index d3e92ea34d..52f3450454 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -19,12 +19,14 @@ import { FieldType, SourceName, Table, + INTERNAL_TABLE_SOURCE_ID, } from "@budibase/types" export function basicTable(): Table { return { name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 18d5128986..10274ed4d9 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -1,6 +1,11 @@ import { inputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" -import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types" +import { + FieldType, + FieldTypeSubtypes, + INTERNAL_TABLE_SOURCE_ID, + Table, +} from "@budibase/types" import * as bbReferenceProcessor from "../bbReferenceProcessor" jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ @@ -20,6 +25,7 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, @@ -70,6 +76,7 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, @@ -110,6 +117,7 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, @@ -150,6 +158,7 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index ecb8856c88..56db285f8b 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -2,6 +2,7 @@ import { FieldSubtype, FieldType, FieldTypeSubtypes, + INTERNAL_TABLE_SOURCE_ID, Table, } from "@budibase/types" import { outputProcessing } from ".." @@ -26,6 +27,7 @@ describe("rowProcessor - outputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, @@ -71,6 +73,7 @@ describe("rowProcessor - outputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, @@ -108,6 +111,7 @@ describe("rowProcessor - outputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 2cff24e635..0428561bfe 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -17,7 +17,7 @@ import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" import { BuilderSocketEvent } from "@budibase/shared-core" import { processInternalTable } from "../sdk/app/tables/getters" -import { isExternalTable, isInternalTable } from "../integrations/utils" +import { isInternalTable } from "../integrations/utils" export default class BuilderSocket extends BaseSocket { constructor(app: Koa, server: http.Server) { diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index 5174ec608f..df17351c12 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -3,14 +3,16 @@ import { View, ViewV2 } from "../view" import { RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" +export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" + export interface Table extends Document { type?: string views?: { [key: string]: View | ViewV2 } name: string + sourceId: string primary?: string[] schema: TableSchema primaryDisplay?: string - sourceId?: string relatedFormula?: string[] constrained?: string[] sql?: boolean @@ -19,10 +21,6 @@ export interface Table extends Document { rowHeight?: number } -export interface ExternalTable extends Table { - sourceId: string -} - export interface TableRequest extends Table { _rename?: RenameColumn created?: boolean diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 39a10961de..7a335eb3b9 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,4 +1,4 @@ -import { ExternalTable, Table } from "../documents" +import { Table } from "../documents" export const PASSWORD_REPLACEMENT = "--secret-value--" @@ -176,7 +176,7 @@ export interface IntegrationBase { } export interface Schema { - tables: Record + tables: Record errors: Record } @@ -187,7 +187,7 @@ export interface DatasourcePlus extends IntegrationBase { getStringConcat(parts: string[]): string buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise getTableNames(): Promise } From fd0d8f17f2a3ac4c4413c5b64ed0c730d3f75bcd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 25 Oct 2023 19:07:51 +0100 Subject: [PATCH 37/56] Making sure single table get also includes sourceId. --- packages/server/src/sdk/app/tables/getters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index af779bcc2b..b9c67c573b 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -67,7 +67,7 @@ export async function getTable(tableId: string): Promise
{ const table = await getExternalTable(datasourceId!, tableName!) return { ...table, sql: isSQL(datasource) } } else { - return db.get(tableId) + return processInternalTable(await db.get
(tableId)) } } From ed0670a00863e27e6ef4f1306792ec5a9ea7cb42 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 26 Oct 2023 13:19:09 +0100 Subject: [PATCH 38/56] Major update to make the table.type always 'table' and then adding a new sourceType which states what source the table came from, external or internal. Don't want to keep using a type that should be static as two different things. --- .../src/builderStore/store/frontend.js | 6 +- .../DataTable/RelationshipDataTable.svelte | 3 +- .../backend/DataTable/TableDataTable.svelte | 8 +- .../backend/DataTable/ViewDataTable.svelte | 2 - .../buttons/grid/GridImportButton.svelte | 2 +- .../DataTable/modals/CreateEditColumn.svelte | 14 ++-- .../ExistingTableDataImport.svelte | 4 +- .../modals/CreateTableModal.svelte | 4 +- .../popovers/EditTablePopover.svelte | 9 ++- .../QueryViewerSidePanel/PreviewPanel.svelte | 2 +- .../Tables/CreateExternalTableModal.svelte | 4 +- .../data/datasource/bb_internal/index.svelte | 4 +- .../index.svelte | 5 +- .../server/src/api/controllers/table/index.ts | 7 +- .../src/api/controllers/table/internal.ts | 6 +- .../server/src/api/routes/tests/row.spec.ts | 9 ++- .../server/src/api/routes/tests/table.spec.ts | 23 +++--- .../src/api/routes/tests/viewV2.spec.ts | 6 +- packages/server/src/constants/index.ts | 7 +- .../db/defaultData/datasource_bb_default.ts | 21 ++++-- .../src/integration-test/postgres.spec.ts | 14 ++-- .../server/src/integrations/googlesheets.ts | 5 +- .../src/integrations/microsoftSqlServer.ts | 3 + packages/server/src/integrations/mysql.ts | 3 + packages/server/src/integrations/oracle.ts | 3 + packages/server/src/integrations/postgres.ts | 3 + .../integrations/tests/googlesheets.spec.ts | 4 +- .../middleware/tests/trimViewRowInfo.spec.ts | 2 + .../server/src/middleware/trimViewRowInfo.ts | 1 - .../app/rows/search/tests/external.spec.ts | 4 +- .../app/rows/search/tests/internal.spec.ts | 2 + .../sdk/app/rows/search/tests/utils.spec.ts | 9 ++- .../src/sdk/app/tables/external/utils.ts | 5 +- packages/server/src/sdk/app/tables/getters.ts | 55 +++++++++----- .../src/sdk/app/tables/tests/tables.spec.ts | 2 + .../sdk/app/tables/tests/validation.spec.ts | 6 +- packages/server/src/sdk/app/tables/utils.ts | 4 +- .../src/sdk/app/views/tests/views.spec.ts | 2 + .../src/tests/utilities/TestConfiguration.ts | 73 ++++++++++--------- .../server/src/tests/utilities/structures.ts | 2 + .../tests/inputProcessing.spec.ts | 5 ++ .../tests/outputProcessing.spec.ts | 4 + packages/server/src/websockets/builder.ts | 7 +- .../types/src/documents/app/table/table.ts | 8 +- 44 files changed, 236 insertions(+), 136 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index a567caf87f..a4729b4a8a 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -580,7 +580,7 @@ export const getFrontendStore = () => { let table = validTables.find(table => { return ( table.sourceId !== BUDIBASE_INTERNAL_DB_ID && - table.type === DB_TYPE_INTERNAL + table.sourceType === DB_TYPE_INTERNAL ) }) if (table) { @@ -591,7 +591,7 @@ export const getFrontendStore = () => { table = validTables.find(table => { return ( table.sourceId === BUDIBASE_INTERNAL_DB_ID && - table.type === DB_TYPE_INTERNAL + table.sourceType === DB_TYPE_INTERNAL ) }) if (table) { @@ -599,7 +599,7 @@ export const getFrontendStore = () => { } // Finally try an external table - return validTables.find(table => table.type === DB_TYPE_EXTERNAL) + return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL) }, enrichEmptySettings: (component, opts) => { if (!component?._component) { diff --git a/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte b/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte index 8ef870caca..4e67a92443 100644 --- a/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte @@ -16,7 +16,6 @@ $: linkedTable = $tables.list.find(table => table._id === linkedTableId) $: schema = linkedTable?.schema $: table = $tables.list.find(table => table._id === tableId) - $: type = table?.type $: fetchData(tableId, rowId) $: { let rowLabel = row?.[table?.primaryDisplay] @@ -41,5 +40,5 @@ {#if row && row._id === rowId} -
+
{/if} diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte index 5fee849afb..c2932d3b10 100644 --- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte @@ -16,6 +16,7 @@ import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte" import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte" import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte" + import { DB_TYPE_EXTERNAL } from "constants/backend" const userSchemaOverrides = { firstName: { displayName: "First name", disabled: true }, @@ -27,7 +28,7 @@ $: id = $tables.selected?._id $: isUsersTable = id === TableNames.USERS - $: isInternal = $tables.selected?.type !== "external" + $: isInternal = $tables.selected?.sourceType !== DB_TYPE_EXTERNAL $: gridDatasource = { type: "table", tableId: id, @@ -46,10 +47,7 @@ tables.replaceTable(id, e.detail) // We need to refresh datasources when an external table changes. - // Type "external" may exist - sometimes type is "table" and sometimes it - // is "external" - it has different meanings in different endpoints. - // If we check both these then we hopefully catch all external tables. - if (e.detail?.type === "external" || e.detail?.sql) { + if (e.detail?.sourceType === DB_TYPE_EXTERNAL || e.detail?.sql) { await datasources.fetch() } } diff --git a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte index f6160e3caa..bdf62f2959 100644 --- a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte @@ -17,7 +17,6 @@ let hideAutocolumns = true let data = [] let loading = false - let type = "internal" $: name = view.name $: calculation = view.calculation @@ -65,7 +64,6 @@ tableId={view.tableId} {data} {loading} - {type} rowCount={10} allowEditing={false} bind:hideAutocolumns diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte index 71d971891c..74e255cf7e 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte @@ -10,6 +10,6 @@ diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 7b51e6c839..8c9b425708 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -26,6 +26,7 @@ ALLOWABLE_NUMBER_TYPES, SWITCHABLE_TYPES, PrettyRelationshipDefinitions, + DB_TYPE_EXTERNAL, } from "constants/backend" import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils" import ConfirmDialog from "components/common/ConfirmDialog.svelte" @@ -254,10 +255,11 @@ !uneditable && editableColumn?.type !== AUTO_TYPE && !editableColumn.autocolumn - $: external = table.type === "external" + $: externalTable = table.sourceType === DB_TYPE_EXTERNAL // in the case of internal tables the sourceId will just be undefined $: tableOptions = $tables.list.filter( - opt => opt.type === table.type && table.sourceId === opt.sourceId + opt => + opt.sourceType === table.sourceType && table.sourceId === opt.sourceId ) $: typeEnabled = !originalName || @@ -409,7 +411,7 @@ editableColumn.type === FieldType.BB_REFERENCE && editableColumn.subtype === FieldSubtype.USERS - if (!external) { + if (!externalTable) { return [ FIELDS.STRING, FIELDS.BARCODEQR, @@ -441,7 +443,7 @@ isUsers ? FIELDS.USERS : FIELDS.USER, ] // no-sql or a spreadsheet - if (!external || table.sql) { + if (!externalTable || table.sql) { fields = [...fields, FIELDS.LINK, FIELDS.ARRAY] } return fields @@ -486,7 +488,7 @@ }) } const newError = {} - if (!external && fieldInfo.name?.startsWith("_")) { + if (!externalTable && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) { newError.name = `Illegal character; must be alpha-numeric.` @@ -498,7 +500,7 @@ newError.name = `Column name already in use.` } - if (fieldInfo.type == "auto" && !fieldInfo.subtype) { + if (fieldInfo.type === "auto" && !fieldInfo.subtype) { newError.subtype = `Auto Column requires a type` } diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index 43751ad944..eb1e7bc7ff 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -1,6 +1,6 @@
-
+