Merge pull request #13594 from Budibase/budi-8123/usercolumn-migration

Fix user relationship to user column migration
This commit is contained in:
Adria Navarro 2024-05-09 18:04:37 +02:00 committed by GitHub
commit 7bcdcda10d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 191 additions and 188 deletions

View File

@ -13,7 +13,11 @@
Layout,
AbsTooltip,
} from "@budibase/bbui"
import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core"
import {
SWITCHABLE_TYPES,
ValidColumnNameRegex,
helpers,
} from "@budibase/shared-core"
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
@ -29,7 +33,11 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { FieldType, SourceName } from "@budibase/types"
import {
BBReferenceFieldSubType,
FieldType,
SourceName,
} from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
import { RowUtils } from "@budibase/frontend-core"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
@ -356,9 +364,29 @@
function getAllowedTypes(datasource) {
if (originalName) {
const possibleTypes = SWITCHABLE_TYPES[field.type] || [
editableColumn.type,
]
let possibleTypes = SWITCHABLE_TYPES[field.type] || [editableColumn.type]
if (helpers.schema.isDeprecatedSingleUserColumn(editableColumn)) {
// This will handle old single users columns
return [
{
...FIELDS.USER,
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
},
]
} else if (
editableColumn.type === FieldType.BB_REFERENCE &&
editableColumn.subtype === BBReferenceFieldSubType.USERS
) {
// This will handle old multi users columns
return [
{
...FIELDS.USERS,
subtype: BBReferenceFieldSubType.USERS,
},
]
}
return Object.entries(FIELDS)
.filter(([_, field]) => possibleTypes.includes(field.type))
.map(([_, fieldDefinition]) => fieldDefinition)

View File

@ -59,7 +59,7 @@
value: FieldType.ATTACHMENTS,
},
{
label: "User",
label: "Users",
value: `${FieldType.BB_REFERENCE}${BBReferenceFieldSubType.USER}`,
},
{

View File

@ -165,20 +165,11 @@ export const FIELDS = {
BBReferenceFieldSubType.USER
],
},
// Used for display of editing existing columns
DEPRECATED_USER: {
name: "User",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
icon: TypeIconMap[FieldType.BB_REFERENCE_SINGLE][
BBReferenceFieldSubType.USER
],
},
USERS: {
name: "User List",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
icon: TypeIconMap[FieldType.BB_REFERENCE][BBReferenceFieldSubType.USERS],
subtype: BBReferenceFieldSubType.USER,
icon: TypeIconMap[FieldType.BB_REFERENCE][BBReferenceFieldSubType.USER],
constraints: {
type: "array",
},

View File

@ -60,7 +60,7 @@
// Persist the initial values as options, allowing them to be present in the dropdown,
// even if they are not in the inital fetch results
let valueAsSafeArray = fieldState.value || []
if (!Array.isArray(fieldState.value)) {
if (!Array.isArray(valueAsSafeArray)) {
valueAsSafeArray = [fieldState.value]
}
optionsObj = valueAsSafeArray.reduce((accumulator, value) => {

View File

@ -1,20 +1,27 @@
<script>
import { getContext } from "svelte"
import { helpers } from "@budibase/shared-core"
import RelationshipCell from "./RelationshipCell.svelte"
import { BBReferenceFieldSubType, RelationshipType } from "@budibase/types"
import {
BBReferenceFieldSubType,
FieldType,
RelationshipType,
} from "@budibase/types"
export let api
export let hideCounter = false
export let schema
const { API } = getContext("grid")
const { subtype } = $$props.schema
const { type, subtype } = schema
const schema = {
$: schema = {
...$$props.schema,
// This is not really used, just adding some content to be able to render the relationship cell
tableId: "external",
relationshipType:
subtype === BBReferenceFieldSubType.USER
type === FieldType.BB_REFERENCE_SINGLE ||
helpers.schema.isDeprecatedSingleUserColumn(schema)
? RelationshipType.ONE_TO_MANY
: RelationshipType.MANY_TO_MANY,
}
@ -45,7 +52,7 @@
<RelationshipCell
bind:api
{...$$props}
{...$$restProps}
{schema}
{searchFunction}
primaryDisplay={"email"}

View File

@ -7,11 +7,6 @@
} from "@budibase/bbui"
import { getContext } from "svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import {
BBReferenceFieldSubType,
FieldType,
RelationshipType,
} from "@budibase/types"
const { API, definition, rows } = getContext("grid")
@ -33,20 +28,11 @@
}
const migrateUserColumn = async () => {
let subtype = BBReferenceFieldSubType.USERS
if (column.schema.relationshipType === RelationshipType.ONE_TO_MANY) {
subtype = BBReferenceFieldSubType.USER
}
try {
await API.migrateColumn({
tableId: $definition._id,
oldColumn: column.schema,
newColumn: {
name: newColumnName,
type: FieldType.BB_REFERENCE,
subtype,
},
oldColumn: column.schema.name,
newColumn: newColumnName,
})
notifications.success("Column migrated")
} catch (e) {

View File

@ -1,3 +1,4 @@
import { helpers } from "@budibase/shared-core"
import { TypeIconMap } from "../../../constants"
export const getColor = (idx, opacity = 0.3) => {
@ -11,8 +12,12 @@ export const getColumnIcon = column => {
if (column.schema.autocolumn) {
return "MagicWand"
}
const { type, subtype } = column.schema
if (helpers.schema.isDeprecatedSingleUserColumn(column.schema)) {
return "User"
}
const { type, subtype } = column.schema
const result =
typeof TypeIconMap[type] === "object" && subtype
? TypeIconMap[type][subtype]

View File

@ -132,7 +132,7 @@ export const TypeIconMap = {
[FieldType.BIGINT]: "TagBold",
[FieldType.AUTO]: "MagicWand",
[FieldType.BB_REFERENCE]: {
[BBReferenceFieldSubType.USER]: "User",
[BBReferenceFieldSubType.USER]: "UserGroup",
[BBReferenceFieldSubType.USERS]: "UserGroup",
},
[FieldType.BB_REFERENCE_SINGLE]: {

View File

@ -1,5 +1,6 @@
// need to handle table name + field or just field, depending on if relationships used
import { FieldType, Row, Table } from "@budibase/types"
import { helpers } from "@budibase/shared-core"
import { generateRowIdField } from "../../../../integrations/utils"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
@ -111,8 +112,10 @@ export function fixArrayTypes(row: Row, table: Table) {
try {
row[fieldName] = JSON.parse(row[fieldName])
} catch (err) {
// couldn't convert back to array, ignore
delete row[fieldName]
if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) {
// couldn't convert back to array, ignore
delete row[fieldName]
}
}
}
}

View File

@ -180,5 +180,5 @@ export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
}
ctx.status = 200
ctx.body = { message: `Column ${oldColumn.name} migrated.` }
ctx.body = { message: `Column ${oldColumn} migrated.` }
}

View File

@ -493,16 +493,16 @@ describe.each([
)
await config.api.table.migrate(table._id!, {
oldColumn: table.schema["user relationship"],
newColumn: {
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
},
oldColumn: "user relationship",
newColumn: "user column",
})
const migratedTable = await config.api.table.get(table._id!)
expect(migratedTable.schema["user column"]).toBeDefined()
expect(migratedTable.schema["user column"]).toEqual({
name: "user column",
type: FieldType.BB_REFERENCE_SINGLE,
subtype: BBReferenceFieldSubType.USER,
})
expect(migratedTable.schema["user relationship"]).not.toBeDefined()
const migratedRows = await config.api.row.fetch(table._id!)
@ -515,7 +515,7 @@ describe.each([
expect(migratedRow["user column"]).toBeDefined()
expect(migratedRow["user relationship"]).not.toBeDefined()
expect(row["user relationship"][0]._id).toEqual(
migratedRow["user column"][0]._id
migratedRow["user column"]._id
)
}
})
@ -558,16 +558,19 @@ describe.each([
)
await config.api.table.migrate(table._id!, {
oldColumn: table.schema["user relationship"],
newColumn: {
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "user relationship",
newColumn: "user column",
})
const migratedTable = await config.api.table.get(table._id!)
expect(migratedTable.schema["user column"]).toBeDefined()
expect(migratedTable.schema["user column"]).toEqual({
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
constraints: {
type: "array",
},
})
expect(migratedTable.schema["user relationship"]).not.toBeDefined()
const migratedRow = await config.api.row.get(table._id!, testRow._id!)
@ -610,16 +613,19 @@ describe.each([
})
await config.api.table.migrate(table._id!, {
oldColumn: table.schema["user relationship"],
newColumn: {
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "user relationship",
newColumn: "user column",
})
const migratedTable = await config.api.table.get(table._id!)
expect(migratedTable.schema["user column"]).toBeDefined()
expect(migratedTable.schema["user column"]).toEqual({
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
constraints: {
type: "array",
},
})
expect(migratedTable.schema["user relationship"]).not.toBeDefined()
const row1Migrated = await config.api.row.get(table._id!, row1._id!)
@ -665,16 +671,19 @@ describe.each([
})
await config.api.table.migrate(table._id!, {
oldColumn: table.schema["user relationship"],
newColumn: {
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "user relationship",
newColumn: "user column",
})
const migratedTable = await config.api.table.get(table._id!)
expect(migratedTable.schema["user column"]).toBeDefined()
expect(migratedTable.schema["user column"]).toEqual({
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
constraints: {
type: "array",
},
})
expect(migratedTable.schema["user relationship"]).not.toBeDefined()
const row1Migrated = await config.api.row.get(table._id!, row1._id!)
@ -724,12 +733,8 @@ describe.each([
await config.api.table.migrate(
table._id!,
{
oldColumn: table.schema["user relationship"],
newColumn: {
name: "",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "user relationship",
newColumn: "",
},
{ status: 400 }
)
@ -739,12 +744,8 @@ describe.each([
await config.api.table.migrate(
table._id!,
{
oldColumn: table.schema["user relationship"],
newColumn: {
name: "_id",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "user relationship",
newColumn: "_id",
},
{ status: 400 }
)
@ -754,12 +755,8 @@ describe.each([
await config.api.table.migrate(
table._id!,
{
oldColumn: table.schema["user relationship"],
newColumn: {
name: "num",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "user relationship",
newColumn: "num",
},
{ status: 400 }
)
@ -769,16 +766,8 @@ describe.each([
await config.api.table.migrate(
table._id!,
{
oldColumn: {
name: "not a column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
newColumn: {
name: "new column",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USERS,
},
oldColumn: "not a column",
newColumn: "new column",
},
{ status: 400 }
)

View File

@ -12,7 +12,6 @@ import SqlTableQueryBuilder from "./sqlTable"
import {
BBReferenceFieldMetadata,
FieldSchema,
BBReferenceFieldSubType,
FieldType,
JsonFieldMetadata,
Operation,
@ -767,7 +766,8 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
return (
field.type === FieldType.JSON ||
(field.type === FieldType.BB_REFERENCE &&
field.subtype === BBReferenceFieldSubType.USERS)
// Handling old single user type
field.constraints?.type === "array")
)
}

View File

@ -1,6 +1,5 @@
import { Knex, knex } from "knex"
import {
BBReferenceFieldSubType,
FieldType,
NumberFieldMetadata,
Operation,
@ -63,20 +62,6 @@ function generateSchema(
case FieldType.BB_REFERENCE_SINGLE:
schema.text(key)
break
case FieldType.BB_REFERENCE: {
const subtype = column.subtype
switch (subtype) {
case BBReferenceFieldSubType.USER:
schema.text(key)
break
case BBReferenceFieldSubType.USERS:
schema.json(key)
break
default:
throw utils.unreachable(subtype)
}
break
}
case FieldType.NUMBER:
// if meta is specified then this is a junction table entry
if (column.meta && column.meta.toKey && column.meta.toTable) {
@ -99,6 +84,7 @@ function generateSchema(
})
break
case FieldType.ARRAY:
case FieldType.BB_REFERENCE:
schema.json(key)
break
case FieldType.LINK:

View File

@ -99,15 +99,7 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
break
}
case FieldType.BB_REFERENCE: {
const subtype = column.subtype
switch (subtype) {
case BBReferenceFieldSubType.USER:
case BBReferenceFieldSubType.USERS:
userColumnMapping(key, options)
break
default:
utils.unreachable(subtype)
}
userColumnMapping(key, options)
break
}
}

View File

@ -4,7 +4,6 @@ import {
FieldSchema,
BBReferenceFieldSubType,
InternalTable,
isBBReferenceField,
isRelationshipField,
LinkDocument,
LinkInfo,
@ -12,6 +11,8 @@ import {
RelationshipType,
Row,
Table,
FieldType,
BBReferenceSingleFieldMetadata,
} from "@budibase/types"
import sdk from "../../../sdk"
import { isExternalTableID } from "../../../integrations/utils"
@ -24,25 +25,58 @@ export interface MigrationResult {
export async function migrate(
table: Table,
oldColumn: FieldSchema,
newColumn: FieldSchema
oldColumnName: string,
newColumnName: string
): Promise<MigrationResult> {
if (newColumn.name in table.schema) {
throw new BadRequestError(`Column "${newColumn.name}" already exists`)
if (newColumnName in table.schema) {
throw new BadRequestError(`Column "${newColumnName}" already exists`)
}
if (newColumn.name === "") {
if (newColumnName === "") {
throw new BadRequestError(`Column name cannot be empty`)
}
if (dbCore.isInternalColumnName(newColumn.name)) {
if (dbCore.isInternalColumnName(newColumnName)) {
throw new BadRequestError(`Column name cannot be a reserved column name`)
}
const oldColumn = table.schema[oldColumnName]
if (!oldColumn) {
throw new BadRequestError(
`Column "${oldColumnName}" does not exist on table "${table.name}"`
)
}
if (
oldColumn.type !== FieldType.LINK ||
oldColumn.tableId !== InternalTable.USER_METADATA
) {
throw new BadRequestError(
`Only user relationship migration columns is currently supported`
)
}
const type =
oldColumn.relationshipType === RelationshipType.ONE_TO_MANY
? FieldType.BB_REFERENCE_SINGLE
: FieldType.BB_REFERENCE
const newColumn: FieldSchema = {
name: newColumnName,
type,
subtype: BBReferenceFieldSubType.USER,
}
if (newColumn.type === FieldType.BB_REFERENCE) {
newColumn.constraints = {
type: "array",
}
}
table.schema[newColumn.name] = newColumn
table = await sdk.tables.saveTable(table)
let migrator = getColumnMigrator(table, oldColumn, newColumn)
const migrator = getColumnMigrator(table, oldColumn, newColumn)
try {
return await migrator.doMigration()
} catch (e) {
@ -75,11 +109,14 @@ function getColumnMigrator(
throw new BadRequestError(`Column "${oldColumn.name}" does not exist`)
}
if (!isBBReferenceField(newColumn)) {
if (
newColumn.type !== FieldType.BB_REFERENCE_SINGLE &&
newColumn.type !== FieldType.BB_REFERENCE
) {
throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
}
if (newColumn.subtype !== "user" && newColumn.subtype !== "users") {
if (newColumn.subtype !== BBReferenceFieldSubType.USER) {
throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
}
@ -96,7 +133,7 @@ function getColumnMigrator(
}
if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) {
if (newColumn.subtype !== BBReferenceFieldSubType.USER) {
if (newColumn.type !== FieldType.BB_REFERENCE_SINGLE) {
throw new BadRequestError(
`Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column`
)
@ -107,22 +144,23 @@ function getColumnMigrator(
oldColumn.relationshipType === RelationshipType.MANY_TO_MANY ||
oldColumn.relationshipType === RelationshipType.MANY_TO_ONE
) {
if (newColumn.subtype !== BBReferenceFieldSubType.USERS) {
if (newColumn.type !== FieldType.BB_REFERENCE) {
throw new BadRequestError(
`Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column`
)
}
return new MultiUserColumnMigrator(table, oldColumn, newColumn)
}
throw new BadRequestError(`Unknown migration type`)
}
abstract class UserColumnMigrator implements ColumnMigrator {
abstract class UserColumnMigrator<T> implements ColumnMigrator {
constructor(
protected table: Table,
protected oldColumn: RelationshipFieldMetadata,
protected newColumn: BBReferenceFieldMetadata
protected newColumn: T
) {}
abstract updateRow(row: Row, linkInfo: LinkInfo): void
@ -192,7 +230,7 @@ abstract class UserColumnMigrator implements ColumnMigrator {
}
}
class SingleUserColumnMigrator extends UserColumnMigrator {
class SingleUserColumnMigrator extends UserColumnMigrator<BBReferenceSingleFieldMetadata> {
updateRow(row: Row, linkInfo: LinkInfo): void {
row[this.newColumn.name] = dbCore.getGlobalIDFromUserMetadataID(
linkInfo.rowId
@ -200,7 +238,7 @@ class SingleUserColumnMigrator extends UserColumnMigrator {
}
}
class MultiUserColumnMigrator extends UserColumnMigrator {
class MultiUserColumnMigrator extends UserColumnMigrator<BBReferenceFieldMetadata> {
updateRow(row: Row, linkInfo: LinkInfo): void {
if (!row[this.newColumn.name]) {
row[this.newColumn.name] = []

View File

@ -129,7 +129,7 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
return
}
const { type: columnType, subtype: columnSubtype } = schema[columnName]
const { type: columnType } = schema[columnName]
if (columnType === FieldType.NUMBER) {
// If provided must be a valid number
parsedRow[columnName] = columnData ? Number(columnData) : columnData
@ -140,21 +140,9 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
: columnData
} else if (columnType === FieldType.BB_REFERENCE) {
const parsedValues =
!!columnData && parseCsvExport<{ _id: string }[]>(columnData)
if (!parsedValues) {
parsedRow[columnName] = undefined
} else {
switch (columnSubtype) {
case BBReferenceFieldSubType.USER:
parsedRow[columnName] = parsedValues[0]?._id
break
case BBReferenceFieldSubType.USERS:
parsedRow[columnName] = parsedValues.map(u => u._id)
break
default:
utils.unreachable(columnSubtype)
}
}
(!!columnData && parseCsvExport<{ _id: string }[]>(columnData)) || []
parsedRow[columnName] = parsedValues?.map(u => u._id)
} else if (columnType === FieldType.BB_REFERENCE_SINGLE) {
const parsedValue =
columnData && parseCsvExport<{ _id: string }>(columnData)
@ -200,10 +188,6 @@ function isValidBBReference(
return false
}
if (subtype === BBReferenceFieldSubType.USER && userArray.length > 1) {
return false
}
const constainsWrongId = userArray.find(
user => !db.isGlobalUserID(user._id)
)

View File

@ -51,7 +51,7 @@ export const getValidOperatorsForType = (
value: string
label: string
}[] = []
const { type, subtype, formulaType } = fieldType
const { type, formulaType } = fieldType
if (type === FieldType.STRING) {
ops = stringOps
} else if (type === FieldType.NUMBER || type === FieldType.BIGINT) {
@ -68,16 +68,9 @@ export const getValidOperatorsForType = (
ops = numOps
} else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
ops = stringOps.concat([Op.MoreThan, Op.LessThan])
} else if (
(type === FieldType.BB_REFERENCE_SINGLE ||
type === FieldType.BB_REFERENCE) &&
subtype == BBReferenceFieldSubType.USER
) {
} else if (type === FieldType.BB_REFERENCE_SINGLE) {
ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
} else if (
type === FieldType.BB_REFERENCE &&
subtype == BBReferenceFieldSubType.USERS
) {
} else if (type === FieldType.BB_REFERENCE) {
ops = [Op.Contains, Op.NotContains, Op.ContainsAny, Op.Empty, Op.NotEmpty]
}

View File

@ -1,3 +1,4 @@
export * from "./helpers"
export * from "./integrations"
export * as cron from "./cron"
export * as schema from "./schema"

View File

@ -0,0 +1,13 @@
import {
BBReferenceFieldSubType,
FieldSchema,
FieldType,
} from "@budibase/types"
export function isDeprecatedSingleUserColumn(schema: FieldSchema) {
const result =
schema.type === FieldType.BB_REFERENCE &&
schema.subtype === BBReferenceFieldSubType.USER &&
schema.constraints?.type !== "array"
return result
}

View File

@ -1,5 +1,4 @@
import {
FieldSchema,
Row,
Table,
TableRequest,
@ -31,8 +30,8 @@ export interface BulkImportResponse {
}
export interface MigrateRequest {
oldColumn: FieldSchema
newColumn: FieldSchema
oldColumn: string
newColumn: string
}
export interface MigrateResponse {

View File

@ -214,15 +214,3 @@ export function isManyToOne(
): field is ManyToOneRelationshipFieldMetadata {
return field.relationshipType === RelationshipType.MANY_TO_ONE
}
export function isBBReferenceField(
field: FieldSchema
): field is BBReferenceFieldMetadata {
return field.type === FieldType.BB_REFERENCE
}
export function isAttachmentField(
field: FieldSchema
): field is AttachmentFieldMetadata {
return field.type === FieldType.ATTACHMENTS
}