MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updating types across old table utilities.

This commit is contained in:
mike12345567 2023-10-19 12:26:29 +01:00
parent d8f1da827b
commit 470735cc97
5 changed files with 133 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/server/src/api/controllers/table/internal.ts
View File

@ -61,6 +61,10 @@ export async function bulkImport(
) { ) {
const table = await sdk.tables.getTable(ctx.params.tableId) const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows, identifierFields } = ctx.request.body const { rows, identifierFields } = ctx.request.body
await handleDataImport(ctx.user, table, rows, identifierFields) await handleDataImport(table, {
importRows: rows,
identifierFields,
user: ctx.user,
})
return table return table
} }

131
packages/server/src/api/controllers/table/utils.ts
View File

@ -26,9 +26,16 @@ import {
Row, Row,
SourceName, SourceName,
Table, Table,
Database,
RenameColumn,
NumberFieldMetadata,
FieldSchema,
View,
RelationshipFieldMetadata,
FieldType,
} from "@budibase/types" } from "@budibase/types"
export async function clearColumns(table: any, columnNames: any) { export async function clearColumns(table: Table, columnNames: string[]) {
const db = context.getAppDB() const db = context.getAppDB()
const rows = await db.allDocs( const rows = await db.allDocs(
getRowParams(table._id, null, { getRowParams(table._id, null, {
@ -43,10 +50,13 @@ export async function clearColumns(table: any, columnNames: any) {
)) as { id: string; _rev?: string }[] )) as { id: string; _rev?: string }[]
} }
export async function checkForColumnUpdates(oldTable: any, updatedTable: any) { export async function checkForColumnUpdates(
updatedTable: Table,
oldTable?: Table,
columnRename?: RenameColumn
) {
const db = context.getAppDB() const db = context.getAppDB()
let updatedRows = [] let updatedRows = []
const rename = updatedTable._rename
let deletedColumns: any = [] let deletedColumns: any = []
if (oldTable && oldTable.schema && updatedTable.schema) { if (oldTable && oldTable.schema && updatedTable.schema) {
deletedColumns = Object.keys(oldTable.schema).filter( deletedColumns = Object.keys(oldTable.schema).filter(
@ -54,7 +64,7 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
) )
} }
// check for renaming of columns or deleted columns // check for renaming of columns or deleted columns
if (rename || deletedColumns.length !== 0) { if (columnRename || deletedColumns.length !== 0) {
// Update all rows // Update all rows
const rows = await db.allDocs( const rows = await db.allDocs(
getRowParams(updatedTable._id, null, { getRowParams(updatedTable._id, null, {
@ -64,9 +74,9 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
const rawRows = rows.rows.map(({ doc }: any) => doc) const rawRows = rows.rows.map(({ doc }: any) => doc)
updatedRows = rawRows.map((row: any) => { updatedRows = rawRows.map((row: any) => {
row = cloneDeep(row) row = cloneDeep(row)
if (rename) { if (columnRename) {
row[rename.updated] = row[rename.old] row[columnRename.updated] = row[columnRename.old]
delete row[rename.old] delete row[columnRename.old]
} else if (deletedColumns.length !== 0) { } else if (deletedColumns.length !== 0) {
deletedColumns.forEach((colName: any) => delete row[colName]) deletedColumns.forEach((colName: any) => delete row[colName])
} }
@ -76,14 +86,13 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
// cleanup any attachments from object storage for deleted attachment columns // cleanup any attachments from object storage for deleted attachment columns
await cleanupAttachments(updatedTable, { oldTable, rows: rawRows }) await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
// Update views // Update views
await checkForViewUpdates(updatedTable, rename, deletedColumns) await checkForViewUpdates(updatedTable, deletedColumns, columnRename)
delete updatedTable._rename
} }
return { rows: updatedRows, table: updatedTable } return { rows: updatedRows, table: updatedTable }
} }
// makes sure the passed in table isn't going to reset the auto ID // makes sure the passed in table isn't going to reset the auto ID
export function makeSureTableUpToDate(table: any, tableToSave: any) { export function makeSureTableUpToDate(table: Table, tableToSave: Table) {
if (!table) { if (!table) {
return tableToSave return tableToSave
} }
@ -99,16 +108,17 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) {
column.subtype === AutoFieldSubTypes.AUTO_ID && column.subtype === AutoFieldSubTypes.AUTO_ID &&
tableToSave.schema[field] tableToSave.schema[field]
) { ) {
tableToSave.schema[field].lastID = column.lastID const tableCol = tableToSave.schema[field] as NumberFieldMetadata
tableCol.lastID = column.lastID
} }
} }
return tableToSave return tableToSave
} }
export async function importToRows( export async function importToRows(
data: any[], data: Row[],
table: Table, table: Table,
user: ContextUser | null = null user?: ContextUser
) { ) {
let originalTable = table let originalTable = table
let finalData: any = [] let finalData: any = []
@ -150,19 +160,20 @@ export async function importToRows(
} }
export async function handleDataImport( export async function handleDataImport(
user: ContextUser,
table: Table, table: Table,
rows: Row[], opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] }
identifierFields: Array<string> = []
) { ) {
const schema = table.schema const schema = table.schema
const identifierFields = opts?.identifierFields || []
const user = opts?.user
const importRows = opts?.importRows
if (!rows || !isRows(rows) || !isSchema(schema)) { if (!importRows || !isRows(importRows) || !isSchema(schema)) {
return table return table
} }
const db = context.getAppDB() const db = context.getAppDB()
const data = parse(rows, schema) const data = parse(importRows, schema)
let finalData: any = await importToRows(data, table, user) let finalData: any = await importToRows(data, table, user)
@ -200,7 +211,7 @@ export async function handleDataImport(
return table return table
} }
export async function handleSearchIndexes(table: any) { export async function handleSearchIndexes(table: Table) {
const db = context.getAppDB() const db = context.getAppDB()
// create relevant search indexes // create relevant search indexes
if (table.indexes && table.indexes.length > 0) { if (table.indexes && table.indexes.length > 0) {
@ -244,13 +255,13 @@ export async function handleSearchIndexes(table: any) {
return table return table
} }
export function checkStaticTables(table: any) { export function checkStaticTables(table: Table) {
// check user schema has all required elements // check user schema has all required elements
if (table._id === InternalTables.USER_METADATA) { if (table._id === InternalTables.USER_METADATA) {
for (let [key, schema] of Object.entries(USERS_TABLE_SCHEMA.schema)) { for (let [key, schema] of Object.entries(USERS_TABLE_SCHEMA.schema)) {
// check if the schema exists on the table to be created/updated // check if the schema exists on the table to be created/updated
if (table.schema[key] == null) { if (table.schema[key] == null) {
table.schema[key] = schema table.schema[key] = schema as FieldSchema
} }
} }
} }
@ -258,13 +269,21 @@ export function checkStaticTables(table: any) {
} }
class TableSaveFunctions { class TableSaveFunctions {
db: any db: Database
user: any user?: ContextUser
oldTable: any oldTable?: Table
importRows: any importRows?: Row[]
rows: any rows: Row[]
constructor({ user, oldTable, importRows }: any) { constructor({
user,
oldTable,
importRows,
}: {
user?: ContextUser
oldTable?: Table
importRows?: Row[]
}) {
this.db = context.getAppDB() this.db = context.getAppDB()
this.user = user this.user = user
this.oldTable = oldTable this.oldTable = oldTable
@ -274,7 +293,7 @@ class TableSaveFunctions {
} }
// before anything is done // before anything is done
async before(table: any) { async before(table: Table) {
if (this.oldTable) { if (this.oldTable) {
table = makeSureTableUpToDate(this.oldTable, table) table = makeSureTableUpToDate(this.oldTable, table)
} }
@ -283,16 +302,23 @@ class TableSaveFunctions {
} }
// when confirmed valid // when confirmed valid
async mid(table: any) { async mid(table: Table, columnRename?: RenameColumn) {
let response = await checkForColumnUpdates(this.oldTable, table) let response = await checkForColumnUpdates(
table,
this.oldTable,
columnRename
)
this.rows = this.rows.concat(response.rows) this.rows = this.rows.concat(response.rows)
return table return table
} }
// after saving // after saving
async after(table: any) { async after(table: Table) {
table = await handleSearchIndexes(table) table = await handleSearchIndexes(table)
table = await handleDataImport(this.user, table, this.importRows) table = await handleDataImport(table, {
importRows: this.importRows,
user: this.user,
})
return table return table
} }
@ -302,9 +328,9 @@ class TableSaveFunctions {
} }
export async function checkForViewUpdates( export async function checkForViewUpdates(
table: any, table: Table,
rename: any, deletedColumns: string[],
deletedColumns: any columnRename?: RenameColumn
) { ) {
const views = await getViews() const views = await getViews()
const tableViews = views.filter(view => view.meta.tableId === table._id) const tableViews = views.filter(view => view.meta.tableId === table._id)
@ -314,30 +340,30 @@ export async function checkForViewUpdates(
let needsUpdated = false let needsUpdated = false
// First check for renames, otherwise check for deletions // First check for renames, otherwise check for deletions
if (rename) { if (columnRename) {
// Update calculation field if required // Update calculation field if required
if (view.meta.field === rename.old) { if (view.meta.field === columnRename.old) {
view.meta.field = rename.updated view.meta.field = columnRename.updated
needsUpdated = true needsUpdated = true
} }
// Update group by field if required // Update group by field if required
if (view.meta.groupBy === rename.old) { if (view.meta.groupBy === columnRename.old) {
view.meta.groupBy = rename.updated view.meta.groupBy = columnRename.updated
needsUpdated = true needsUpdated = true
} }
// Update filters if required // Update filters if required
if (view.meta.filters) { if (view.meta.filters) {
view.meta.filters.forEach((filter: any) => { view.meta.filters.forEach((filter: any) => {
if (filter.key === rename.old) { if (filter.key === columnRename.old) {
filter.key = rename.updated filter.key = columnRename.updated
needsUpdated = true needsUpdated = true
} }
}) })
} }
} else if (deletedColumns) { } else if (deletedColumns) {
deletedColumns.forEach((column: any) => { deletedColumns.forEach((column: string) => {
// Remove calculation statement if required // Remove calculation statement if required
if (view.meta.field === column) { if (view.meta.field === column) {
delete view.meta.field delete view.meta.field
@ -378,24 +404,29 @@ export async function checkForViewUpdates(
if (!newViewTemplate.meta.schema) { if (!newViewTemplate.meta.schema) {
newViewTemplate.meta.schema = table.schema newViewTemplate.meta.schema = table.schema
} }
table.views[view.name] = newViewTemplate.meta if (table.views?.[view.name]) {
table.views[view.name] = newViewTemplate.meta as View
}
} }
} }
} }
export function generateForeignKey(column: any, relatedTable: any) { export function generateForeignKey(
column: RelationshipFieldMetadata,
relatedTable: Table
) {
return `fk_${relatedTable.name}_${column.fieldName}` return `fk_${relatedTable.name}_${column.fieldName}`
} }
export function generateJunctionTableName( export function generateJunctionTableName(
column: any, column: RelationshipFieldMetadata,
table: any, table: Table,
relatedTable: any relatedTable: Table
) { ) {
return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}` return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
} }
export function foreignKeyStructure(keyName: any, meta?: any) { export function foreignKeyStructure(keyName: string, meta?: any) {
const structure: any = { const structure: any = {
type: FieldTypes.NUMBER, type: FieldTypes.NUMBER,
constraints: {}, constraints: {},
@ -407,7 +438,7 @@ export function foreignKeyStructure(keyName: any, meta?: any) {
return structure return structure
} }
export function areSwitchableTypes(type1: any, type2: any) { export function areSwitchableTypes(type1: FieldType, type2: FieldType) {
if ( if (
SwitchableTypes.indexOf(type1) === -1 && SwitchableTypes.indexOf(type1) === -1 &&
SwitchableTypes.indexOf(type2) === -1 SwitchableTypes.indexOf(type2) === -1

89
packages/server/src/api/routes/tests/misc.spec.js
View File

@ -12,14 +12,14 @@ describe("run misc tests", () => {
}) })
describe("/bbtel", () => { describe("/bbtel", () => {
it("check if analytics enabled", async () => { it("check if analytics enabled", async () => {
const res = await request const res = await request
.get(`/api/bbtel`) .get(`/api/bbtel`)
.set(config.defaultHeaders()) .set(config.defaultHeaders())
.expect("Content-Type", /json/) .expect("Content-Type", /json/)
.expect(200) .expect(200)
expect(typeof res.body.enabled).toEqual("boolean") expect(typeof res.body.enabled).toEqual("boolean")
}) })
}) })
describe("/health", () => { describe("/health", () => {
@ -37,7 +37,6 @@ describe("run misc tests", () => {
} else { } else {
expect(text.split(".").length).toEqual(3) expect(text.split(".").length).toEqual(3)
} }
}) })
}) })
@ -93,77 +92,79 @@ describe("run misc tests", () => {
constraints: { constraints: {
type: "array", type: "array",
presence: { presence: {
"allowEmpty": true allowEmpty: true,
}, },
inclusion: [ inclusion: ["One", "Two", "Three"],
"One",
"Two",
"Three",
]
}, },
name: "Sample Tags", name: "Sample Tags",
sortable: false sortable: false,
}, },
g: { g: {
type: "options", type: "options",
constraints: { constraints: {
type: "string", type: "string",
presence: false, presence: false,
inclusion: [ inclusion: ["Alpha", "Beta", "Gamma"],
"Alpha",
"Beta",
"Gamma"
]
}, },
name: "Sample Opts" name: "Sample Opts",
} },
}, },
}) })
const importRows = [
{ a: "1", b: "2", c: "3", d: "4", f: "['One']", g: "Alpha" },
{ a: "5", b: "6", c: "7", d: "8", f: "[]", g: undefined },
{ a: "9", b: "10", c: "11", d: "12", f: "['Two','Four']", g: "" },
{ a: "13", b: "14", c: "15", d: "16", g: "Omega" },
]
// Shift specific row tests to the row spec // Shift specific row tests to the row spec
await tableUtils.handleDataImport( await tableUtils.handleDataImport(table, {
{ userId: "test" }, importRows,
table, user: { userId: "test" },
[ })
{ a: '1', b: '2', c: '3', d: '4', f: "['One']", g: "Alpha" },
{ a: '5', b: '6', c: '7', d: '8', f: "[]", g: undefined},
{ a: '9', b: '10', c: '11', d: '12', f: "['Two','Four']", g: ""},
{ a: '13', b: '14', c: '15', d: '16', g: "Omega"}
]
)
// 4 rows imported, the auto ID starts at 1 // 4 rows imported, the auto ID starts at 1
// We expect the handleDataImport function to update the lastID // We expect the handleDataImport function to update the lastID
expect(table.schema.e.lastID).toEqual(4); expect(table.schema.e.lastID).toEqual(4)
// Array/Multi - should have added a new value to the inclusion. // Array/Multi - should have added a new value to the inclusion.
expect(table.schema.f.constraints.inclusion).toEqual(['Four','One','Three','Two']); expect(table.schema.f.constraints.inclusion).toEqual([
"Four",
"One",
"Three",
"Two",
])
// Options - should have a new value in the inclusion // Options - should have a new value in the inclusion
expect(table.schema.g.constraints.inclusion).toEqual(['Alpha','Beta','Gamma','Omega']); expect(table.schema.g.constraints.inclusion).toEqual([
"Alpha",
"Beta",
"Gamma",
"Omega",
])
const rows = await config.getRows() const rows = await config.getRows()
expect(rows.length).toEqual(4); expect(rows.length).toEqual(4)
const rowOne = rows.find(row => row.e === 1) const rowOne = rows.find(row => row.e === 1)
expect(rowOne.a).toEqual("1") expect(rowOne.a).toEqual("1")
expect(rowOne.f).toEqual(['One']) expect(rowOne.f).toEqual(["One"])
expect(rowOne.g).toEqual('Alpha') expect(rowOne.g).toEqual("Alpha")
const rowTwo = rows.find(row => row.e === 2) const rowTwo = rows.find(row => row.e === 2)
expect(rowTwo.a).toEqual("5") expect(rowTwo.a).toEqual("5")
expect(rowTwo.f).toEqual([]) expect(rowTwo.f).toEqual([])
expect(rowTwo.g).toEqual(undefined) expect(rowTwo.g).toEqual(undefined)
const rowThree = rows.find(row => row.e === 3) const rowThree = rows.find(row => row.e === 3)
expect(rowThree.a).toEqual("9") expect(rowThree.a).toEqual("9")
expect(rowThree.f).toEqual(['Two','Four']) expect(rowThree.f).toEqual(["Two", "Four"])
expect(rowThree.g).toEqual(null) expect(rowThree.g).toEqual(null)
const rowFour = rows.find(row => row.e === 4) const rowFour = rows.find(row => row.e === 4)
expect(rowFour.a).toEqual("13") expect(rowFour.a).toEqual("13")
expect(rowFour.f).toEqual(undefined) expect(rowFour.f).toEqual(undefined)
expect(rowFour.g).toEqual('Omega') expect(rowFour.g).toEqual("Omega")
}) })
}) })
}) })

2
packages/server/src/sdk/app/tables/internal/index.ts
View File

@ -68,7 +68,7 @@ export async function save(
throw new Error("Cannot rename a linked column.") throw new Error("Cannot rename a linked column.")
} }
table = await tableSaveFunctions.mid(table) table = await tableSaveFunctions.mid(table, renaming)
// update schema of non-statistics views when new columns are added // update schema of non-statistics views when new columns are added
for (let view in table.views) { for (let view in table.views) {

2
packages/types/src/documents/app/view.ts
View File

@ -2,7 +2,7 @@ import { SearchFilter, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table" import { UIFieldMetadata } from "./table"
export interface View { export interface View {
name: string name?: string
tableId: string tableId: string
field?: string field?: string
filters: ViewFilter[] filters: ViewFilter[]