MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding the formula bulk recalculation when adding/changing a formula field.

This commit is contained in:
mike12345567 2022-01-25 16:01:04 +00:00
parent da04db39c3
commit b4eef68d71
6 changed files with 257 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
packages/server/src/api/controllers/row/internal.js
View File

@ -1,8 +1,8 @@
const CouchDB = require("../../../db")
const linkRows = require("../../../db/linkedRows")
const {
getRowParams,
generateRowID,
getRowParams,
DocumentTypes,
InternalTables,
} = require("../../../db/utils")
@ -10,12 +10,9 @@ const userController = require("../user")
const {
inputProcessing,
outputProcessing,
processAutoColumn,
cleanupAttachments,
processFormulas,
} = require("../../../utilities/rowProcessor")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { isEqual } = require("lodash")
const { FieldTypes } = require("../../../constants")
const { validate, findRow } = require("./utils")
const { fullSearch, paginatedSearch } = require("./internalSearch")
const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@ -28,6 +25,7 @@ const {
getFromMemoryDoc,
} = require("../view/utils")
const { cloneDeep } = require("lodash/fp")
const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
const CALCULATION_TYPES = {
SUM: "sum",
@ -35,135 +33,6 @@ const CALCULATION_TYPES = {
STATS: "stats",
}
/**
* This function runs through a list of enriched rows, looks at the rows which
* are related and then checks if they need the state of their formulas
* updated.
* NOTE: this will only for affect static formulas.
*/
async function updateRelatedFormula(appId, db, table, enrichedRows) {
// no formula to update, we're done
if (!table.relatedFormula) {
return
}
let promises = []
for (let enrichedRow of Array.isArray(enrichedRows)
? enrichedRows
: [enrichedRows]) {
// the related rows by tableId
let relatedRows = {}
for (let [key, field] of Object.entries(enrichedRow)) {
const columnDefinition = table.schema[key]
if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
const relatedTableId = columnDefinition.tableId
if (!relatedRows[relatedTableId]) {
relatedRows[relatedTableId] = []
}
relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
}
}
for (let tableId of table.relatedFormula) {
try {
// no rows to update, skip
if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
continue
}
const relatedTable = await db.get(tableId)
for (let column of Object.values(relatedTable.schema)) {
// needs updated in related rows
if (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
) {
// re-enrich rows for all the related, don't update the related formula for them
promises = promises.concat(
relatedRows[tableId].map(related =>
storeResponse(appId, db, relatedTable, related, {
updateFormula: false,
})
)
)
}
}
} catch (err) {
// no error scenario, table doesn't seem to exist anymore, ignore
}
}
}
await Promise.all(promises)
}
/**
* This function runs at the end of the save/patch functions of the row controller, all this
* really does is enrich the row, handle any static formula processing, then return the enriched
* row. The reason we need to return the enriched row is that the automation row created trigger
* expects the row to be totally enriched/contain all relationships.
*/
async function storeResponse(
appId,
db,
table,
row,
{ oldTable, updateFormula } = { updateFormula: true }
) {
row.type = "row"
// process the row before return, to include relationships
let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
squash: false,
})
// use enriched row to generate formulas for saving, specifically only use as context
row = processFormulas(table, row, {
dynamic: false,
contextRows: enrichedRow,
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (oldTable && !isEqual(oldTable, table)) {
try {
await db.put(table)
} catch (err) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})
await db.put(response.table)
row = response.row
} else {
throw err
}
}
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await updateRelatedFormula(appId, db, table, enrichedRow)
}
return { row: enrichedRow, table }
}
// doesn't do the outputProcessing
async function getRawTableData(ctx, db, tableId) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows
}
async function getView(db, viewName) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@ -190,6 +59,22 @@ async function getView(db, viewName) {
return viewInfo
}
async function getRawTableData(ctx, db, tableId) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows
}
exports.patch = async ctx => {
const appId = ctx.appId
const db = new CouchDB(appId)
@ -247,7 +132,7 @@ exports.patch = async ctx => {
return { row: ctx.body, table }
}
return storeResponse(ctx.appId, db, table, row, {
return finaliseRow(ctx.appId, table, row, {
oldTable: dbTable,
updateFormula: true,
})
@ -284,7 +169,7 @@ exports.save = async function (ctx) {
table,
})
return storeResponse(ctx.appId, db, table, row, {
return finaliseRow(ctx.appId, table, row, {
oldTable: dbTable,
updateFormula: true,
})
@ -394,7 +279,7 @@ exports.destroy = async function (ctx) {
// remove any attachments that were on the row from object storage
await cleanupAttachments(appId, table, { row })
// remove any static formula
await updateRelatedFormula(appId, db, table, row)
await updateRelatedFormula(appId, table, row)
let response
if (ctx.params.tableId === InternalTables.USER_METADATA) {
@ -443,7 +328,7 @@ exports.bulkDestroy = async ctx => {
}
// remove any attachments that were on the rows from object storage
await cleanupAttachments(appId, table, { rows })
await updateRelatedFormula(appId, db, table, rows)
await updateRelatedFormula(appId, table, rows)
await Promise.all(updates)
return { response: { ok: true }, rows }
}

156
packages/server/src/api/controllers/row/staticFormula.js Normal file
View File

@ -0,0 +1,156 @@
const CouchDB = require("../../../db")
const { getRowParams } = require("../../../db/utils")
const {
outputProcessing,
processAutoColumn,
processFormulas,
} = require("../../../utilities/rowProcessor")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { isEqual } = require("lodash")
const { cloneDeep } = require("lodash/fp")
/**
* This function runs through a list of enriched rows, looks at the rows which
* are related and then checks if they need the state of their formulas
* updated.
* NOTE: this will only for affect static formulas.
*/
exports.updateRelatedFormula = async (appId, table, enrichedRows) => {
const db = new CouchDB(appId)
// no formula to update, we're done
if (!table.relatedFormula) {
return
}
let promises = []
for (let enrichedRow of Array.isArray(enrichedRows)
? enrichedRows
: [enrichedRows]) {
// the related rows by tableId
let relatedRows = {}
for (let [key, field] of Object.entries(enrichedRow)) {
const columnDefinition = table.schema[key]
if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
const relatedTableId = columnDefinition.tableId
if (!relatedRows[relatedTableId]) {
relatedRows[relatedTableId] = []
}
relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
}
}
for (let tableId of table.relatedFormula) {
try {
// no rows to update, skip
if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
continue
}
const relatedTable = await db.get(tableId)
for (let column of Object.values(relatedTable.schema)) {
// needs updated in related rows
if (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
) {
// re-enrich rows for all the related, don't update the related formula for them
promises = promises.concat(
relatedRows[tableId].map(related =>
exports.finaliseRow(appId, relatedTable, related, {
updateFormula: false,
})
)
)
}
}
} catch (err) {
// no error scenario, table doesn't seem to exist anymore, ignore
}
}
}
await Promise.all(promises)
}
exports.updateAllFormulasInTable = async (appId, table) => {
const db = new CouchDB(appId)
// start by getting the raw rows (which will be written back to DB after update)
let rows = (
await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
// now enrich the rows, note the clone so that we have the base state of the
// rows so that we don't write any of the enriched information back
let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), {
squash: false,
})
const updatedRows = []
for (let row of rows) {
// find the enriched row, if found process the formulas
const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
if (enrichedRow) {
const processed = processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: enrichedRow,
})
// values have changed, need to add to bulk docs to update
if (!isEqual(processed, row)) {
updatedRows.push(processed)
}
}
}
await db.bulkDocs(updatedRows)
}
/**
* This function runs at the end of the save/patch functions of the row controller, all this
* really does is enrich the row, handle any static formula processing, then return the enriched
* row. The reason we need to return the enriched row is that the automation row created trigger
* expects the row to be totally enriched/contain all relationships.
*/
exports.finaliseRow = async (
appId,
table,
row,
{ oldTable, updateFormula } = { updateFormula: true }
) => {
const db = new CouchDB(appId)
row.type = "row"
// process the row before return, to include relationships
let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
squash: false,
})
// use enriched row to generate formulas for saving, specifically only use as context
row = processFormulas(table, row, {
dynamic: false,
contextRows: enrichedRow,
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (oldTable && !isEqual(oldTable, table)) {
try {
await db.put(table)
} catch (err) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})
await db.put(response.table)
row = response.row
} else {
throw err
}
}
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await exports.updateRelatedFormula(appId, table, enrichedRow)
}
return { row: enrichedRow, table }
}

108
packages/server/src/api/controllers/table/bulkFormula.js
View File

@ -1,8 +1,17 @@
const CouchDB = require("../../../db")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { getAllInternalTables, clearColumns } = require("./utils")
const { doesContainStrings } = require("@budibase/string-templates")
const { cloneDeep } = require("lodash/fp")
const { isEqual, uniq } = require("lodash")
const { updateAllFormulasInTable } = require("../row/staticFormula")
function isStaticFormula(column) {
return (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
)
}
/**
* This retrieves the formula columns from a table schema that use a specified column name
@ -13,10 +22,7 @@ function getFormulaThatUseColumn(table, columnNames) {
columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
for (let column of Object.values(table.schema)) {
// not a static formula, or doesn't contain a relationship
if (
column.type !== FieldTypes.FORMULA ||
column.formulaType !== FormulaTypes.STATIC
) {
if (!isStaticFormula(column)) {
continue
}
if (!doesContainStrings(column.formula, columnNames)) {
@ -28,16 +34,18 @@ function getFormulaThatUseColumn(table, columnNames) {
}
/**
* This functions checks two things:
* 1. when a related table, column or related column is deleted, if any
* This functions checks for when a related table, column or related column is deleted, if any
* tables need to have the formula column removed.
* 2. If a formula has been added, or updated bulk update all the rows
* in the table as per the new formula.
*/
async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
async function checkIfFormulaNeedsCleared(
appId,
table,
{ oldTable, deletion }
) {
const db = new CouchDB(appId)
// start by retrieving all tables, remove the current table from the list
const tables = (await getAllInternalTables({ db })).filter(
const tables = (await getAllInternalTables(appId)).filter(
tbl => tbl._id !== table._id
)
const schemaToUse = oldTable ? oldTable.schema : table.schema
@ -57,31 +65,28 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
}
// need a special case, where a column has been removed from this table, but was used
// in a different, related tables formula
if (table.relatedFormula) {
for (let relatedTableId of table.relatedFormula) {
const relatedColumns = Object.values(table.schema).filter(
column => column.tableId === relatedTableId
)
const relatedTable = tables.find(table => table._id === relatedTableId)
// look to see if the column was used in a relationship formula,
// relationships won't be used for this
if (
relatedTable &&
relatedColumns &&
removed.type !== FieldTypes.LINK
) {
let relatedFormulaToRemove = []
for (let column of relatedColumns) {
relatedFormulaToRemove = relatedFormulaToRemove.concat(
getFormulaThatUseColumn(relatedTable, [
column.fieldName,
removed.name,
])
)
}
if (relatedFormulaToRemove.length > 0) {
await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove))
}
if (!table.relatedFormula) {
continue
}
for (let relatedTableId of table.relatedFormula) {
const relatedColumns = Object.values(table.schema).filter(
column => column.tableId === relatedTableId
)
const relatedTable = tables.find(table => table._id === relatedTableId)
// look to see if the column was used in a relationship formula,
// relationships won't be used for this
if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
let relatedFormulaToRemove = []
for (let column of relatedColumns) {
relatedFormulaToRemove = relatedFormulaToRemove.concat(
getFormulaThatUseColumn(relatedTable, [
column.fieldName,
removed.name,
])
)
}
if (relatedFormulaToRemove.length > 0) {
await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove))
}
}
}
@ -95,12 +100,13 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
* specifically only for static formula.
*/
async function updateRelatedFormulaLinksOnTables(
db,
appId,
table,
{ deletion } = { deletion: false }
) {
const db = new CouchDB(appId)
// start by retrieving all tables, remove the current table from the list
const tables = (await getAllInternalTables({ db })).filter(
const tables = (await getAllInternalTables(appId)).filter(
tbl => tbl._id !== table._id
)
// clone the tables, so we can compare at end
@ -150,7 +156,29 @@ async function updateRelatedFormulaLinksOnTables(
}
}
exports.runStaticFormulaChecks = async (db, table, { oldTable, deletion }) => {
await updateRelatedFormulaLinksOnTables(db, table, { deletion })
await checkRequiredFormulaUpdates(db, table, { oldTable, deletion })
async function checkIfFormulaUpdated(appId, table, { oldTable }) {
// look to see if any formula values have changed
const shouldUpdate = Object.values(table.schema).find(
column =>
isStaticFormula(column) &&
(!oldTable ||
!oldTable.schema[column.name] ||
!isEqual(oldTable.schema[column.name], column))
)
// if a static formula column has updated, then need to run the update
if (shouldUpdate != null) {
await updateAllFormulasInTable(appId, table)
}
}
exports.runStaticFormulaChecks = async (
appId,
table,
{ oldTable, deletion }
) => {
await updateRelatedFormulaLinksOnTables(appId, table, { deletion })
await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion })
if (!deletion) {
await checkIfFormulaUpdated(appId, table, { oldTable })
}
}

2
packages/server/src/api/controllers/table/index.js
View File

@ -22,7 +22,7 @@ function pickApi({ tableId, table }) {
exports.fetch = async function (ctx) {
const db = new CouchDB(ctx.appId)
const internal = await getAllInternalTables({ db })
const internal = await getAllInternalTables(ctx.appId)
const externalTables = await db.allDocs(
getDatasourceParams("plus", {

4
packages/server/src/api/controllers/table/internal.js
View File

@ -107,7 +107,7 @@ exports.save = async function (ctx) {
tableToSave = await tableSaveFunctions.after(tableToSave)
// has to run after, make sure it has _id
await runStaticFormulaChecks(db, tableToSave, { oldTable })
await runStaticFormulaChecks(appId, tableToSave, { oldTable })
return tableToSave
}
@ -145,7 +145,7 @@ exports.destroy = async function (ctx) {
}
// has to run after, make sure it has _id
await runStaticFormulaChecks(db, tableToDelete, { deletion: true })
await runStaticFormulaChecks(appId, tableToDelete, { deletion: true })
await cleanupAttachments(appId, tableToDelete, { rows })
return tableToDelete
}

17
packages/server/src/api/controllers/table/utils.js
View File

@ -24,7 +24,8 @@ const viewTemplate = require("../view/viewBuilder")
const usageQuota = require("../../../utilities/usageQuota")
const { cloneDeep } = require("lodash/fp")
exports.clearColumns = async (db, table, columnNames) => {
exports.clearColumns = async (appId, table, columnNames) => {
const db = new CouchDB(appId)
const rows = await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
@ -255,10 +256,8 @@ class TableSaveFunctions {
}
}
exports.getAllInternalTables = async ({ db, appId }) => {
if (appId && !db) {
db = new CouchDB(appId)
}
exports.getAllInternalTables = async appId => {
const db = new CouchDB(appId)
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
@ -271,10 +270,8 @@ exports.getAllInternalTables = async ({ db, appId }) => {
}))
}
exports.getAllExternalTables = async ({ db, appId }, datasourceId) => {
if (appId && !db) {
db = new CouchDB(appId)
}
exports.getAllExternalTables = async (appId, datasourceId) => {
const db = new CouchDB(appId)
const datasource = await db.get(datasourceId)
if (!datasource || !datasource.entities) {
throw "Datasource is not configured fully."
@ -283,7 +280,7 @@ exports.getAllExternalTables = async ({ db, appId }, datasourceId) => {
}
exports.getExternalTable = async (appId, datasourceId, tableName) => {
const entities = await exports.getAllExternalTables({ appId }, datasourceId)
const entities = await exports.getAllExternalTables(appId, datasourceId)
return entities[tableName]
}