From 1fade3404f1e899d6d075dc44fdd6009293dff24 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 25 Jan 2022 16:01:04 +0000 Subject: [PATCH] Adding the formula bulk recalculation when adding/changing a formula field. --- .../src/api/controllers/row/internal.js | 161 +++--------------- .../src/api/controllers/row/staticFormula.js | 156 +++++++++++++++++ .../src/api/controllers/table/bulkFormula.js | 108 +++++++----- .../server/src/api/controllers/table/index.js | 2 +- .../src/api/controllers/table/internal.js | 4 +- .../server/src/api/controllers/table/utils.js | 17 +- 6 files changed, 257 insertions(+), 191 deletions(-) create mode 100644 packages/server/src/api/controllers/row/staticFormula.js diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 8ee908c9a9..0e9c2e651d 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -1,8 +1,8 @@ const CouchDB = require("../../../db") const linkRows = require("../../../db/linkedRows") const { - getRowParams, generateRowID, + getRowParams, DocumentTypes, InternalTables, } = require("../../../db/utils") @@ -10,12 +10,9 @@ const userController = require("../user") const { inputProcessing, outputProcessing, - processAutoColumn, cleanupAttachments, - processFormulas, } = require("../../../utilities/rowProcessor") -const { FieldTypes, FormulaTypes } = require("../../../constants") -const { isEqual } = require("lodash") +const { FieldTypes } = require("../../../constants") const { validate, findRow } = require("./utils") const { fullSearch, paginatedSearch } = require("./internalSearch") const { getGlobalUsersFromMetadata } = require("../../../utilities/global") @@ -28,6 +25,7 @@ const { getFromMemoryDoc, } = require("../view/utils") const { cloneDeep } = require("lodash/fp") +const { finaliseRow, updateRelatedFormula } = require("./staticFormula") const CALCULATION_TYPES = { SUM: "sum", @@ -35,135 +33,6 @@ const CALCULATION_TYPES = { STATS: "stats", } -/** - * This function runs through a list of enriched rows, looks at the rows which - * are related and then checks if they need the state of their formulas - * updated. - * NOTE: this will only for affect static formulas. - */ -async function updateRelatedFormula(appId, db, table, enrichedRows) { - // no formula to update, we're done - if (!table.relatedFormula) { - return - } - let promises = [] - for (let enrichedRow of Array.isArray(enrichedRows) - ? enrichedRows - : [enrichedRows]) { - // the related rows by tableId - let relatedRows = {} - for (let [key, field] of Object.entries(enrichedRow)) { - const columnDefinition = table.schema[key] - if (columnDefinition && columnDefinition.type === FieldTypes.LINK) { - const relatedTableId = columnDefinition.tableId - if (!relatedRows[relatedTableId]) { - relatedRows[relatedTableId] = [] - } - relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field) - } - } - for (let tableId of table.relatedFormula) { - try { - // no rows to update, skip - if (!relatedRows[tableId] || relatedRows[tableId].length === 0) { - continue - } - const relatedTable = await db.get(tableId) - for (let column of Object.values(relatedTable.schema)) { - // needs updated in related rows - if ( - column.type === FieldTypes.FORMULA && - column.formulaType === FormulaTypes.STATIC - ) { - // re-enrich rows for all the related, don't update the related formula for them - promises = promises.concat( - relatedRows[tableId].map(related => - storeResponse(appId, db, relatedTable, related, { - updateFormula: false, - }) - ) - ) - } - } - } catch (err) { - // no error scenario, table doesn't seem to exist anymore, ignore - } - } - } - await Promise.all(promises) -} - -/** - * This function runs at the end of the save/patch functions of the row controller, all this - * really does is enrich the row, handle any static formula processing, then return the enriched - * row. The reason we need to return the enriched row is that the automation row created trigger - * expects the row to be totally enriched/contain all relationships. - */ -async function storeResponse( - appId, - db, - table, - row, - { oldTable, updateFormula } = { updateFormula: true } -) { - row.type = "row" - // process the row before return, to include relationships - let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), { - squash: false, - }) - // use enriched row to generate formulas for saving, specifically only use as context - row = processFormulas(table, row, { - dynamic: false, - contextRows: enrichedRow, - }) - - // don't worry about rev, tables handle rev/lastID updates - // if another row has been written since processing this will - // handle the auto ID clash - if (oldTable && !isEqual(oldTable, table)) { - try { - await db.put(table) - } catch (err) { - if (err.status === 409) { - const updatedTable = await db.get(table._id) - let response = processAutoColumn(null, updatedTable, row, { - reprocessing: true, - }) - await db.put(response.table) - row = response.row - } else { - throw err - } - } - } - const response = await db.put(row) - // for response, calculate the formulas for the enriched row - enrichedRow._rev = response.rev - enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) - // this updates the related formulas in other rows based on the relations to this row - if (updateFormula) { - await updateRelatedFormula(appId, db, table, enrichedRow) - } - return { row: enrichedRow, table } -} - -// doesn't do the outputProcessing -async function getRawTableData(ctx, db, tableId) { - let rows - if (tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body - } else { - const response = await db.allDocs( - getRowParams(tableId, null, { - include_docs: true, - }) - ) - rows = response.rows.map(row => row.doc) - } - return rows -} - async function getView(db, viewName) { let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc @@ -190,6 +59,22 @@ async function getView(db, viewName) { return viewInfo } +async function getRawTableData(ctx, db, tableId) { + let rows + if (tableId === InternalTables.USER_METADATA) { + await userController.fetchMetadata(ctx) + rows = ctx.body + } else { + const response = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + rows = response.rows.map(row => row.doc) + } + return rows +} + exports.patch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) @@ -247,7 +132,7 @@ exports.patch = async ctx => { return { row: ctx.body, table } } - return storeResponse(ctx.appId, db, table, row, { + return finaliseRow(ctx.appId, table, row, { oldTable: dbTable, updateFormula: true, }) @@ -284,7 +169,7 @@ exports.save = async function (ctx) { table, }) - return storeResponse(ctx.appId, db, table, row, { + return finaliseRow(ctx.appId, table, row, { oldTable: dbTable, updateFormula: true, }) @@ -394,7 +279,7 @@ exports.destroy = async function (ctx) { // remove any attachments that were on the row from object storage await cleanupAttachments(appId, table, { row }) // remove any static formula - await updateRelatedFormula(appId, db, table, row) + await updateRelatedFormula(appId, table, row) let response if (ctx.params.tableId === InternalTables.USER_METADATA) { @@ -443,7 +328,7 @@ exports.bulkDestroy = async ctx => { } // remove any attachments that were on the rows from object storage await cleanupAttachments(appId, table, { rows }) - await updateRelatedFormula(appId, db, table, rows) + await updateRelatedFormula(appId, table, rows) await Promise.all(updates) return { response: { ok: true }, rows } } diff --git a/packages/server/src/api/controllers/row/staticFormula.js b/packages/server/src/api/controllers/row/staticFormula.js new file mode 100644 index 0000000000..2508af3bdd --- /dev/null +++ b/packages/server/src/api/controllers/row/staticFormula.js @@ -0,0 +1,156 @@ +const CouchDB = require("../../../db") +const { getRowParams } = require("../../../db/utils") +const { + outputProcessing, + processAutoColumn, + processFormulas, +} = require("../../../utilities/rowProcessor") +const { FieldTypes, FormulaTypes } = require("../../../constants") +const { isEqual } = require("lodash") +const { cloneDeep } = require("lodash/fp") + +/** + * This function runs through a list of enriched rows, looks at the rows which + * are related and then checks if they need the state of their formulas + * updated. + * NOTE: this will only for affect static formulas. + */ +exports.updateRelatedFormula = async (appId, table, enrichedRows) => { + const db = new CouchDB(appId) + // no formula to update, we're done + if (!table.relatedFormula) { + return + } + let promises = [] + for (let enrichedRow of Array.isArray(enrichedRows) + ? enrichedRows + : [enrichedRows]) { + // the related rows by tableId + let relatedRows = {} + for (let [key, field] of Object.entries(enrichedRow)) { + const columnDefinition = table.schema[key] + if (columnDefinition && columnDefinition.type === FieldTypes.LINK) { + const relatedTableId = columnDefinition.tableId + if (!relatedRows[relatedTableId]) { + relatedRows[relatedTableId] = [] + } + relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field) + } + } + for (let tableId of table.relatedFormula) { + try { + // no rows to update, skip + if (!relatedRows[tableId] || relatedRows[tableId].length === 0) { + continue + } + const relatedTable = await db.get(tableId) + for (let column of Object.values(relatedTable.schema)) { + // needs updated in related rows + if ( + column.type === FieldTypes.FORMULA && + column.formulaType === FormulaTypes.STATIC + ) { + // re-enrich rows for all the related, don't update the related formula for them + promises = promises.concat( + relatedRows[tableId].map(related => + exports.finaliseRow(appId, relatedTable, related, { + updateFormula: false, + }) + ) + ) + } + } + } catch (err) { + // no error scenario, table doesn't seem to exist anymore, ignore + } + } + } + await Promise.all(promises) +} + +exports.updateAllFormulasInTable = async (appId, table) => { + const db = new CouchDB(appId) + // start by getting the raw rows (which will be written back to DB after update) + let rows = ( + await db.allDocs( + getRowParams(table._id, null, { + include_docs: true, + }) + ) + ).rows.map(row => row.doc) + // now enrich the rows, note the clone so that we have the base state of the + // rows so that we don't write any of the enriched information back + let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), { + squash: false, + }) + const updatedRows = [] + for (let row of rows) { + // find the enriched row, if found process the formulas + const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id) + if (enrichedRow) { + const processed = processFormulas(table, cloneDeep(row), { + dynamic: false, + contextRows: enrichedRow, + }) + // values have changed, need to add to bulk docs to update + if (!isEqual(processed, row)) { + updatedRows.push(processed) + } + } + } + await db.bulkDocs(updatedRows) +} + +/** + * This function runs at the end of the save/patch functions of the row controller, all this + * really does is enrich the row, handle any static formula processing, then return the enriched + * row. The reason we need to return the enriched row is that the automation row created trigger + * expects the row to be totally enriched/contain all relationships. + */ +exports.finaliseRow = async ( + appId, + table, + row, + { oldTable, updateFormula } = { updateFormula: true } +) => { + const db = new CouchDB(appId) + row.type = "row" + // process the row before return, to include relationships + let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), { + squash: false, + }) + // use enriched row to generate formulas for saving, specifically only use as context + row = processFormulas(table, row, { + dynamic: false, + contextRows: enrichedRow, + }) + + // don't worry about rev, tables handle rev/lastID updates + // if another row has been written since processing this will + // handle the auto ID clash + if (oldTable && !isEqual(oldTable, table)) { + try { + await db.put(table) + } catch (err) { + if (err.status === 409) { + const updatedTable = await db.get(table._id) + let response = processAutoColumn(null, updatedTable, row, { + reprocessing: true, + }) + await db.put(response.table) + row = response.row + } else { + throw err + } + } + } + const response = await db.put(row) + // for response, calculate the formulas for the enriched row + enrichedRow._rev = response.rev + enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + // this updates the related formulas in other rows based on the relations to this row + if (updateFormula) { + await exports.updateRelatedFormula(appId, table, enrichedRow) + } + return { row: enrichedRow, table } +} diff --git a/packages/server/src/api/controllers/table/bulkFormula.js b/packages/server/src/api/controllers/table/bulkFormula.js index ea5abce249..cb2446eb73 100644 --- a/packages/server/src/api/controllers/table/bulkFormula.js +++ b/packages/server/src/api/controllers/table/bulkFormula.js @@ -1,8 +1,17 @@ +const CouchDB = require("../../../db") const { FieldTypes, FormulaTypes } = require("../../../constants") const { getAllInternalTables, clearColumns } = require("./utils") const { doesContainStrings } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") const { isEqual, uniq } = require("lodash") +const { updateAllFormulasInTable } = require("../row/staticFormula") + +function isStaticFormula(column) { + return ( + column.type === FieldTypes.FORMULA && + column.formulaType === FormulaTypes.STATIC + ) +} /** * This retrieves the formula columns from a table schema that use a specified column name @@ -13,10 +22,7 @@ function getFormulaThatUseColumn(table, columnNames) { columnNames = Array.isArray(columnNames) ? columnNames : [columnNames] for (let column of Object.values(table.schema)) { // not a static formula, or doesn't contain a relationship - if ( - column.type !== FieldTypes.FORMULA || - column.formulaType !== FormulaTypes.STATIC - ) { + if (!isStaticFormula(column)) { continue } if (!doesContainStrings(column.formula, columnNames)) { @@ -28,16 +34,18 @@ function getFormulaThatUseColumn(table, columnNames) { } /** - * This functions checks two things: - * 1. when a related table, column or related column is deleted, if any + * This functions checks for when a related table, column or related column is deleted, if any * tables need to have the formula column removed. - * 2. If a formula has been added, or updated bulk update all the rows - * in the table as per the new formula. */ -async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) { +async function checkIfFormulaNeedsCleared( + appId, + table, + { oldTable, deletion } +) { + const db = new CouchDB(appId) // start by retrieving all tables, remove the current table from the list - const tables = (await getAllInternalTables({ db })).filter( + const tables = (await getAllInternalTables(appId)).filter( tbl => tbl._id !== table._id ) const schemaToUse = oldTable ? oldTable.schema : table.schema @@ -57,31 +65,28 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) { } // need a special case, where a column has been removed from this table, but was used // in a different, related tables formula - if (table.relatedFormula) { - for (let relatedTableId of table.relatedFormula) { - const relatedColumns = Object.values(table.schema).filter( - column => column.tableId === relatedTableId - ) - const relatedTable = tables.find(table => table._id === relatedTableId) - // look to see if the column was used in a relationship formula, - // relationships won't be used for this - if ( - relatedTable && - relatedColumns && - removed.type !== FieldTypes.LINK - ) { - let relatedFormulaToRemove = [] - for (let column of relatedColumns) { - relatedFormulaToRemove = relatedFormulaToRemove.concat( - getFormulaThatUseColumn(relatedTable, [ - column.fieldName, - removed.name, - ]) - ) - } - if (relatedFormulaToRemove.length > 0) { - await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove)) - } + if (!table.relatedFormula) { + continue + } + for (let relatedTableId of table.relatedFormula) { + const relatedColumns = Object.values(table.schema).filter( + column => column.tableId === relatedTableId + ) + const relatedTable = tables.find(table => table._id === relatedTableId) + // look to see if the column was used in a relationship formula, + // relationships won't be used for this + if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) { + let relatedFormulaToRemove = [] + for (let column of relatedColumns) { + relatedFormulaToRemove = relatedFormulaToRemove.concat( + getFormulaThatUseColumn(relatedTable, [ + column.fieldName, + removed.name, + ]) + ) + } + if (relatedFormulaToRemove.length > 0) { + await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove)) } } } @@ -95,12 +100,13 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) { * specifically only for static formula. */ async function updateRelatedFormulaLinksOnTables( - db, + appId, table, { deletion } = { deletion: false } ) { + const db = new CouchDB(appId) // start by retrieving all tables, remove the current table from the list - const tables = (await getAllInternalTables({ db })).filter( + const tables = (await getAllInternalTables(appId)).filter( tbl => tbl._id !== table._id ) // clone the tables, so we can compare at end @@ -150,7 +156,29 @@ async function updateRelatedFormulaLinksOnTables( } } -exports.runStaticFormulaChecks = async (db, table, { oldTable, deletion }) => { - await updateRelatedFormulaLinksOnTables(db, table, { deletion }) - await checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) +async function checkIfFormulaUpdated(appId, table, { oldTable }) { + // look to see if any formula values have changed + const shouldUpdate = Object.values(table.schema).find( + column => + isStaticFormula(column) && + (!oldTable || + !oldTable.schema[column.name] || + !isEqual(oldTable.schema[column.name], column)) + ) + // if a static formula column has updated, then need to run the update + if (shouldUpdate != null) { + await updateAllFormulasInTable(appId, table) + } +} + +exports.runStaticFormulaChecks = async ( + appId, + table, + { oldTable, deletion } +) => { + await updateRelatedFormulaLinksOnTables(appId, table, { deletion }) + await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion }) + if (!deletion) { + await checkIfFormulaUpdated(appId, table, { oldTable }) + } } diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index 5c36e5ad5e..2f6bfd0cb3 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -22,7 +22,7 @@ function pickApi({ tableId, table }) { exports.fetch = async function (ctx) { const db = new CouchDB(ctx.appId) - const internal = await getAllInternalTables({ db }) + const internal = await getAllInternalTables(ctx.appId) const externalTables = await db.allDocs( getDatasourceParams("plus", { diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js index 9a477d25e7..f38a114c25 100644 --- a/packages/server/src/api/controllers/table/internal.js +++ b/packages/server/src/api/controllers/table/internal.js @@ -107,7 +107,7 @@ exports.save = async function (ctx) { tableToSave = await tableSaveFunctions.after(tableToSave) // has to run after, make sure it has _id - await runStaticFormulaChecks(db, tableToSave, { oldTable }) + await runStaticFormulaChecks(appId, tableToSave, { oldTable }) return tableToSave } @@ -145,7 +145,7 @@ exports.destroy = async function (ctx) { } // has to run after, make sure it has _id - await runStaticFormulaChecks(db, tableToDelete, { deletion: true }) + await runStaticFormulaChecks(appId, tableToDelete, { deletion: true }) await cleanupAttachments(appId, tableToDelete, { rows }) return tableToDelete } diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 5e3377901a..81c22ab9b7 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -24,7 +24,8 @@ const viewTemplate = require("../view/viewBuilder") const usageQuota = require("../../../utilities/usageQuota") const { cloneDeep } = require("lodash/fp") -exports.clearColumns = async (db, table, columnNames) => { +exports.clearColumns = async (appId, table, columnNames) => { + const db = new CouchDB(appId) const rows = await db.allDocs( getRowParams(table._id, null, { include_docs: true, @@ -255,10 +256,8 @@ class TableSaveFunctions { } } -exports.getAllInternalTables = async ({ db, appId }) => { - if (appId && !db) { - db = new CouchDB(appId) - } +exports.getAllInternalTables = async appId => { + const db = new CouchDB(appId) const internalTables = await db.allDocs( getTableParams(null, { include_docs: true, @@ -271,10 +270,8 @@ exports.getAllInternalTables = async ({ db, appId }) => { })) } -exports.getAllExternalTables = async ({ db, appId }, datasourceId) => { - if (appId && !db) { - db = new CouchDB(appId) - } +exports.getAllExternalTables = async (appId, datasourceId) => { + const db = new CouchDB(appId) const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." @@ -283,7 +280,7 @@ exports.getAllExternalTables = async ({ db, appId }, datasourceId) => { } exports.getExternalTable = async (appId, datasourceId, tableName) => { - const entities = await exports.getAllExternalTables({ appId }, datasourceId) + const entities = await exports.getAllExternalTables(appId, datasourceId) return entities[tableName] }