Adding the formula bulk recalculation when adding/changing a formula field.

packages/server/src/api/controllers/row/internal.js

@@ -1,8 +1,8 @@
 const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const {
-  getRowParams,
   generateRowID,
+  getRowParams,
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
@@ -10,12 +10,9 @@ const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
-  processAutoColumn,
   cleanupAttachments,
-  processFormulas,
 } = require("../../../utilities/rowProcessor")
-const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { FieldTypes } = require("../../../constants")
-const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@@ -28,6 +25,7 @@ const {
   getFromMemoryDoc,
 } = require("../view/utils")
 const { cloneDeep } = require("lodash/fp")
+const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -35,135 +33,6 @@ const CALCULATION_TYPES = {
   STATS: "stats",
 }
 
-/**
- * This function runs through a list of enriched rows, looks at the rows which
- * are related and then checks if they need the state of their formulas
- * updated.
- * NOTE: this will only for affect static formulas.
- */
-async function updateRelatedFormula(appId, db, table, enrichedRows) {
-  // no formula to update, we're done
-  if (!table.relatedFormula) {
-    return
-  }
-  let promises = []
-  for (let enrichedRow of Array.isArray(enrichedRows)
-    ? enrichedRows
-    : [enrichedRows]) {
-    // the related rows by tableId
-    let relatedRows = {}
-    for (let [key, field] of Object.entries(enrichedRow)) {
-      const columnDefinition = table.schema[key]
-      if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
-        const relatedTableId = columnDefinition.tableId
-        if (!relatedRows[relatedTableId]) {
-          relatedRows[relatedTableId] = []
-        }
-        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
-      }
-    }
-    for (let tableId of table.relatedFormula) {
-      try {
-        // no rows to update, skip
-        if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
-          continue
-        }
-        const relatedTable = await db.get(tableId)
-        for (let column of Object.values(relatedTable.schema)) {
-          // needs updated in related rows
-          if (
-            column.type === FieldTypes.FORMULA &&
-            column.formulaType === FormulaTypes.STATIC
-          ) {
-            // re-enrich rows for all the related, don't update the related formula for them
-            promises = promises.concat(
-              relatedRows[tableId].map(related =>
-                storeResponse(appId, db, relatedTable, related, {
-                  updateFormula: false,
-                })
-              )
-            )
-          }
-        }
-      } catch (err) {
-        // no error scenario, table doesn't seem to exist anymore, ignore
-      }
-    }
-  }
-  await Promise.all(promises)
-}
-
-/**
- * This function runs at the end of the save/patch functions of the row controller, all this
- * really does is enrich the row, handle any static formula processing, then return the enriched
- * row. The reason we need to return the enriched row is that the automation row created trigger
- * expects the row to be totally enriched/contain all relationships.
- */
-async function storeResponse(
-  appId,
-  db,
-  table,
-  row,
-  { oldTable, updateFormula } = { updateFormula: true }
-) {
-  row.type = "row"
-  // process the row before return, to include relationships
-  let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
-    squash: false,
-  })
-  // use enriched row to generate formulas for saving, specifically only use as context
-  row = processFormulas(table, row, {
-    dynamic: false,
-    contextRows: enrichedRow,
-  })
-
-  // don't worry about rev, tables handle rev/lastID updates
-  // if another row has been written since processing this will
-  // handle the auto ID clash
-  if (oldTable && !isEqual(oldTable, table)) {
-    try {
-      await db.put(table)
-    } catch (err) {
-      if (err.status === 409) {
-        const updatedTable = await db.get(table._id)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
-      } else {
-        throw err
-      }
-    }
-  }
-  const response = await db.put(row)
-  // for response, calculate the formulas for the enriched row
-  enrichedRow._rev = response.rev
-  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
-  // this updates the related formulas in other rows based on the relations to this row
-  if (updateFormula) {
-    await updateRelatedFormula(appId, db, table, enrichedRow)
-  }
-  return { row: enrichedRow, table }
-}
-
-// doesn't do the outputProcessing
-async function getRawTableData(ctx, db, tableId) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows
-}
-
 async function getView(db, viewName) {
   let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
   let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@@ -190,6 +59,22 @@ async function getView(db, viewName) {
   return viewInfo
 }
 
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
@@ -247,7 +132,7 @@ exports.patch = async ctx => {
     return { row: ctx.body, table }
   }
 
-  return storeResponse(ctx.appId, db, table, row, {
+  return finaliseRow(ctx.appId, table, row, {
     oldTable: dbTable,
     updateFormula: true,
   })
@@ -284,7 +169,7 @@ exports.save = async function (ctx) {
     table,
   })
 
-  return storeResponse(ctx.appId, db, table, row, {
+  return finaliseRow(ctx.appId, table, row, {
     oldTable: dbTable,
     updateFormula: true,
   })
@@ -394,7 +279,7 @@ exports.destroy = async function (ctx) {
   // remove any attachments that were on the row from object storage
   await cleanupAttachments(appId, table, { row })
   // remove any static formula
-  await updateRelatedFormula(appId, db, table, row)
+  await updateRelatedFormula(appId, table, row)
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -443,7 +328,7 @@ exports.bulkDestroy = async ctx => {
   }
   // remove any attachments that were on the rows from object storage
   await cleanupAttachments(appId, table, { rows })
-  await updateRelatedFormula(appId, db, table, rows)
+  await updateRelatedFormula(appId, table, rows)
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }

packages/server/src/api/controllers/row/staticFormula.js

@@ -0,0 +1,156 @@
+const CouchDB = require("../../../db")
+const { getRowParams } = require("../../../db/utils")
+const {
+  outputProcessing,
+  processAutoColumn,
+  processFormulas,
+} = require("../../../utilities/rowProcessor")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { isEqual } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+
+/**
+ * This function runs through a list of enriched rows, looks at the rows which
+ * are related and then checks if they need the state of their formulas
+ * updated.
+ * NOTE: this will only for affect static formulas.
+ */
+exports.updateRelatedFormula = async (appId, table, enrichedRows) => {
+  const db = new CouchDB(appId)
+  // no formula to update, we're done
+  if (!table.relatedFormula) {
+    return
+  }
+  let promises = []
+  for (let enrichedRow of Array.isArray(enrichedRows)
+    ? enrichedRows
+    : [enrichedRows]) {
+    // the related rows by tableId
+    let relatedRows = {}
+    for (let [key, field] of Object.entries(enrichedRow)) {
+      const columnDefinition = table.schema[key]
+      if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
+        const relatedTableId = columnDefinition.tableId
+        if (!relatedRows[relatedTableId]) {
+          relatedRows[relatedTableId] = []
+        }
+        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+      }
+    }
+    for (let tableId of table.relatedFormula) {
+      try {
+        // no rows to update, skip
+        if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
+          continue
+        }
+        const relatedTable = await db.get(tableId)
+        for (let column of Object.values(relatedTable.schema)) {
+          // needs updated in related rows
+          if (
+            column.type === FieldTypes.FORMULA &&
+            column.formulaType === FormulaTypes.STATIC
+          ) {
+            // re-enrich rows for all the related, don't update the related formula for them
+            promises = promises.concat(
+              relatedRows[tableId].map(related =>
+                exports.finaliseRow(appId, relatedTable, related, {
+                  updateFormula: false,
+                })
+              )
+            )
+          }
+        }
+      } catch (err) {
+        // no error scenario, table doesn't seem to exist anymore, ignore
+      }
+    }
+  }
+  await Promise.all(promises)
+}
+
+exports.updateAllFormulasInTable = async (appId, table) => {
+  const db = new CouchDB(appId)
+  // start by getting the raw rows (which will be written back to DB after update)
+  let rows = (
+    await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+  // now enrich the rows, note the clone so that we have the base state of the
+  // rows so that we don't write any of the enriched information back
+  let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), {
+    squash: false,
+  })
+  const updatedRows = []
+  for (let row of rows) {
+    // find the enriched row, if found process the formulas
+    const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
+    if (enrichedRow) {
+      const processed = processFormulas(table, cloneDeep(row), {
+        dynamic: false,
+        contextRows: enrichedRow,
+      })
+      // values have changed, need to add to bulk docs to update
+      if (!isEqual(processed, row)) {
+        updatedRows.push(processed)
+      }
+    }
+  }
+  await db.bulkDocs(updatedRows)
+}
+
+/**
+ * This function runs at the end of the save/patch functions of the row controller, all this
+ * really does is enrich the row, handle any static formula processing, then return the enriched
+ * row. The reason we need to return the enriched row is that the automation row created trigger
+ * expects the row to be totally enriched/contain all relationships.
+ */
+exports.finaliseRow = async (
+  appId,
+  table,
+  row,
+  { oldTable, updateFormula } = { updateFormula: true }
+) => {
+  const db = new CouchDB(appId)
+  row.type = "row"
+  // process the row before return, to include relationships
+  let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
+    squash: false,
+  })
+  // use enriched row to generate formulas for saving, specifically only use as context
+  row = processFormulas(table, row, {
+    dynamic: false,
+    contextRows: enrichedRow,
+  })
+
+  // don't worry about rev, tables handle rev/lastID updates
+  // if another row has been written since processing this will
+  // handle the auto ID clash
+  if (oldTable && !isEqual(oldTable, table)) {
+    try {
+      await db.put(table)
+    } catch (err) {
+      if (err.status === 409) {
+        const updatedTable = await db.get(table._id)
+        let response = processAutoColumn(null, updatedTable, row, {
+          reprocessing: true,
+        })
+        await db.put(response.table)
+        row = response.row
+      } else {
+        throw err
+      }
+    }
+  }
+  const response = await db.put(row)
+  // for response, calculate the formulas for the enriched row
+  enrichedRow._rev = response.rev
+  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  // this updates the related formulas in other rows based on the relations to this row
+  if (updateFormula) {
+    await exports.updateRelatedFormula(appId, table, enrichedRow)
+  }
+  return { row: enrichedRow, table }
+}

packages/server/src/api/controllers/table/bulkFormula.js

@@ -1,8 +1,17 @@
+const CouchDB = require("../../../db")
 const { FieldTypes, FormulaTypes } = require("../../../constants")
 const { getAllInternalTables, clearColumns } = require("./utils")
 const { doesContainStrings } = require("@budibase/string-templates")
 const { cloneDeep } = require("lodash/fp")
 const { isEqual, uniq } = require("lodash")
+const { updateAllFormulasInTable } = require("../row/staticFormula")
+
+function isStaticFormula(column) {
+  return (
+    column.type === FieldTypes.FORMULA &&
+    column.formulaType === FormulaTypes.STATIC
+  )
+}
 
 /**
  * This retrieves the formula columns from a table schema that use a specified column name
@@ -13,10 +22,7 @@ function getFormulaThatUseColumn(table, columnNames) {
   columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
   for (let column of Object.values(table.schema)) {
     // not a static formula, or doesn't contain a relationship
-    if (
+    if (!isStaticFormula(column)) {
-      column.type !== FieldTypes.FORMULA ||
-      column.formulaType !== FormulaTypes.STATIC
-    ) {
       continue
     }
     if (!doesContainStrings(column.formula, columnNames)) {
@@ -28,16 +34,18 @@ function getFormulaThatUseColumn(table, columnNames) {
 }
 
 /**
- * This functions checks two things:
+ * This functions checks for when a related table, column or related column is deleted, if any
- * 1. when a related table, column or related column is deleted, if any
  * tables need to have the formula column removed.
- * 2. If a formula has been added, or updated bulk update all the rows
- * in the table as per the new formula.
  */
 
-async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
+async function checkIfFormulaNeedsCleared(
+  appId,
+  table,
+  { oldTable, deletion }
+) {
+  const db = new CouchDB(appId)
   // start by retrieving all tables, remove the current table from the list
-  const tables = (await getAllInternalTables({ db })).filter(
+  const tables = (await getAllInternalTables(appId)).filter(
     tbl => tbl._id !== table._id
   )
   const schemaToUse = oldTable ? oldTable.schema : table.schema
@@ -57,7 +65,9 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
     }
     // need a special case, where a column has been removed from this table, but was used
     // in a different, related tables formula
-    if (table.relatedFormula) {
+    if (!table.relatedFormula) {
+      continue
+    }
     for (let relatedTableId of table.relatedFormula) {
       const relatedColumns = Object.values(table.schema).filter(
         column => column.tableId === relatedTableId
@@ -65,11 +75,7 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
       const relatedTable = tables.find(table => table._id === relatedTableId)
       // look to see if the column was used in a relationship formula,
       // relationships won't be used for this
-        if (
+      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
-          relatedTable &&
-          relatedColumns &&
-          removed.type !== FieldTypes.LINK
-        ) {
         let relatedFormulaToRemove = []
         for (let column of relatedColumns) {
           relatedFormulaToRemove = relatedFormulaToRemove.concat(
@@ -86,7 +92,6 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
     }
   }
 }
-}
 
 /**
  * This function adds a note to related tables that they are
@@ -95,12 +100,13 @@ async function checkRequiredFormulaUpdates(db, table, { oldTable, deletion }) {
  * specifically only for static formula.
  */
 async function updateRelatedFormulaLinksOnTables(
-  db,
+  appId,
   table,
   { deletion } = { deletion: false }
 ) {
+  const db = new CouchDB(appId)
   // start by retrieving all tables, remove the current table from the list
-  const tables = (await getAllInternalTables({ db })).filter(
+  const tables = (await getAllInternalTables(appId)).filter(
     tbl => tbl._id !== table._id
   )
   // clone the tables, so we can compare at end
@@ -150,7 +156,29 @@ async function updateRelatedFormulaLinksOnTables(
   }
 }
 
-exports.runStaticFormulaChecks = async (db, table, { oldTable, deletion }) => {
+async function checkIfFormulaUpdated(appId, table, { oldTable }) {
-  await updateRelatedFormulaLinksOnTables(db, table, { deletion })
+  // look to see if any formula values have changed
-  await checkRequiredFormulaUpdates(db, table, { oldTable, deletion })
+  const shouldUpdate = Object.values(table.schema).find(
+    column =>
+      isStaticFormula(column) &&
+      (!oldTable ||
+        !oldTable.schema[column.name] ||
+        !isEqual(oldTable.schema[column.name], column))
+  )
+  // if a static formula column has updated, then need to run the update
+  if (shouldUpdate != null) {
+    await updateAllFormulasInTable(appId, table)
+  }
+}
+
+exports.runStaticFormulaChecks = async (
+  appId,
+  table,
+  { oldTable, deletion }
+) => {
+  await updateRelatedFormulaLinksOnTables(appId, table, { deletion })
+  await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion })
+  if (!deletion) {
+    await checkIfFormulaUpdated(appId, table, { oldTable })
+  }
 }

packages/server/src/api/controllers/table/index.js

@@ -22,7 +22,7 @@ function pickApi({ tableId, table }) {
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
-  const internal = await getAllInternalTables({ db })
+  const internal = await getAllInternalTables(ctx.appId)
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {

packages/server/src/api/controllers/table/internal.js

@@ -107,7 +107,7 @@ exports.save = async function (ctx) {
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
   // has to run after, make sure it has _id
-  await runStaticFormulaChecks(db, tableToSave, { oldTable })
+  await runStaticFormulaChecks(appId, tableToSave, { oldTable })
   return tableToSave
 }
 
@@ -145,7 +145,7 @@ exports.destroy = async function (ctx) {
   }
 
   // has to run after, make sure it has _id
-  await runStaticFormulaChecks(db, tableToDelete, { deletion: true })
+  await runStaticFormulaChecks(appId, tableToDelete, { deletion: true })
   await cleanupAttachments(appId, tableToDelete, { rows })
   return tableToDelete
 }

packages/server/src/api/controllers/table/utils.js

@@ -24,7 +24,8 @@ const viewTemplate = require("../view/viewBuilder")
 const usageQuota = require("../../../utilities/usageQuota")
 const { cloneDeep } = require("lodash/fp")
 
-exports.clearColumns = async (db, table, columnNames) => {
+exports.clearColumns = async (appId, table, columnNames) => {
+  const db = new CouchDB(appId)
   const rows = await db.allDocs(
     getRowParams(table._id, null, {
       include_docs: true,
@@ -255,10 +256,8 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllInternalTables = async ({ db, appId }) => {
+exports.getAllInternalTables = async appId => {
-  if (appId && !db) {
+  const db = new CouchDB(appId)
-    db = new CouchDB(appId)
-  }
   const internalTables = await db.allDocs(
     getTableParams(null, {
       include_docs: true,
@@ -271,10 +270,8 @@ exports.getAllInternalTables = async ({ db, appId }) => {
   }))
 }
 
-exports.getAllExternalTables = async ({ db, appId }, datasourceId) => {
+exports.getAllExternalTables = async (appId, datasourceId) => {
-  if (appId && !db) {
+  const db = new CouchDB(appId)
-    db = new CouchDB(appId)
-  }
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -283,7 +280,7 @@ exports.getAllExternalTables = async ({ db, appId }, datasourceId) => {
 }
 
 exports.getExternalTable = async (appId, datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables({ appId }, datasourceId)
+  const entities = await exports.getAllExternalTables(appId, datasourceId)
   return entities[tableName]
 }