diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index 8d7f50a527..ac97bf6065 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -131,7 +131,7 @@ {bindings} on:change={event => (filter.value = event.detail)} /> - {:else if ["string", "longform", "number"].includes(filter.type)} + {:else if ["string", "longform", "number", "formula"].includes(filter.type)} {:else if ["options", "array"].includes(filter.type)} { const Op = OperatorOptions + const stringOps = [ + Op.Equals, + Op.NotEquals, + Op.StartsWith, + Op.Like, + Op.Empty, + Op.NotEmpty, + ] + const numOps = [ + Op.Equals, + Op.NotEquals, + Op.MoreThan, + Op.LessThan, + Op.Empty, + Op.NotEmpty, + ] if (type === "string") { - return [ - Op.Equals, - Op.NotEquals, - Op.StartsWith, - Op.Like, - Op.Empty, - Op.NotEmpty, - ] + return stringOps } else if (type === "number") { - return [ - Op.Equals, - Op.NotEquals, - Op.MoreThan, - Op.LessThan, - Op.Empty, - Op.NotEmpty, - ] + return numOps } else if (type === "options") { return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] } else if (type === "array") { @@ -84,23 +86,11 @@ export const getValidOperatorsForType = type => { } else if (type === "boolean") { return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] } else if (type === "longform") { - return [ - Op.Equals, - Op.NotEquals, - Op.StartsWith, - Op.Like, - Op.Empty, - Op.NotEmpty, - ] + return stringOps } else if (type === "datetime") { - return [ - Op.Equals, - Op.NotEquals, - Op.MoreThan, - Op.LessThan, - Op.Empty, - Op.NotEmpty, - ] + return numOps + } else if (type === "formula") { + return stringOps.concat([Op.MoreThan, Op.LessThan]) } return [] } diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js index 650e04a680..a9c837d570 100644 --- a/packages/builder/src/helpers/searchFields.js +++ b/packages/builder/src/helpers/searchFields.js @@ -27,5 +27,8 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) { filteredFields = filteredFields.concat(getTableFields(linkField)) } } - return filteredFields + const staticFormulaFields = fields.filter( + field => field.type === "formula" && field.formulaType === "static" + ) + return filteredFields.concat(staticFormulaFields) } diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 75caaf2fda..db573f45a4 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -12,6 +12,7 @@ const { outputProcessing, processAutoColumn, cleanupAttachments, + processFormulas, } = require("../../../utilities/rowProcessor") const { FieldTypes } = require("../../../constants") const { isEqual } = require("lodash") @@ -36,6 +37,17 @@ const CALCULATION_TYPES = { async function storeResponse(ctx, db, row, oldTable, table) { row.type = "row" + let rowToSave = cloneDeep(row) + // process the row before return, to include relationships + let enrichedRow = await outputProcessing(ctx, table, cloneDeep(row), { + squash: false, + }) + // use enriched row to generate formulas for saving, specifically only use as context + row = processFormulas(table, row, { + dynamic: false, + contextRows: enrichedRow, + }) + // don't worry about rev, tables handle rev/lastID updates // if another row has been written since processing this will // handle the auto ID clash @@ -45,7 +57,7 @@ async function storeResponse(ctx, db, row, oldTable, table) { } catch (err) { if (err.status === 409) { const updatedTable = await db.get(table._id) - let response = processAutoColumn(null, updatedTable, row, { + let response = processAutoColumn(null, updatedTable, rowToSave, { reprocessing: true, }) await db.put(response.table) @@ -56,10 +68,10 @@ async function storeResponse(ctx, db, row, oldTable, table) { } } const response = await db.put(row) - row._rev = response.rev - // process the row before return, to include relationships - row = await outputProcessing(ctx, table, row, { squash: false }) - return { row, table } + // for response, calculate the formulas for the enriched row + enrichedRow._rev = response.rev + enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + return { row: enrichedRow, table } } // doesn't do the outputProcessing diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index 20dc10017d..5c36e5ad5e 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -3,12 +3,8 @@ const internal = require("./internal") const external = require("./external") const csvParser = require("../../../utilities/csvParser") const { isExternalTable, isSQL } = require("../../../integrations/utils") -const { - getTableParams, - getDatasourceParams, - BudibaseInternalDB, -} = require("../../../db/utils") -const { getTable } = require("./utils") +const { getDatasourceParams } = require("../../../db/utils") +const { getTable, getAllInternalTables } = require("./utils") function pickApi({ tableId, table }) { if (table && !tableId) { @@ -26,17 +22,7 @@ function pickApi({ tableId, table }) { exports.fetch = async function (ctx) { const db = new CouchDB(ctx.appId) - const internalTables = await db.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - - const internal = internalTables.rows.map(tableDoc => ({ - ...tableDoc.doc, - type: "internal", - sourceId: BudibaseInternalDB._id, - })) + const internal = await getAllInternalTables({ db }) const externalTables = await db.allDocs( getDatasourceParams("plus", { diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js index 9f09e78219..9e48fd471a 100644 --- a/packages/server/src/api/controllers/table/internal.js +++ b/packages/server/src/api/controllers/table/internal.js @@ -1,14 +1,82 @@ const CouchDB = require("../../../db") const linkRows = require("../../../db/linkedRows") const { getRowParams, generateTableID } = require("../../../db/utils") -const { FieldTypes } = require("../../../constants") +const { FieldTypes, FormulaTypes } = require("../../../constants") const { TableSaveFunctions, hasTypeChanged, getTable, handleDataImport, + getAllInternalTables, } = require("./utils") const usageQuota = require("../../../utilities/usageQuota") +const { doesContainString } = require("@budibase/string-templates") +const { cloneDeep } = require("lodash/fp") +const { isEqual } = require("lodash") + +/** + * This function adds a note to related tables that they are + * used in a static formula - so that the link controller + * can manage hydrating related rows formula fields. This is + * specifically only for static formula. + */ +async function updateRelatedTablesForFormula(db, table) { + // start by retrieving all tables, remove the current table from the list + const tables = (await getAllInternalTables({ db })).filter( + tbl => tbl._id !== table._id + ) + // clone the tables, so we can compare at end + const initialTables = cloneDeep(tables) + // first find the related column names + const relatedColumns = Object.values(table.schema).filter( + col => col.type === FieldTypes.LINK + ) + // we start by removing the formula field from all tables + for (let otherTable of tables) { + if (!otherTable.relatedFormula) { + continue + } + const index = otherTable.relatedFormula.indexOf(table._id) + if (index !== -1) { + otherTable.relatedFormula.splice(index, 1) + } + } + for (let column of Object.values(table.schema)) { + // not a static formula, or doesn't contain a relationship + if ( + column.type !== FieldTypes.FORMULA || + column.formulaType !== FormulaTypes.STATIC + ) { + continue + } + // check to see if any relationship columns are used in formula + for (let relatedCol of relatedColumns) { + if (!doesContainString(column.formula, relatedCol.name)) { + continue + } + const relatedTable = tables.find( + related => related._id === relatedCol.tableId + ) + // check if the table is already in the list of related formula, if it isn't, then add it + if ( + relatedTable && + (!relatedTable.relatedFormula || + relatedTable.relatedFormula.indexOf(table._id) === -1) + ) { + relatedTable.relatedFormula = relatedTable.relatedFormula + ? [...relatedTable.relatedFormula, table._id] + : [table._id] + } + } + } + // now we just need to compare all the tables and see if any need saved + for (let initial of initialTables) { + const found = tables.find(tbl => initial._id === tbl._id) + if (found && !isEqual(initial, found)) { + await db.put(found) + } + } +} exports.save = async function (ctx) { const appId = ctx.appId @@ -104,6 +172,8 @@ exports.save = async function (ctx) { tableToSave._rev = result.rev tableToSave = await tableSaveFunctions.after(tableToSave) + // has to run after, make sure it has _id + await updateRelatedTablesForFormula(db, tableToSave) return tableToSave } diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 86e2837e15..3db7fd9d9f 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -4,6 +4,8 @@ const { getRowParams, generateRowID, InternalTables, + getTableParams, + BudibaseInternalDB, } = require("../../../db/utils") const { isEqual } = require("lodash/fp") const { AutoFieldSubTypes, FieldTypes } = require("../../../constants") @@ -230,8 +232,26 @@ class TableSaveFunctions { } } -exports.getAllExternalTables = async (appId, datasourceId) => { - const db = new CouchDB(appId) +exports.getAllInternalTables = async ({ db, appId }) => { + if (appId && !db) { + db = new CouchDB(appId) + } + const internalTables = await db.allDocs( + getTableParams(null, { + include_docs: true, + }) + ) + return internalTables.rows.map(tableDoc => ({ + ...tableDoc.doc, + type: "internal", + sourceId: BudibaseInternalDB._id, + })) +} + +exports.getAllExternalTables = async ({ db, appId }, datasourceId) => { + if (appId && !db) { + db = new CouchDB(appId) + } const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." @@ -240,7 +260,7 @@ exports.getAllExternalTables = async (appId, datasourceId) => { } exports.getExternalTable = async (appId, datasourceId, tableName) => { - const entities = await exports.getAllExternalTables(appId, datasourceId) + const entities = await exports.getAllExternalTables({ appId }, datasourceId) return entities[tableName] } diff --git a/packages/server/src/db/linkedRows/LinkController.js b/packages/server/src/db/linkedRows/LinkController.js index b66e2debb5..832f6fecdd 100644 --- a/packages/server/src/db/linkedRows/LinkController.js +++ b/packages/server/src/db/linkedRows/LinkController.js @@ -156,6 +156,8 @@ class LinkController { return true } + updateRelatedFormula() {} + /** * Given the link field of this table, and the link field of the linked table, this makes sure * the state of relationship type is accurate on both. diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 472471855c..16885973f5 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -17,6 +17,8 @@ export interface FieldSchema { autocolumn?: boolean throughFrom?: string throughTo?: string + formula?: string + formulaType?: string main?: boolean meta?: { toTable: string @@ -46,6 +48,7 @@ export interface Table extends Base { schema: TableSchema primaryDisplay?: string sourceId?: string + relatedFormula?: string[] constrained?: string[] } diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index 330947090a..f6c3a66edf 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -180,6 +180,8 @@ function processAutoColumn( } exports.processAutoColumn = processAutoColumn +exports.processFormulas = processFormulas + /** * This will coerce a value to the correct types based on the type transform map * @param {object} row The value to coerce @@ -244,9 +246,6 @@ exports.inputProcessing = ( clonedRow._rev = row._rev } - // now process the static formulas - clonedRow = processFormulas(table, clonedRow, { dynamic: false }) - // handle auto columns - this returns an object like {table, row} return processAutoColumn(user, copiedTable, clonedRow, opts) } diff --git a/packages/server/src/utilities/rowProcessor/utils.js b/packages/server/src/utilities/rowProcessor/utils.js index 30e0b75e0f..30185caca6 100644 --- a/packages/server/src/utilities/rowProcessor/utils.js +++ b/packages/server/src/utilities/rowProcessor/utils.js @@ -4,10 +4,15 @@ const { processStringSync } = require("@budibase/string-templates") /** * Looks through the rows provided and finds formulas - which it then processes. */ -exports.processFormulas = (table, rows, { dynamic } = { dynamic: true }) => { +exports.processFormulas = ( + table, + rows, + { dynamic, contextRows } = { dynamic: true } +) => { const single = !Array.isArray(rows) if (single) { rows = [rows] + contextRows = contextRows ? [contextRows] : contextRows } for (let [column, schema] of Object.entries(table.schema)) { const isStatic = schema.formulaType === FormulaTypes.STATIC @@ -19,10 +24,14 @@ exports.processFormulas = (table, rows, { dynamic } = { dynamic: true }) => { continue } // iterate through rows and process formula - rows = rows.map(row => ({ - ...row, - [column]: processStringSync(schema.formula, row), - })) + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + rows[i] = { + ...row, + [column]: processStringSync(schema.formula, context), + } + } } return single ? rows[0] : rows } diff --git a/packages/string-templates/src/index.cjs b/packages/string-templates/src/index.cjs index 5d05f0f57f..e82e8a688d 100644 --- a/packages/string-templates/src/index.cjs +++ b/packages/string-templates/src/index.cjs @@ -15,6 +15,7 @@ module.exports.processStringSync = templates.processStringSync module.exports.processObjectSync = templates.processObjectSync module.exports.processString = templates.processString module.exports.processObject = templates.processObject +module.exports.doesContainString = templates.doesContainString /** * Use vm2 to run JS scripts in a node env diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 820b8da290..514a762e92 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -3,6 +3,7 @@ const { registerAll } = require("./helpers/index") const processors = require("./processors") const { atob, btoa } = require("./utilities") const manifest = require("../manifest.json") +const { FIND_HBS_REGEX } = require("./utilities") const hbsInstance = handlebars.create() registerAll(hbsInstance) @@ -26,7 +27,7 @@ function testObject(object) { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {Promise} The structure input, as fully updated as possible. */ module.exports.processObject = async (object, context, opts) => { @@ -57,7 +58,7 @@ module.exports.processObject = async (object, context, opts) => { * then nothing will occur. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {Promise} The enriched string, all templates should have been replaced if they can be. */ module.exports.processString = async (string, context, opts) => { @@ -71,7 +72,7 @@ module.exports.processString = async (string, context, opts) => { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {object|array} The structure input, as fully updated as possible. */ module.exports.processObjectSync = (object, context, opts) => { @@ -92,7 +93,7 @@ module.exports.processObjectSync = (object, context, opts) => { * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {string} The enriched string, all templates should have been replaced if they can be. */ module.exports.processStringSync = (string, context, opts) => { @@ -221,3 +222,30 @@ module.exports.decodeJSBinding = handlebars => { } return atob(match[1]) } + +/** + * This function looks in the supplied template for handlebars instances, if they contain + * JS the JS will be decoded and then the supplied string will be looked for. For example + * if the template "Hello, your name is {{ related }}" this function would return that true + * for the string "related" but not for "name" as it is not within the handlebars statement. + * @param {string} template A template string to search for handlebars instances. + * @param {string} string The word or sentence to search for. + * @returns {boolean} The this return true if the string is found, false if not. + */ +module.exports.doesContainString = (template, string) => { + let regexp = new RegExp(FIND_HBS_REGEX) + let matches = template.match(regexp) + if (matches == null) { + return false + } + for (let match of matches) { + let hbs = match + if (exports.isJSBinding(match)) { + hbs = exports.decodeJSBinding(match) + } + if (hbs.includes(string)) { + return true + } + } + return false +} diff --git a/packages/string-templates/src/index.mjs b/packages/string-templates/src/index.mjs index 446e71ef88..49eb191ab9 100644 --- a/packages/string-templates/src/index.mjs +++ b/packages/string-templates/src/index.mjs @@ -15,6 +15,7 @@ export const processStringSync = templates.processStringSync export const processObjectSync = templates.processObjectSync export const processString = templates.processString export const processObject = templates.processObject +export const doesContainString = templates.doesContainString /** * Use polyfilled vm to run JS scripts in a browser Env diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js index 2fd6505410..490c0aa514 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.js @@ -4,6 +4,8 @@ const { isValid, makePropSafe, getManifest, + encodeJSBinding, + doesContainString, } = require("../src/index.cjs") describe("Test that the string processing works correctly", () => { @@ -157,3 +159,20 @@ describe("check full stops that are safe", () => { expect(output).toEqual("1") }) }) + +describe("check does contain string function", () => { + it("should work for a simple case", () => { + const hbs = "hello {{ name }}" + expect(doesContainString(hbs, "name")).toEqual(true) + }) + + it("should reject a case where its in the string, but not the handlebars", () => { + const hbs = "hello {{ name }}" + expect(doesContainString(hbs, "hello")).toEqual(false) + }) + + it("should handle if its in javascript", () => { + const js = encodeJSBinding(`return $("foo")`) + expect(doesContainString(js, "foo")).toEqual(true) + }) +})