diff --git a/packages/bbui/src/Form/Field.svelte b/packages/bbui/src/Form/Field.svelte index a2e41072a5..5033c28f7d 100644 --- a/packages/bbui/src/Form/Field.svelte +++ b/packages/bbui/src/Form/Field.svelte @@ -6,11 +6,12 @@ export let label = null export let labelPosition = "above" export let error = null + export let tooltip = ""
{#if label} - + {/if}
diff --git a/packages/bbui/src/Form/FieldLabel.svelte b/packages/bbui/src/Form/FieldLabel.svelte index b070df8cae..3606d77c7b 100644 --- a/packages/bbui/src/Form/FieldLabel.svelte +++ b/packages/bbui/src/Form/FieldLabel.svelte @@ -1,19 +1,24 @@ - + + + diff --git a/packages/bbui/src/Tooltip/TooltipWrapper.svelte b/packages/bbui/src/Tooltip/TooltipWrapper.svelte new file mode 100644 index 0000000000..c587dec1dc --- /dev/null +++ b/packages/bbui/src/Tooltip/TooltipWrapper.svelte @@ -0,0 +1,60 @@ + + +
+ + {#if tooltip} +
+
(showTooltip = true)} + on:mouseleave={() => (showTooltip = false)} + > + +
+ {#if showTooltip} +
+ +
+ {/if} +
+ {/if} +
+ + diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index c6f33b5d31..0d73f3d36d 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -152,6 +152,7 @@ delete field.subtype delete field.tableId delete field.relationshipType + delete field.formulaType // Add in defaults and initial definition const definition = fieldDefinitions[event.detail?.toUpperCase()] @@ -163,6 +164,9 @@ if (field.type === LINK_TYPE) { field.relationshipType = RelationshipTypes.MANY_TO_MANY } + if (field.type === FORMULA_TYPE) { + field.formulaType = "dynamic" + } } function onChangeRequired(e) { @@ -438,8 +442,22 @@ error={errors.relatedName} /> {:else if field.type === FORMULA_TYPE} + {#if !table.sql} + (field.subtype = e.detail)} options={Object.entries(getAutoColumnInformation())} diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index 8d7f50a527..ac97bf6065 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -131,7 +131,7 @@ {bindings} on:change={event => (filter.value = event.detail)} /> - {:else if ["string", "longform", "number"].includes(filter.type)} + {:else if ["string", "longform", "number", "formula"].includes(filter.type)} {:else if ["options", "array"].includes(filter.type)} { const Op = OperatorOptions + const stringOps = [ + Op.Equals, + Op.NotEquals, + Op.StartsWith, + Op.Like, + Op.Empty, + Op.NotEmpty, + ] + const numOps = [ + Op.Equals, + Op.NotEquals, + Op.MoreThan, + Op.LessThan, + Op.Empty, + Op.NotEmpty, + ] if (type === "string") { - return [ - Op.Equals, - Op.NotEquals, - Op.StartsWith, - Op.Like, - Op.Empty, - Op.NotEmpty, - ] + return stringOps } else if (type === "number") { - return [ - Op.Equals, - Op.NotEquals, - Op.MoreThan, - Op.LessThan, - Op.Empty, - Op.NotEmpty, - ] + return numOps } else if (type === "options") { return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] } else if (type === "array") { @@ -84,23 +86,11 @@ export const getValidOperatorsForType = type => { } else if (type === "boolean") { return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] } else if (type === "longform") { - return [ - Op.Equals, - Op.NotEquals, - Op.StartsWith, - Op.Like, - Op.Empty, - Op.NotEmpty, - ] + return stringOps } else if (type === "datetime") { - return [ - Op.Equals, - Op.NotEquals, - Op.MoreThan, - Op.LessThan, - Op.Empty, - Op.NotEmpty, - ] + return numOps + } else if (type === "formula") { + return stringOps.concat([Op.MoreThan, Op.LessThan]) } return [] } diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js index 650e04a680..a9c837d570 100644 --- a/packages/builder/src/helpers/searchFields.js +++ b/packages/builder/src/helpers/searchFields.js @@ -27,5 +27,8 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) { filteredFields = filteredFields.concat(getTableFields(linkField)) } } - return filteredFields + const staticFormulaFields = fields.filter( + field => field.type === "formula" && field.formulaType === "static" + ) + return filteredFields.concat(staticFormulaFields) } diff --git a/packages/client/src/components/app/blocks/CardsBlock.svelte b/packages/client/src/components/app/blocks/CardsBlock.svelte index f0892ca447..fe2af2e12d 100644 --- a/packages/client/src/components/app/blocks/CardsBlock.svelte +++ b/packages/client/src/components/app/blocks/CardsBlock.svelte @@ -39,6 +39,7 @@ number: "numberfield", datetime: "datetimefield", boolean: "booleanfield", + formula: "stringfield", } let formId diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index 3de4497731..cd38908545 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -35,6 +35,7 @@ number: "numberfield", datetime: "datetimefield", boolean: "booleanfield", + formula: "stringfield", } let formId @@ -60,10 +61,11 @@ let enrichedFilter = [...(filter || [])] columns?.forEach(column => { const safePath = column.name.split(".").map(safe).join(".") + const stringType = column.type === "string" || column.type === "formula" enrichedFilter.push({ field: column.name, - operator: column.type === "string" ? "string" : "equal", - type: column.type === "string" ? "string" : "number", + operator: stringType ? "string" : "equal", + type: stringType ? "string" : "number", valueType: "Binding", value: `{{ ${safe(formId)}.${safePath} }}`, }) diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte index f303c69aaf..16d5bb0ee5 100644 --- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte +++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte @@ -19,10 +19,14 @@ export let schemaFields export let filters = [] - const BannedTypes = ["link", "attachment", "formula", "json"] + const BannedTypes = ["link", "attachment", "json"] $: fieldOptions = (schemaFields ?? []) - .filter(field => !BannedTypes.includes(field.type)) + .filter( + field => + !BannedTypes.includes(field.type) || + (field.type === "formula" && field.formulaType === "static") + ) .map(field => field.name) const addFilter = () => { @@ -114,7 +118,7 @@ on:change={e => onOperatorChange(filter, e.detail)} placeholder={null} /> - {#if ["string", "longform", "number"].includes(filter.type)} + {#if ["string", "longform", "number", "formula"].includes(filter.type)} {:else if ["options", "array"].includes(filter.type)} - {:else if fieldSchema?.type && fieldSchema?.type !== type && type !== "options"} + {:else if schemaType && schemaType !== type && type !== "options"} diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 75caaf2fda..0e9c2e651d 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -1,8 +1,8 @@ const CouchDB = require("../../../db") const linkRows = require("../../../db/linkedRows") const { - getRowParams, generateRowID, + getRowParams, DocumentTypes, InternalTables, } = require("../../../db/utils") @@ -10,11 +10,9 @@ const userController = require("../user") const { inputProcessing, outputProcessing, - processAutoColumn, cleanupAttachments, } = require("../../../utilities/rowProcessor") const { FieldTypes } = require("../../../constants") -const { isEqual } = require("lodash") const { validate, findRow } = require("./utils") const { fullSearch, paginatedSearch } = require("./internalSearch") const { getGlobalUsersFromMetadata } = require("../../../utilities/global") @@ -27,6 +25,7 @@ const { getFromMemoryDoc, } = require("../view/utils") const { cloneDeep } = require("lodash/fp") +const { finaliseRow, updateRelatedFormula } = require("./staticFormula") const CALCULATION_TYPES = { SUM: "sum", @@ -34,51 +33,6 @@ const CALCULATION_TYPES = { STATS: "stats", } -async function storeResponse(ctx, db, row, oldTable, table) { - row.type = "row" - // don't worry about rev, tables handle rev/lastID updates - // if another row has been written since processing this will - // handle the auto ID clash - if (!isEqual(oldTable, table)) { - try { - await db.put(table) - } catch (err) { - if (err.status === 409) { - const updatedTable = await db.get(table._id) - let response = processAutoColumn(null, updatedTable, row, { - reprocessing: true, - }) - await db.put(response.table) - row = response.row - } else { - throw err - } - } - } - const response = await db.put(row) - row._rev = response.rev - // process the row before return, to include relationships - row = await outputProcessing(ctx, table, row, { squash: false }) - return { row, table } -} - -// doesn't do the outputProcessing -async function getRawTableData(ctx, db, tableId) { - let rows - if (tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body - } else { - const response = await db.allDocs( - getRowParams(tableId, null, { - include_docs: true, - }) - ) - rows = response.rows.map(row => row.doc) - } - return rows -} - async function getView(db, viewName) { let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc @@ -105,6 +59,22 @@ async function getView(db, viewName) { return viewInfo } +async function getRawTableData(ctx, db, tableId) { + let rows + if (tableId === InternalTables.USER_METADATA) { + await userController.fetchMetadata(ctx) + rows = ctx.body + } else { + const response = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + rows = response.rows.map(row => row.doc) + } + return rows +} + exports.patch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) @@ -162,7 +132,10 @@ exports.patch = async ctx => { return { row: ctx.body, table } } - return storeResponse(ctx, db, row, dbTable, table) + return finaliseRow(ctx.appId, table, row, { + oldTable: dbTable, + updateFormula: true, + }) } exports.save = async function (ctx) { @@ -196,7 +169,10 @@ exports.save = async function (ctx) { table, }) - return storeResponse(ctx, db, row, dbTable, table) + return finaliseRow(ctx.appId, table, row, { + oldTable: dbTable, + updateFormula: true, + }) } exports.fetchView = async ctx => { @@ -302,6 +278,8 @@ exports.destroy = async function (ctx) { }) // remove any attachments that were on the row from object storage await cleanupAttachments(appId, table, { row }) + // remove any static formula + await updateRelatedFormula(appId, table, row) let response if (ctx.params.tableId === InternalTables.USER_METADATA) { @@ -350,6 +328,7 @@ exports.bulkDestroy = async ctx => { } // remove any attachments that were on the rows from object storage await cleanupAttachments(appId, table, { rows }) + await updateRelatedFormula(appId, table, rows) await Promise.all(updates) return { response: { ok: true }, rows } } diff --git a/packages/server/src/api/controllers/row/internalSearch.js b/packages/server/src/api/controllers/row/internalSearch.js index 3a2586331a..a185386b7a 100644 --- a/packages/server/src/api/controllers/row/internalSearch.js +++ b/packages/server/src/api/controllers/row/internalSearch.js @@ -37,22 +37,30 @@ class QueryBuilder { } setLimit(limit) { - this.limit = limit + if (limit != null) { + this.limit = limit + } return this } setSort(sort) { - this.sort = sort + if (sort != null) { + this.sort = sort + } return this } setSortOrder(sortOrder) { - this.sortOrder = sortOrder + if (sortOrder != null) { + this.sortOrder = sortOrder + } return this } setSortType(sortType) { - this.sortType = sortType + if (sortType != null) { + this.sortType = sortType + } return this } diff --git a/packages/server/src/api/controllers/row/staticFormula.js b/packages/server/src/api/controllers/row/staticFormula.js new file mode 100644 index 0000000000..fc0edd1cb4 --- /dev/null +++ b/packages/server/src/api/controllers/row/staticFormula.js @@ -0,0 +1,157 @@ +const CouchDB = require("../../../db") +const { getRowParams } = require("../../../db/utils") +const { + outputProcessing, + processAutoColumn, + processFormulas, +} = require("../../../utilities/rowProcessor") +const { FieldTypes, FormulaTypes } = require("../../../constants") +const { isEqual } = require("lodash") +const { cloneDeep } = require("lodash/fp") + +/** + * This function runs through a list of enriched rows, looks at the rows which + * are related and then checks if they need the state of their formulas + * updated. + * NOTE: this will only for affect static formulas. + */ +exports.updateRelatedFormula = async (appId, table, enrichedRows) => { + const db = new CouchDB(appId) + // no formula to update, we're done + if (!table.relatedFormula) { + return + } + let promises = [] + for (let enrichedRow of Array.isArray(enrichedRows) + ? enrichedRows + : [enrichedRows]) { + // the related rows by tableId + let relatedRows = {} + for (let [key, field] of Object.entries(enrichedRow)) { + const columnDefinition = table.schema[key] + if (columnDefinition && columnDefinition.type === FieldTypes.LINK) { + const relatedTableId = columnDefinition.tableId + if (!relatedRows[relatedTableId]) { + relatedRows[relatedTableId] = [] + } + relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field) + } + } + for (let tableId of table.relatedFormula) { + let relatedTable + try { + // no rows to update, skip + if (!relatedRows[tableId] || relatedRows[tableId].length === 0) { + continue + } + relatedTable = await db.get(tableId) + } catch (err) { + // no error scenario, table doesn't seem to exist anymore, ignore + } + for (let column of Object.values(relatedTable.schema)) { + // needs updated in related rows + if ( + column.type === FieldTypes.FORMULA && + column.formulaType === FormulaTypes.STATIC + ) { + // re-enrich rows for all the related, don't update the related formula for them + promises = promises.concat( + relatedRows[tableId].map(related => + exports.finaliseRow(appId, relatedTable, related, { + updateFormula: false, + }) + ) + ) + } + } + } + } + await Promise.all(promises) +} + +exports.updateAllFormulasInTable = async (appId, table) => { + const db = new CouchDB(appId) + // start by getting the raw rows (which will be written back to DB after update) + let rows = ( + await db.allDocs( + getRowParams(table._id, null, { + include_docs: true, + }) + ) + ).rows.map(row => row.doc) + // now enrich the rows, note the clone so that we have the base state of the + // rows so that we don't write any of the enriched information back + let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), { + squash: false, + }) + const updatedRows = [] + for (let row of rows) { + // find the enriched row, if found process the formulas + const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id) + if (enrichedRow) { + const processed = processFormulas(table, cloneDeep(row), { + dynamic: false, + contextRows: enrichedRow, + }) + // values have changed, need to add to bulk docs to update + if (!isEqual(processed, row)) { + updatedRows.push(processed) + } + } + } + await db.bulkDocs(updatedRows) +} + +/** + * This function runs at the end of the save/patch functions of the row controller, all this + * really does is enrich the row, handle any static formula processing, then return the enriched + * row. The reason we need to return the enriched row is that the automation row created trigger + * expects the row to be totally enriched/contain all relationships. + */ +exports.finaliseRow = async ( + appId, + table, + row, + { oldTable, updateFormula } = { updateFormula: true } +) => { + const db = new CouchDB(appId) + row.type = "row" + // process the row before return, to include relationships + let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), { + squash: false, + }) + // use enriched row to generate formulas for saving, specifically only use as context + row = processFormulas(table, row, { + dynamic: false, + contextRows: enrichedRow, + }) + + // don't worry about rev, tables handle rev/lastID updates + // if another row has been written since processing this will + // handle the auto ID clash + if (oldTable && !isEqual(oldTable, table)) { + try { + await db.put(table) + } catch (err) { + if (err.status === 409) { + const updatedTable = await db.get(table._id) + let response = processAutoColumn(null, updatedTable, row, { + reprocessing: true, + }) + await db.put(response.table) + row = response.row + } else { + throw err + } + } + } + const response = await db.put(row) + // for response, calculate the formulas for the enriched row + enrichedRow._rev = response.rev + enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + // this updates the related formulas in other rows based on the relations to this row + if (updateFormula) { + await exports.updateRelatedFormula(appId, table, enrichedRow) + } + return { row: enrichedRow, table } +} diff --git a/packages/server/src/api/controllers/table/bulkFormula.js b/packages/server/src/api/controllers/table/bulkFormula.js new file mode 100644 index 0000000000..1866d8e650 --- /dev/null +++ b/packages/server/src/api/controllers/table/bulkFormula.js @@ -0,0 +1,183 @@ +const CouchDB = require("../../../db") +const { FieldTypes, FormulaTypes } = require("../../../constants") +const { getAllInternalTables, clearColumns } = require("./utils") +const { doesContainStrings } = require("@budibase/string-templates") +const { cloneDeep } = require("lodash/fp") +const { isEqual, uniq } = require("lodash") +const { updateAllFormulasInTable } = require("../row/staticFormula") + +function isStaticFormula(column) { + return ( + column.type === FieldTypes.FORMULA && + column.formulaType === FormulaTypes.STATIC + ) +} + +/** + * This retrieves the formula columns from a table schema that use a specified column name + * in the formula. + */ +function getFormulaThatUseColumn(table, columnNames) { + let formula = [] + columnNames = Array.isArray(columnNames) ? columnNames : [columnNames] + for (let column of Object.values(table.schema)) { + // not a static formula, or doesn't contain a relationship + if (!isStaticFormula(column)) { + continue + } + if (!doesContainStrings(column.formula, columnNames)) { + continue + } + formula.push(column.name) + } + return formula +} + +/** + * This functions checks for when a related table, column or related column is deleted, if any + * tables need to have the formula column removed. + */ +async function checkIfFormulaNeedsCleared( + appId, + table, + { oldTable, deletion } +) { + const db = new CouchDB(appId) + // start by retrieving all tables, remove the current table from the list + const tables = (await getAllInternalTables(appId)).filter( + tbl => tbl._id !== table._id + ) + const schemaToUse = oldTable ? oldTable.schema : table.schema + let removedColumns = Object.values(schemaToUse).filter( + column => deletion || !table.schema[column.name] + ) + // remove any formula columns that used related columns + for (let removed of removedColumns) { + let tableToUse = table + // if relationship, get the related table + if (removed.type === FieldTypes.LINK) { + tableToUse = tables.find(table => table._id === removed.tableId) + } + const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name) + if (columnsToDelete.length > 0) { + await clearColumns(db, table, columnsToDelete) + } + // need a special case, where a column has been removed from this table, but was used + // in a different, related tables formula + if (!table.relatedFormula) { + continue + } + for (let relatedTableId of table.relatedFormula) { + const relatedColumns = Object.values(table.schema).filter( + column => column.tableId === relatedTableId + ) + const relatedTable = tables.find(table => table._id === relatedTableId) + // look to see if the column was used in a relationship formula, + // relationships won't be used for this + if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) { + let relatedFormulaToRemove = [] + for (let column of relatedColumns) { + relatedFormulaToRemove = relatedFormulaToRemove.concat( + getFormulaThatUseColumn(relatedTable, [ + column.fieldName, + removed.name, + ]) + ) + } + if (relatedFormulaToRemove.length > 0) { + await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove)) + } + } + } + } +} + +/** + * This function adds a note to related tables that they are + * used in a static formula - so that the link controller + * can manage hydrating related rows formula fields. This is + * specifically only for static formula. + */ +async function updateRelatedFormulaLinksOnTables( + appId, + table, + { deletion } = { deletion: false } +) { + const db = new CouchDB(appId) + // start by retrieving all tables, remove the current table from the list + const tables = (await getAllInternalTables(appId)).filter( + tbl => tbl._id !== table._id + ) + // clone the tables, so we can compare at end + const initialTables = cloneDeep(tables) + // first find the related column names + const relatedColumns = Object.values(table.schema).filter( + col => col.type === FieldTypes.LINK + ) + // we start by removing the formula field from all tables + for (let otherTable of tables) { + if (!otherTable.relatedFormula) { + continue + } + const index = otherTable.relatedFormula.indexOf(table._id) + if (index !== -1) { + otherTable.relatedFormula.splice(index, 1) + } + } + // if deleting, just remove the table IDs, don't try add + if (!deletion) { + for (let relatedCol of relatedColumns) { + let columns = getFormulaThatUseColumn(table, relatedCol.name) + if (!columns || columns.length === 0) { + continue + } + const relatedTable = tables.find( + related => related._id === relatedCol.tableId + ) + // check if the table is already in the list of related formula, if it isn't, then add it + if ( + relatedTable && + (!relatedTable.relatedFormula || + !relatedTable.relatedFormula.includes(table._id)) + ) { + relatedTable.relatedFormula = relatedTable.relatedFormula + ? [...relatedTable.relatedFormula, table._id] + : [table._id] + } + } + } + // now we just need to compare all the tables and see if any need saved + for (let initial of initialTables) { + const found = tables.find(tbl => initial._id === tbl._id) + if (found && !isEqual(initial, found)) { + await db.put(found) + } + } +} + +async function checkIfFormulaUpdated(appId, table, { oldTable }) { + // look to see if any formula values have changed + const shouldUpdate = Object.values(table.schema).find( + column => + isStaticFormula(column) && + (!oldTable || + !oldTable.schema[column.name] || + !isEqual(oldTable.schema[column.name], column)) + ) + // if a static formula column has updated, then need to run the update + if (shouldUpdate != null) { + await updateAllFormulasInTable(appId, table) + } +} + +exports.runStaticFormulaChecks = async ( + appId, + table, + { oldTable, deletion } +) => { + await updateRelatedFormulaLinksOnTables(appId, table, { deletion }) + await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion }) + if (!deletion) { + await checkIfFormulaUpdated(appId, table, { oldTable }) + } +} diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index 20dc10017d..2f6bfd0cb3 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -3,12 +3,8 @@ const internal = require("./internal") const external = require("./external") const csvParser = require("../../../utilities/csvParser") const { isExternalTable, isSQL } = require("../../../integrations/utils") -const { - getTableParams, - getDatasourceParams, - BudibaseInternalDB, -} = require("../../../db/utils") -const { getTable } = require("./utils") +const { getDatasourceParams } = require("../../../db/utils") +const { getTable, getAllInternalTables } = require("./utils") function pickApi({ tableId, table }) { if (table && !tableId) { @@ -26,17 +22,7 @@ function pickApi({ tableId, table }) { exports.fetch = async function (ctx) { const db = new CouchDB(ctx.appId) - const internalTables = await db.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - - const internal = internalTables.rows.map(tableDoc => ({ - ...tableDoc.doc, - type: "internal", - sourceId: BudibaseInternalDB._id, - })) + const internal = await getAllInternalTables(ctx.appId) const externalTables = await db.allDocs( getDatasourceParams("plus", { diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js index 9f09e78219..f38a114c25 100644 --- a/packages/server/src/api/controllers/table/internal.js +++ b/packages/server/src/api/controllers/table/internal.js @@ -9,6 +9,8 @@ const { handleDataImport, } = require("./utils") const usageQuota = require("../../../utilities/usageQuota") +const { cleanupAttachments } = require("../../../utilities/rowProcessor") +const { runStaticFormulaChecks } = require("./bulkFormula") exports.save = async function (ctx) { const appId = ctx.appId @@ -104,7 +106,8 @@ exports.save = async function (ctx) { tableToSave._rev = result.rev tableToSave = await tableSaveFunctions.after(tableToSave) - + // has to run after, make sure it has _id + await runStaticFormulaChecks(appId, tableToSave, { oldTable }) return tableToSave } @@ -141,6 +144,9 @@ exports.destroy = async function (ctx) { await db.deleteIndex(existingIndex) } + // has to run after, make sure it has _id + await runStaticFormulaChecks(appId, tableToDelete, { deletion: true }) + await cleanupAttachments(appId, tableToDelete, { rows }) return tableToDelete } diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 0f4c75e197..f1907666c9 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -4,10 +4,15 @@ const { getRowParams, generateRowID, InternalTables, + getTableParams, + BudibaseInternalDB, } = require("../../../db/utils") -const { isEqual } = require("lodash/fp") +const { isEqual } = require("lodash") const { AutoFieldSubTypes, FieldTypes } = require("../../../constants") -const { inputProcessing } = require("../../../utilities/rowProcessor") +const { + inputProcessing, + cleanupAttachments, +} = require("../../../utilities/rowProcessor") const { USERS_TABLE_SCHEMA, SwitchableTypes, @@ -21,8 +26,24 @@ const { const { getViews, saveView } = require("../view/utils") const viewTemplate = require("../view/viewBuilder") const usageQuota = require("../../../utilities/usageQuota") +const { cloneDeep } = require("lodash/fp") -exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => { +exports.clearColumns = async (appId, table, columnNames) => { + const db = new CouchDB(appId) + const rows = await db.allDocs( + getRowParams(table._id, null, { + include_docs: true, + }) + ) + return db.bulkDocs( + rows.rows.map(({ doc }) => { + columnNames.forEach(colName => delete doc[colName]) + return doc + }) + ) +} + +exports.checkForColumnUpdates = async (appId, db, oldTable, updatedTable) => { let updatedRows = [] const rename = updatedTable._rename let deletedColumns = [] @@ -39,16 +60,20 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => { include_docs: true, }) ) - updatedRows = rows.rows.map(({ doc }) => { + const rawRows = rows.rows.map(({ doc }) => doc) + updatedRows = rawRows.map(row => { + row = cloneDeep(row) if (rename) { - doc[rename.updated] = doc[rename.old] - delete doc[rename.old] + row[rename.updated] = row[rename.old] + delete row[rename.old] } else if (deletedColumns.length !== 0) { - deletedColumns.forEach(colName => delete doc[colName]) + deletedColumns.forEach(colName => delete row[colName]) } - return doc + return row }) + // cleanup any attachments from object storage for deleted attachment columns + await cleanupAttachments(appId, updatedTable, { oldTable, rows: rawRows }) // Update views await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns) delete updatedTable._rename @@ -209,6 +234,7 @@ class TableSaveFunctions { // when confirmed valid async mid(table) { let response = await exports.checkForColumnUpdates( + this.appId, this.db, this.oldTable, table @@ -234,6 +260,20 @@ class TableSaveFunctions { } } +exports.getAllInternalTables = async appId => { + const db = new CouchDB(appId) + const internalTables = await db.allDocs( + getTableParams(null, { + include_docs: true, + }) + ) + return internalTables.rows.map(tableDoc => ({ + ...tableDoc.doc, + type: "internal", + sourceId: BudibaseInternalDB._id, + })) +} + exports.getAllExternalTables = async (appId, datasourceId) => { const db = new CouchDB(appId) const datasource = await db.get(datasourceId) diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index ac18196edc..16a98e5c58 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -61,6 +61,11 @@ exports.RelationshipTypes = { MANY_TO_MANY: "many-to-many", } +exports.FormulaTypes = { + STATIC: "static", + DYNAMIC: "dynamic", +} + exports.AuthTypes = { APP: "app", BUILDER: "builder", diff --git a/packages/server/src/db/linkedRows/index.js b/packages/server/src/db/linkedRows/index.js index 6835719e5f..eab287aa33 100644 --- a/packages/server/src/db/linkedRows/index.js +++ b/packages/server/src/db/linkedRows/index.js @@ -72,7 +72,7 @@ async function getLinksForRows(appId, rows) { ) } -async function getFullLinkedDocs(ctx, appId, links) { +async function getFullLinkedDocs(appId, links) { // create DBs const db = new CouchDB(appId) const linkedRowIds = links.map(link => link.id) @@ -146,13 +146,12 @@ exports.updateLinks = async function (args) { /** * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row. * This is required for formula fields, this may only be utilised internally (for now). - * @param {object} ctx The request which is looking for rows. + * @param {string} appId The ID of the app which this request is in the context of. * @param {object} table The table from which the rows originated. * @param {array} rows The rows which are to be enriched. * @return {Promise<*>} returns the rows with all of the enriched relationships on it. */ -exports.attachFullLinkedDocs = async (ctx, table, rows) => { - const appId = ctx.appId +exports.attachFullLinkedDocs = async (appId, table, rows) => { const linkedTableIds = getLinkedTableIDs(table) if (linkedTableIds.length === 0) { return rows @@ -166,7 +165,7 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => { // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows - let linked = await getFullLinkedDocs(ctx, appId, links) + let linked = await getFullLinkedDocs(appId, links) const linkedTables = [] for (let row of rows) { for (let link of links.filter(link => link.thisId === row._id)) { diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 472471855c..16885973f5 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -17,6 +17,8 @@ export interface FieldSchema { autocolumn?: boolean throughFrom?: string throughTo?: string + formula?: string + formulaType?: string main?: boolean meta?: { toTable: string @@ -46,6 +48,7 @@ export interface Table extends Base { schema: TableSchema primaryDisplay?: string sourceId?: string + relatedFormula?: string[] constrained?: string[] } diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index 4f5d72c179..dc56312d63 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -180,6 +180,8 @@ function processAutoColumn( } exports.processAutoColumn = processAutoColumn +exports.processFormulas = processFormulas + /** * This will coerce a value to the correct types based on the type transform map * @param {object} row The value to coerce @@ -229,11 +231,12 @@ exports.inputProcessing = ( } continue } - // specific case to delete formula values if they get saved - // type coercion cannot completely remove the field, so have to do it here + // remove any formula values, they are to be generated if (field.type === FieldTypes.FORMULA) { delete clonedRow[key] - } else { + } + // otherwise coerce what is there to correct types + else { clonedRow[key] = exports.coerce(value, field.type) } } @@ -250,7 +253,7 @@ exports.inputProcessing = ( /** * This function enriches the input rows with anything they are supposed to contain, for example * link records or attachment links. - * @param {object} ctx the request which is looking for enriched rows. + * @param {string} appId the app in which the request is looking for enriched rows. * @param {object} table the table from which these rows came from originally, this is used to determine * the schema of the rows and then enrich. * @param {object[]|object} rows the rows which are to be enriched. @@ -258,22 +261,21 @@ exports.inputProcessing = ( * @returns {object[]|object} the enriched rows will be returned. */ exports.outputProcessing = async ( - ctx, + { appId }, table, rows, opts = { squash: true } ) => { - const appId = ctx.appId let wasArray = true if (!(rows instanceof Array)) { rows = [rows] wasArray = false } // attach any linked row information - let enriched = await linkRows.attachFullLinkedDocs(ctx, table, rows) + let enriched = await linkRows.attachFullLinkedDocs(appId, table, rows) // process formulas - enriched = processFormulas(table, enriched) + enriched = processFormulas(table, enriched, { dynamic: true }) // update the attachments URL depending on hosting for (let [property, column] of Object.entries(table.schema)) { @@ -305,9 +307,15 @@ exports.outputProcessing = async ( * @param {any} row optional - the row being removed. * @param {any} rows optional - if multiple rows being deleted can do this in bulk. * @param {any} oldRow optional - if updating a row this will determine the difference. + * @param {any} oldTable optional - if updating a table, can supply the old table to look for + * deleted attachment columns. * @return {Promise} When all attachments have been removed this will return. */ -exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => { +exports.cleanupAttachments = async ( + appId, + table, + { row, rows, oldRow, oldTable } +) => { if (!isProdAppID(appId)) { const prodAppId = getDeployedAppID(appId) // if prod exists, then don't allow deleting @@ -322,12 +330,16 @@ exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => { files = files.concat(row[key].map(attachment => attachment.key)) } } - for (let [key, schema] of Object.entries(table.schema)) { + const schemaToUse = oldTable ? oldTable.schema : table.schema + for (let [key, schema] of Object.entries(schemaToUse)) { if (schema.type !== FieldTypes.ATTACHMENT) { continue } - // if updating, need to manage the differences - if (oldRow && row) { + // old table had this column, new table doesn't - delete it + if (oldTable && !table.schema[key]) { + rows.forEach(row => addFiles(row, key)) + } else if (oldRow && row) { + // if updating, need to manage the differences files = files.concat(getRemovedAttachmentKeys(oldRow, row, key)) } else if (row) { addFiles(row, key) diff --git a/packages/server/src/utilities/rowProcessor/utils.js b/packages/server/src/utilities/rowProcessor/utils.js index 8cbecfe6c5..95b7828084 100644 --- a/packages/server/src/utilities/rowProcessor/utils.js +++ b/packages/server/src/utilities/rowProcessor/utils.js @@ -1,23 +1,39 @@ -const { FieldTypes } = require("../../constants") +const { FieldTypes, FormulaTypes } = require("../../constants") const { processStringSync } = require("@budibase/string-templates") /** * Looks through the rows provided and finds formulas - which it then processes. */ -exports.processFormulas = (table, rows) => { +exports.processFormulas = ( + table, + rows, + { dynamic, contextRows } = { dynamic: true } +) => { const single = !Array.isArray(rows) if (single) { rows = [rows] + contextRows = contextRows ? [contextRows] : contextRows } for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { + const isStatic = schema.formulaType === FormulaTypes.STATIC + if ( + schema.type !== FieldTypes.FORMULA || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { continue } // iterate through rows and process formula - rows = rows.map(row => ({ - ...row, - [column]: processStringSync(schema.formula, row), - })) + for (let i = 0; i < rows.length; i++) { + if (schema.formula) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + rows[i] = { + ...row, + [column]: processStringSync(schema.formula, context), + } + } + } } return single ? rows[0] : rows } diff --git a/packages/string-templates/src/index.cjs b/packages/string-templates/src/index.cjs index 5d05f0f57f..bc9a410813 100644 --- a/packages/string-templates/src/index.cjs +++ b/packages/string-templates/src/index.cjs @@ -15,6 +15,8 @@ module.exports.processStringSync = templates.processStringSync module.exports.processObjectSync = templates.processObjectSync module.exports.processString = templates.processString module.exports.processObject = templates.processObject +module.exports.doesContainStrings = templates.doesContainStrings +module.exports.doesContainString = templates.doesContainString /** * Use vm2 to run JS scripts in a node env diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index d824d5f1db..616981995d 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -3,6 +3,7 @@ const { registerAll } = require("./helpers/index") const processors = require("./processors") const { atob, btoa } = require("./utilities") const manifest = require("../manifest.json") +const { FIND_HBS_REGEX } = require("./utilities") const hbsInstance = handlebars.create() registerAll(hbsInstance) @@ -26,7 +27,7 @@ function testObject(object) { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {Promise} The structure input, as fully updated as possible. */ module.exports.processObject = async (object, context, opts) => { @@ -57,7 +58,7 @@ module.exports.processObject = async (object, context, opts) => { * then nothing will occur. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {Promise} The enriched string, all templates should have been replaced if they can be. */ module.exports.processString = async (string, context, opts) => { @@ -71,7 +72,7 @@ module.exports.processString = async (string, context, opts) => { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {object|array} The structure input, as fully updated as possible. */ module.exports.processObjectSync = (object, context, opts) => { @@ -92,7 +93,7 @@ module.exports.processObjectSync = (object, context, opts) => { * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object} opts optional - specify some options for processing. + * @param {object|null} opts optional - specify some options for processing. * @returns {string} The enriched string, all templates should have been replaced if they can be. */ module.exports.processStringSync = (string, context, opts) => { @@ -222,3 +223,47 @@ module.exports.decodeJSBinding = handlebars => { } return atob(match[1]) } + +/** + * Same as the doesContainString function, but will check for all the strings + * before confirming it contains. + * @param {string} template The template string to search. + * @param {string[]} strings The strings to look for. + * @returns {boolean} Will return true if all strings found in HBS statement. + */ +module.exports.doesContainStrings = (template, strings) => { + let regexp = new RegExp(FIND_HBS_REGEX) + let matches = template.match(regexp) + if (matches == null) { + return false + } + for (let match of matches) { + let hbs = match + if (exports.isJSBinding(match)) { + hbs = exports.decodeJSBinding(match) + } + let allFound = true + for (let string of strings) { + if (!hbs.includes(string)) { + allFound = false + } + } + if (allFound) { + return true + } + } + return false +} + +/** + * This function looks in the supplied template for handlebars instances, if they contain + * JS the JS will be decoded and then the supplied string will be looked for. For example + * if the template "Hello, your name is {{ related }}" this function would return that true + * for the string "related" but not for "name" as it is not within the handlebars statement. + * @param {string} template A template string to search for handlebars instances. + * @param {string} string The word or sentence to search for. + * @returns {boolean} The this return true if the string is found, false if not. + */ +module.exports.doesContainString = (template, string) => { + return exports.doesContainStrings(template, [string]) +} diff --git a/packages/string-templates/src/index.mjs b/packages/string-templates/src/index.mjs index 446e71ef88..a592ae26d5 100644 --- a/packages/string-templates/src/index.mjs +++ b/packages/string-templates/src/index.mjs @@ -15,6 +15,8 @@ export const processStringSync = templates.processStringSync export const processObjectSync = templates.processObjectSync export const processString = templates.processString export const processObject = templates.processObject +export const doesContainStrings = templates.doesContainStrings +export const doesContainString = templates.doesContainString /** * Use polyfilled vm to run JS scripts in a browser Env diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js index 2fd6505410..490c0aa514 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.js @@ -4,6 +4,8 @@ const { isValid, makePropSafe, getManifest, + encodeJSBinding, + doesContainString, } = require("../src/index.cjs") describe("Test that the string processing works correctly", () => { @@ -157,3 +159,20 @@ describe("check full stops that are safe", () => { expect(output).toEqual("1") }) }) + +describe("check does contain string function", () => { + it("should work for a simple case", () => { + const hbs = "hello {{ name }}" + expect(doesContainString(hbs, "name")).toEqual(true) + }) + + it("should reject a case where its in the string, but not the handlebars", () => { + const hbs = "hello {{ name }}" + expect(doesContainString(hbs, "hello")).toEqual(false) + }) + + it("should handle if its in javascript", () => { + const js = encodeJSBinding(`return $("foo")`) + expect(doesContainString(js, "foo")).toEqual(true) + }) +})