MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4214 from Budibase/feature/formula-filtering 

Feature/formula filtering
This commit is contained in:
Michael Drury 2022-01-31 15:56:04 +00:00 committed by GitHub
parent 4b9acb21ae f9c7877e02
commit 5ba6afb399
29 changed files with 709 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/bbui/src/Form/Field.svelte
View File

@ -6,11 +6,12 @@
export let label = null
export let labelPosition = "above"
export let error = null
export let tooltip = ""
</script>
<div class="spectrum-Form-item" class:above={labelPosition === "above"}>
{#if label}
<FieldLabel forId={id} {label} position={labelPosition} />
<FieldLabel forId={id} {label} position={labelPosition} {tooltip} />
{/if}
<div class="spectrum-Form-itemField">
<slot />

5
packages/bbui/src/Form/FieldLabel.svelte
View File

@ -1,19 +1,24 @@
<script>
import TooltipWrapper from "../Tooltip/TooltipWrapper.svelte"
import "@spectrum-css/fieldlabel/dist/index-vars.css"
export let forId
export let label
export let position = "above"
export let tooltip = ""
$: className = position === "above" ? "" : `spectrum-FieldLabel--${position}`
</script>
<TooltipWrapper {tooltip} size="S">
<label
for={forId}
class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${className}`}
>
{label || ""}
</label>
</TooltipWrapper>
<style>
label {

3
packages/bbui/src/Form/Select.svelte
View File

@ -17,6 +17,7 @@
export let quiet = false
export let autoWidth = false
export let sort = false
export let tooltip = ""
const dispatch = createEventDispatcher()
const onChange = e => {
@ -32,7 +33,7 @@
}
</script>
<Field {label} {labelPosition} {error}>
<Field {label} {labelPosition} {error} {tooltip}>
<Select
{quiet}
{error}

59
packages/bbui/src/Label/Label.svelte
View File

@ -1,73 +1,20 @@
<script>
import "@spectrum-css/fieldlabel/dist/index-vars.css"
import Tooltip from "../Tooltip/Tooltip.svelte"
import Icon from "../Icon/Icon.svelte"
import TooltipWrapper from "../Tooltip/TooltipWrapper.svelte"
export let size = "M"
export let tooltip = ""
export let showTooltip = false
</script>
{#if tooltip}
<div class="container">
<label
for=""
class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}
>
<slot />
</label>
<div class="icon-container">
<div
class="icon"
class:icon-small={size === "M" || size === "S"}
on:mouseover={() => (showTooltip = true)}
on:mouseleave={() => (showTooltip = false)}
>
<Icon name="InfoOutline" size="S" disabled={true} />
</div>
{#if showTooltip}
<div class="tooltip">
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
</div>
{/if}
</div>
</div>
{:else}
<TooltipWrapper {tooltip} {size}>
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}>
<slot />
</label>
{/if}
</TooltipWrapper>
<style>
label {
padding: 0;
white-space: nowrap;
}
.container {
display: flex;
align-items: center;
}
.icon-container {
position: relative;
display: flex;
justify-content: center;
margin-top: 1px;
margin-left: 5px;
margin-right: 5px;
}
.tooltip {
position: absolute;
display: flex;
justify-content: center;
top: 15px;
z-index: 1;
width: 160px;
}
.icon {
transform: scale(0.75);
}
.icon-small {
margin-top: -2px;
margin-bottom: -5px;
}
</style>

60
packages/bbui/src/Tooltip/TooltipWrapper.svelte Normal file
View File

@ -0,0 +1,60 @@
<script>
import Tooltip from "./Tooltip.svelte"
import Icon from "../Icon/Icon.svelte"
export let tooltip = ""
export let size = "M"
let showTooltip = false
</script>
<div class:container={!!tooltip}>
<slot />
{#if tooltip}
<div class="icon-container">
<div
class="icon"
class:icon-small={size === "M" || size === "S"}
on:mouseover={() => (showTooltip = true)}
on:mouseleave={() => (showTooltip = false)}
>
<Icon name="InfoOutline" size="S" disabled={true} />
</div>
{#if showTooltip}
<div class="tooltip">
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
</div>
{/if}
</div>
{/if}
</div>
<style>
.container {
display: flex;
align-items: center;
}
.icon-container {
position: relative;
display: flex;
justify-content: center;
margin-top: 1px;
margin-left: 5px;
margin-right: 5px;
}
.tooltip {
position: absolute;
display: flex;
justify-content: center;
top: 15px;
z-index: 1;
width: 160px;
}
.icon {
transform: scale(0.75);
}
.icon-small {
margin-top: -2px;
margin-bottom: -5px;
}
</style>

22
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -152,6 +152,7 @@
delete field.subtype
delete field.tableId
delete field.relationshipType
delete field.formulaType
// Add in defaults and initial definition
const definition = fieldDefinitions[event.detail?.toUpperCase()]
@ -163,6 +164,9 @@
if (field.type === LINK_TYPE) {
field.relationshipType = RelationshipTypes.MANY_TO_MANY
}
if (field.type === FORMULA_TYPE) {
field.formulaType = "dynamic"
}
}
function onChangeRequired(e) {
@ -438,8 +442,22 @@
error={errors.relatedName}
/>
{:else if field.type === FORMULA_TYPE}
{#if !table.sql}
<Select
label="Formula type"
bind:value={field.formulaType}
options={[
{ label: "Dynamic", value: "dynamic" },
{ label: "Static", value: "static" },
]}
getOptionLabel={option => option.label}
getOptionValue={option => option.value}
tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered,
while static formula are calculated when the row is saved."
/>
{/if}
<ModalBindableInput
title="Handlebars Formula"
title="Formula"
label="Formula"
value={field.formula}
on:change={e => (field.formula = e.detail)}
@ -448,7 +466,7 @@
/>
{:else if field.type === AUTO_TYPE}
<Select
label="Auto Column Type"
label="Auto column type"
value={field.subtype}
on:change={e => (field.subtype = e.detail)}
options={Object.entries(getAutoColumnInformation())}

2
packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte
View File

@ -131,7 +131,7 @@
{bindings}
on:change={event => (filter.value = event.detail)}
/>
{:else if ["string", "longform", "number"].includes(filter.type)}
{:else if ["string", "longform", "number", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
<Combobox

30
packages/builder/src/constants/lucene.js
View File

@ -59,8 +59,7 @@ export const NoEmptyFilterStrings = [
*/
export const getValidOperatorsForType = type => {
const Op = OperatorOptions
if (type === "string") {
return [
const stringOps = [
Op.Equals,
Op.NotEquals,
Op.StartsWith,
@ -68,8 +67,7 @@ export const getValidOperatorsForType = type => {
Op.Empty,
Op.NotEmpty,
]
} else if (type === "number") {
return [
const numOps = [
Op.Equals,
Op.NotEquals,
Op.MoreThan,
@ -77,6 +75,10 @@ export const getValidOperatorsForType = type => {
Op.Empty,
Op.NotEmpty,
]
if (type === "string") {
return stringOps
} else if (type === "number") {
return numOps
} else if (type === "options") {
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
} else if (type === "array") {
@ -84,23 +86,11 @@ export const getValidOperatorsForType = type => {
} else if (type === "boolean") {
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
} else if (type === "longform") {
return [
Op.Equals,
Op.NotEquals,
Op.StartsWith,
Op.Like,
Op.Empty,
Op.NotEmpty,
]
return stringOps
} else if (type === "datetime") {
return [
Op.Equals,
Op.NotEquals,
Op.MoreThan,
Op.LessThan,
Op.Empty,
Op.NotEmpty,
]
return numOps
} else if (type === "formula") {
return stringOps.concat([Op.MoreThan, Op.LessThan])
}
return []
}

5
packages/builder/src/helpers/searchFields.js
View File

@ -27,5 +27,8 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) {
filteredFields = filteredFields.concat(getTableFields(linkField))
}
}
return filteredFields
const staticFormulaFields = fields.filter(
field => field.type === "formula" && field.formulaType === "static"
)
return filteredFields.concat(staticFormulaFields)
}

1
packages/client/src/components/app/blocks/CardsBlock.svelte
View File

@ -39,6 +39,7 @@
number: "numberfield",
datetime: "datetimefield",
boolean: "booleanfield",
formula: "stringfield",
}
let formId

6
packages/client/src/components/app/blocks/TableBlock.svelte
View File

@ -35,6 +35,7 @@
number: "numberfield",
datetime: "datetimefield",
boolean: "booleanfield",
formula: "stringfield",
}
let formId
@ -60,10 +61,11 @@
let enrichedFilter = [...(filter || [])]
columns?.forEach(column => {
const safePath = column.name.split(".").map(safe).join(".")
const stringType = column.type === "string" || column.type === "formula"
enrichedFilter.push({
field: column.name,
operator: column.type === "string" ? "string" : "equal",
type: column.type === "string" ? "string" : "number",
operator: stringType ? "string" : "equal",
type: stringType ? "string" : "number",
valueType: "Binding",
value: `{{ ${safe(formId)}.${safePath} }}`,
})

10
packages/client/src/components/app/dynamic-filter/FilterModal.svelte
View File

@ -19,10 +19,14 @@
export let schemaFields
export let filters = []
const BannedTypes = ["link", "attachment", "formula", "json"]
const BannedTypes = ["link", "attachment", "json"]
$: fieldOptions = (schemaFields ?? [])
.filter(field => !BannedTypes.includes(field.type))
.filter(
field =>
!BannedTypes.includes(field.type) ||
(field.type === "formula" && field.formulaType === "static")
)
.map(field => field.name)
const addFilter = () => {
@ -114,7 +118,7 @@
on:change={e => onOperatorChange(filter, e.detail)}
placeholder={null}
/>
{#if ["string", "longform", "number"].includes(filter.type)}
{#if ["string", "longform", "number", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
<Combobox

3
packages/client/src/components/app/forms/Field.svelte
View File

@ -32,6 +32,7 @@
validation,
formStep
)
$: schemaType = fieldSchema?.type !== "formula" ? fieldSchema?.type : "string"
// Focus label when editing
let labelNode
@ -72,7 +73,7 @@
<Placeholder
text="Add the Field setting to start using your component"
/>
{:else if fieldSchema?.type && fieldSchema?.type !== type && type !== "options"}
{:else if schemaType && schemaType !== type && type !== "options"}
<Placeholder
text="This Field setting is the wrong data type for this component"
/>

79
packages/server/src/api/controllers/row/internal.js
View File

@ -1,8 +1,8 @@
const CouchDB = require("../../../db")
const linkRows = require("../../../db/linkedRows")
const {
getRowParams,
generateRowID,
getRowParams,
DocumentTypes,
InternalTables,
} = require("../../../db/utils")
@ -10,11 +10,9 @@ const userController = require("../user")
const {
inputProcessing,
outputProcessing,
processAutoColumn,
cleanupAttachments,
} = require("../../../utilities/rowProcessor")
const { FieldTypes } = require("../../../constants")
const { isEqual } = require("lodash")
const { validate, findRow } = require("./utils")
const { fullSearch, paginatedSearch } = require("./internalSearch")
const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@ -27,6 +25,7 @@ const {
getFromMemoryDoc,
} = require("../view/utils")
const { cloneDeep } = require("lodash/fp")
const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
const CALCULATION_TYPES = {
SUM: "sum",
@ -34,51 +33,6 @@ const CALCULATION_TYPES = {
STATS: "stats",
}
async function storeResponse(ctx, db, row, oldTable, table) {
row.type = "row"
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (!isEqual(oldTable, table)) {
try {
await db.put(table)
} catch (err) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})
await db.put(response.table)
row = response.row
} else {
throw err
}
}
}
const response = await db.put(row)
row._rev = response.rev
// process the row before return, to include relationships
row = await outputProcessing(ctx, table, row, { squash: false })
return { row, table }
}
// doesn't do the outputProcessing
async function getRawTableData(ctx, db, tableId) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows
}
async function getView(db, viewName) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@ -105,6 +59,22 @@ async function getView(db, viewName) {
return viewInfo
}
async function getRawTableData(ctx, db, tableId) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows
}
exports.patch = async ctx => {
const appId = ctx.appId
const db = new CouchDB(appId)
@ -162,7 +132,10 @@ exports.patch = async ctx => {
return { row: ctx.body, table }
}
return storeResponse(ctx, db, row, dbTable, table)
return finaliseRow(ctx.appId, table, row, {
oldTable: dbTable,
updateFormula: true,
})
}
exports.save = async function (ctx) {
@ -196,7 +169,10 @@ exports.save = async function (ctx) {
table,
})
return storeResponse(ctx, db, row, dbTable, table)
return finaliseRow(ctx.appId, table, row, {
oldTable: dbTable,
updateFormula: true,
})
}
exports.fetchView = async ctx => {
@ -302,6 +278,8 @@ exports.destroy = async function (ctx) {
})
// remove any attachments that were on the row from object storage
await cleanupAttachments(appId, table, { row })
// remove any static formula
await updateRelatedFormula(appId, table, row)
let response
if (ctx.params.tableId === InternalTables.USER_METADATA) {
@ -350,6 +328,7 @@ exports.bulkDestroy = async ctx => {
}
// remove any attachments that were on the rows from object storage
await cleanupAttachments(appId, table, { rows })
await updateRelatedFormula(appId, table, rows)
await Promise.all(updates)
return { response: { ok: true }, rows }
}

8
packages/server/src/api/controllers/row/internalSearch.js
View File

@ -37,22 +37,30 @@ class QueryBuilder {
}
setLimit(limit) {
if (limit != null) {
this.limit = limit
}
return this
}
setSort(sort) {
if (sort != null) {
this.sort = sort
}
return this
}
setSortOrder(sortOrder) {
if (sortOrder != null) {
this.sortOrder = sortOrder
}
return this
}
setSortType(sortType) {
if (sortType != null) {
this.sortType = sortType
}
return this
}

157
packages/server/src/api/controllers/row/staticFormula.js Normal file
View File

@ -0,0 +1,157 @@
const CouchDB = require("../../../db")
const { getRowParams } = require("../../../db/utils")
const {
outputProcessing,
processAutoColumn,
processFormulas,
} = require("../../../utilities/rowProcessor")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { isEqual } = require("lodash")
const { cloneDeep } = require("lodash/fp")
/**
* This function runs through a list of enriched rows, looks at the rows which
* are related and then checks if they need the state of their formulas
* updated.
* NOTE: this will only for affect static formulas.
*/
exports.updateRelatedFormula = async (appId, table, enrichedRows) => {
const db = new CouchDB(appId)
// no formula to update, we're done
if (!table.relatedFormula) {
return
}
let promises = []
for (let enrichedRow of Array.isArray(enrichedRows)
? enrichedRows
: [enrichedRows]) {
// the related rows by tableId
let relatedRows = {}
for (let [key, field] of Object.entries(enrichedRow)) {
const columnDefinition = table.schema[key]
if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
const relatedTableId = columnDefinition.tableId
if (!relatedRows[relatedTableId]) {
relatedRows[relatedTableId] = []
}
relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
}
}
for (let tableId of table.relatedFormula) {
let relatedTable
try {
// no rows to update, skip
if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
continue
}
relatedTable = await db.get(tableId)
} catch (err) {
// no error scenario, table doesn't seem to exist anymore, ignore
}
for (let column of Object.values(relatedTable.schema)) {
// needs updated in related rows
if (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
) {
// re-enrich rows for all the related, don't update the related formula for them
promises = promises.concat(
relatedRows[tableId].map(related =>
exports.finaliseRow(appId, relatedTable, related, {
updateFormula: false,
})
)
)
}
}
}
}
await Promise.all(promises)
}
exports.updateAllFormulasInTable = async (appId, table) => {
const db = new CouchDB(appId)
// start by getting the raw rows (which will be written back to DB after update)
let rows = (
await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
// now enrich the rows, note the clone so that we have the base state of the
// rows so that we don't write any of the enriched information back
let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), {
squash: false,
})
const updatedRows = []
for (let row of rows) {
// find the enriched row, if found process the formulas
const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
if (enrichedRow) {
const processed = processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: enrichedRow,
})
// values have changed, need to add to bulk docs to update
if (!isEqual(processed, row)) {
updatedRows.push(processed)
}
}
}
await db.bulkDocs(updatedRows)
}
/**
* This function runs at the end of the save/patch functions of the row controller, all this
* really does is enrich the row, handle any static formula processing, then return the enriched
* row. The reason we need to return the enriched row is that the automation row created trigger
* expects the row to be totally enriched/contain all relationships.
*/
exports.finaliseRow = async (
appId,
table,
row,
{ oldTable, updateFormula } = { updateFormula: true }
) => {
const db = new CouchDB(appId)
row.type = "row"
// process the row before return, to include relationships
let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
squash: false,
})
// use enriched row to generate formulas for saving, specifically only use as context
row = processFormulas(table, row, {
dynamic: false,
contextRows: enrichedRow,
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (oldTable && !isEqual(oldTable, table)) {
try {
await db.put(table)
} catch (err) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})
await db.put(response.table)
row = response.row
} else {
throw err
}
}
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await exports.updateRelatedFormula(appId, table, enrichedRow)
}
return { row: enrichedRow, table }
}

183
packages/server/src/api/controllers/table/bulkFormula.js Normal file
View File

@ -0,0 +1,183 @@
const CouchDB = require("../../../db")
const { FieldTypes, FormulaTypes } = require("../../../constants")
const { getAllInternalTables, clearColumns } = require("./utils")
const { doesContainStrings } = require("@budibase/string-templates")
const { cloneDeep } = require("lodash/fp")
const { isEqual, uniq } = require("lodash")
const { updateAllFormulasInTable } = require("../row/staticFormula")
function isStaticFormula(column) {
return (
column.type === FieldTypes.FORMULA &&
column.formulaType === FormulaTypes.STATIC
)
}
/**
* This retrieves the formula columns from a table schema that use a specified column name
* in the formula.
*/
function getFormulaThatUseColumn(table, columnNames) {
let formula = []
columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
for (let column of Object.values(table.schema)) {
// not a static formula, or doesn't contain a relationship
if (!isStaticFormula(column)) {
continue
}
if (!doesContainStrings(column.formula, columnNames)) {
continue
}
formula.push(column.name)
}
return formula
}
/**
* This functions checks for when a related table, column or related column is deleted, if any
* tables need to have the formula column removed.
*/
async function checkIfFormulaNeedsCleared(
appId,
table,
{ oldTable, deletion }
) {
const db = new CouchDB(appId)
// start by retrieving all tables, remove the current table from the list
const tables = (await getAllInternalTables(appId)).filter(
tbl => tbl._id !== table._id
)
const schemaToUse = oldTable ? oldTable.schema : table.schema
let removedColumns = Object.values(schemaToUse).filter(
column => deletion || !table.schema[column.name]
)
// remove any formula columns that used related columns
for (let removed of removedColumns) {
let tableToUse = table
// if relationship, get the related table
if (removed.type === FieldTypes.LINK) {
tableToUse = tables.find(table => table._id === removed.tableId)
}
const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name)
if (columnsToDelete.length > 0) {
await clearColumns(db, table, columnsToDelete)
}
// need a special case, where a column has been removed from this table, but was used
// in a different, related tables formula
if (!table.relatedFormula) {
continue
}
for (let relatedTableId of table.relatedFormula) {
const relatedColumns = Object.values(table.schema).filter(
column => column.tableId === relatedTableId
)
const relatedTable = tables.find(table => table._id === relatedTableId)
// look to see if the column was used in a relationship formula,
// relationships won't be used for this
if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
let relatedFormulaToRemove = []
for (let column of relatedColumns) {
relatedFormulaToRemove = relatedFormulaToRemove.concat(
getFormulaThatUseColumn(relatedTable, [
column.fieldName,
removed.name,
])
)
}
if (relatedFormulaToRemove.length > 0) {
await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove))
}
}
}
}
}
/**
* This function adds a note to related tables that they are
* used in a static formula - so that the link controller
* can manage hydrating related rows formula fields. This is
* specifically only for static formula.
*/
async function updateRelatedFormulaLinksOnTables(
appId,
table,
{ deletion } = { deletion: false }
) {
const db = new CouchDB(appId)
// start by retrieving all tables, remove the current table from the list
const tables = (await getAllInternalTables(appId)).filter(
tbl => tbl._id !== table._id
)
// clone the tables, so we can compare at end
const initialTables = cloneDeep(tables)
// first find the related column names
const relatedColumns = Object.values(table.schema).filter(
col => col.type === FieldTypes.LINK
)
// we start by removing the formula field from all tables
for (let otherTable of tables) {
if (!otherTable.relatedFormula) {
continue
}
const index = otherTable.relatedFormula.indexOf(table._id)
if (index !== -1) {
otherTable.relatedFormula.splice(index, 1)
}
}
// if deleting, just remove the table IDs, don't try add
if (!deletion) {
for (let relatedCol of relatedColumns) {
let columns = getFormulaThatUseColumn(table, relatedCol.name)
if (!columns || columns.length === 0) {
continue
}
const relatedTable = tables.find(
related => related._id === relatedCol.tableId
)
// check if the table is already in the list of related formula, if it isn't, then add it
if (
relatedTable &&
(!relatedTable.relatedFormula ||
!relatedTable.relatedFormula.includes(table._id))
) {
relatedTable.relatedFormula = relatedTable.relatedFormula
? [...relatedTable.relatedFormula, table._id]
: [table._id]
}
}
}
// now we just need to compare all the tables and see if any need saved
for (let initial of initialTables) {
const found = tables.find(tbl => initial._id === tbl._id)
if (found && !isEqual(initial, found)) {
await db.put(found)
}
}
}
async function checkIfFormulaUpdated(appId, table, { oldTable }) {
// look to see if any formula values have changed
const shouldUpdate = Object.values(table.schema).find(
column =>
isStaticFormula(column) &&
(!oldTable ||
!oldTable.schema[column.name] ||
!isEqual(oldTable.schema[column.name], column))
)
// if a static formula column has updated, then need to run the update
if (shouldUpdate != null) {
await updateAllFormulasInTable(appId, table)
}
}
exports.runStaticFormulaChecks = async (
appId,
table,
{ oldTable, deletion }
) => {
await updateRelatedFormulaLinksOnTables(appId, table, { deletion })
await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion })
if (!deletion) {
await checkIfFormulaUpdated(appId, table, { oldTable })
}
}

20
packages/server/src/api/controllers/table/index.js
View File

@ -3,12 +3,8 @@ const internal = require("./internal")
const external = require("./external")
const csvParser = require("../../../utilities/csvParser")
const { isExternalTable, isSQL } = require("../../../integrations/utils")
const {
getTableParams,
getDatasourceParams,
BudibaseInternalDB,
} = require("../../../db/utils")
const { getTable } = require("./utils")
const { getDatasourceParams } = require("../../../db/utils")
const { getTable, getAllInternalTables } = require("./utils")
function pickApi({ tableId, table }) {
if (table && !tableId) {
@ -26,17 +22,7 @@ function pickApi({ tableId, table }) {
exports.fetch = async function (ctx) {
const db = new CouchDB(ctx.appId)
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
})
)
const internal = internalTables.rows.map(tableDoc => ({
...tableDoc.doc,
type: "internal",
sourceId: BudibaseInternalDB._id,
}))
const internal = await getAllInternalTables(ctx.appId)
const externalTables = await db.allDocs(
getDatasourceParams("plus", {

8
packages/server/src/api/controllers/table/internal.js
View File

@ -9,6 +9,8 @@ const {
handleDataImport,
} = require("./utils")
const usageQuota = require("../../../utilities/usageQuota")
const { cleanupAttachments } = require("../../../utilities/rowProcessor")
const { runStaticFormulaChecks } = require("./bulkFormula")
exports.save = async function (ctx) {
const appId = ctx.appId
@ -104,7 +106,8 @@ exports.save = async function (ctx) {
tableToSave._rev = result.rev
tableToSave = await tableSaveFunctions.after(tableToSave)
// has to run after, make sure it has _id
await runStaticFormulaChecks(appId, tableToSave, { oldTable })
return tableToSave
}
@ -141,6 +144,9 @@ exports.destroy = async function (ctx) {
await db.deleteIndex(existingIndex)
}
// has to run after, make sure it has _id
await runStaticFormulaChecks(appId, tableToDelete, { deletion: true })
await cleanupAttachments(appId, tableToDelete, { rows })
return tableToDelete
}

56
packages/server/src/api/controllers/table/utils.js
View File

@ -4,10 +4,15 @@ const {
getRowParams,
generateRowID,
InternalTables,
getTableParams,
BudibaseInternalDB,
} = require("../../../db/utils")
const { isEqual } = require("lodash/fp")
const { isEqual } = require("lodash")
const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
const { inputProcessing } = require("../../../utilities/rowProcessor")
const {
inputProcessing,
cleanupAttachments,
} = require("../../../utilities/rowProcessor")
const {
USERS_TABLE_SCHEMA,
SwitchableTypes,
@ -21,8 +26,24 @@ const {
const { getViews, saveView } = require("../view/utils")
const viewTemplate = require("../view/viewBuilder")
const usageQuota = require("../../../utilities/usageQuota")
const { cloneDeep } = require("lodash/fp")
exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
exports.clearColumns = async (appId, table, columnNames) => {
const db = new CouchDB(appId)
const rows = await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,
})
)
return db.bulkDocs(
rows.rows.map(({ doc }) => {
columnNames.forEach(colName => delete doc[colName])
return doc
})
)
}
exports.checkForColumnUpdates = async (appId, db, oldTable, updatedTable) => {
let updatedRows = []
const rename = updatedTable._rename
let deletedColumns = []
@ -39,16 +60,20 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
include_docs: true,
})
)
updatedRows = rows.rows.map(({ doc }) => {
const rawRows = rows.rows.map(({ doc }) => doc)
updatedRows = rawRows.map(row => {
row = cloneDeep(row)
if (rename) {
doc[rename.updated] = doc[rename.old]
delete doc[rename.old]
row[rename.updated] = row[rename.old]
delete row[rename.old]
} else if (deletedColumns.length !== 0) {
deletedColumns.forEach(colName => delete doc[colName])
deletedColumns.forEach(colName => delete row[colName])
}
return doc
return row
})
// cleanup any attachments from object storage for deleted attachment columns
await cleanupAttachments(appId, updatedTable, { oldTable, rows: rawRows })
// Update views
await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
delete updatedTable._rename
@ -209,6 +234,7 @@ class TableSaveFunctions {
// when confirmed valid
async mid(table) {
let response = await exports.checkForColumnUpdates(
this.appId,
this.db,
this.oldTable,
table
@ -234,6 +260,20 @@ class TableSaveFunctions {
}
}
exports.getAllInternalTables = async appId => {
const db = new CouchDB(appId)
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
})
)
return internalTables.rows.map(tableDoc => ({
...tableDoc.doc,
type: "internal",
sourceId: BudibaseInternalDB._id,
}))
}
exports.getAllExternalTables = async (appId, datasourceId) => {
const db = new CouchDB(appId)
const datasource = await db.get(datasourceId)

5
packages/server/src/constants/index.js
View File

@ -61,6 +61,11 @@ exports.RelationshipTypes = {
MANY_TO_MANY: "many-to-many",
}
exports.FormulaTypes = {
STATIC: "static",
DYNAMIC: "dynamic",
}
exports.AuthTypes = {
APP: "app",
BUILDER: "builder",

9
packages/server/src/db/linkedRows/index.js
View File

@ -72,7 +72,7 @@ async function getLinksForRows(appId, rows) {
)
}
async function getFullLinkedDocs(ctx, appId, links) {
async function getFullLinkedDocs(appId, links) {
// create DBs
const db = new CouchDB(appId)
const linkedRowIds = links.map(link => link.id)
@ -146,13 +146,12 @@ exports.updateLinks = async function (args) {
/**
* Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
* This is required for formula fields, this may only be utilised internally (for now).
* @param {object} ctx The request which is looking for rows.
* @param {string} appId The ID of the app which this request is in the context of.
* @param {object} table The table from which the rows originated.
* @param {array<object>} rows The rows which are to be enriched.
* @return {Promise<*>} returns the rows with all of the enriched relationships on it.
*/
exports.attachFullLinkedDocs = async (ctx, table, rows) => {
const appId = ctx.appId
exports.attachFullLinkedDocs = async (appId, table, rows) => {
const linkedTableIds = getLinkedTableIDs(table)
if (linkedTableIds.length === 0) {
return rows
@ -166,7 +165,7 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => {
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
// now get the docs and combine into the rows
let linked = await getFullLinkedDocs(ctx, appId, links)
let linked = await getFullLinkedDocs(appId, links)
const linkedTables = []
for (let row of rows) {
for (let link of links.filter(link => link.thisId === row._id)) {

3
packages/server/src/definitions/common.ts
View File

@ -17,6 +17,8 @@ export interface FieldSchema {
autocolumn?: boolean
throughFrom?: string
throughTo?: string
formula?: string
formulaType?: string
main?: boolean
meta?: {
toTable: string
@ -46,6 +48,7 @@ export interface Table extends Base {
schema: TableSchema
primaryDisplay?: string
sourceId?: string
relatedFormula?: string[]
constrained?: string[]
}

34
packages/server/src/utilities/rowProcessor/index.js
View File

@ -180,6 +180,8 @@ function processAutoColumn(
}
exports.processAutoColumn = processAutoColumn
exports.processFormulas = processFormulas
/**
* This will coerce a value to the correct types based on the type transform map
* @param {object} row The value to coerce
@ -229,11 +231,12 @@ exports.inputProcessing = (
}
continue
}
// specific case to delete formula values if they get saved
// type coercion cannot completely remove the field, so have to do it here
// remove any formula values, they are to be generated
if (field.type === FieldTypes.FORMULA) {
delete clonedRow[key]
} else {
}
// otherwise coerce what is there to correct types
else {
clonedRow[key] = exports.coerce(value, field.type)
}
}
@ -250,7 +253,7 @@ exports.inputProcessing = (
/**
* This function enriches the input rows with anything they are supposed to contain, for example
* link records or attachment links.
* @param {object} ctx the request which is looking for enriched rows.
* @param {string} appId the app in which the request is looking for enriched rows.
* @param {object} table the table from which these rows came from originally, this is used to determine
* the schema of the rows and then enrich.
* @param {object[]|object} rows the rows which are to be enriched.
@ -258,22 +261,21 @@ exports.inputProcessing = (
* @returns {object[]|object} the enriched rows will be returned.
*/
exports.outputProcessing = async (
ctx,
{ appId },
table,
rows,
opts = { squash: true }
) => {
const appId = ctx.appId
let wasArray = true
if (!(rows instanceof Array)) {
rows = [rows]
wasArray = false
}
// attach any linked row information
let enriched = await linkRows.attachFullLinkedDocs(ctx, table, rows)
let enriched = await linkRows.attachFullLinkedDocs(appId, table, rows)
// process formulas
enriched = processFormulas(table, enriched)
enriched = processFormulas(table, enriched, { dynamic: true })
// update the attachments URL depending on hosting
for (let [property, column] of Object.entries(table.schema)) {
@ -305,9 +307,15 @@ exports.outputProcessing = async (
* @param {any} row optional - the row being removed.
* @param {any} rows optional - if multiple rows being deleted can do this in bulk.
* @param {any} oldRow optional - if updating a row this will determine the difference.
* @param {any} oldTable optional - if updating a table, can supply the old table to look for
* deleted attachment columns.
* @return {Promise<void>} When all attachments have been removed this will return.
*/
exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
exports.cleanupAttachments = async (
appId,
table,
{ row, rows, oldRow, oldTable }
) => {
if (!isProdAppID(appId)) {
const prodAppId = getDeployedAppID(appId)
// if prod exists, then don't allow deleting
@ -322,12 +330,16 @@ exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
files = files.concat(row[key].map(attachment => attachment.key))
}
}
for (let [key, schema] of Object.entries(table.schema)) {
const schemaToUse = oldTable ? oldTable.schema : table.schema
for (let [key, schema] of Object.entries(schemaToUse)) {
if (schema.type !== FieldTypes.ATTACHMENT) {
continue
}
// old table had this column, new table doesn't - delete it
if (oldTable && !table.schema[key]) {
rows.forEach(row => addFiles(row, key))
} else if (oldRow && row) {
// if updating, need to manage the differences
if (oldRow && row) {
files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
} else if (row) {
addFiles(row, key)

28
packages/server/src/utilities/rowProcessor/utils.js
View File

@ -1,23 +1,39 @@
const { FieldTypes } = require("../../constants")
const { FieldTypes, FormulaTypes } = require("../../constants")
const { processStringSync } = require("@budibase/string-templates")
/**
* Looks through the rows provided and finds formulas - which it then processes.
*/
exports.processFormulas = (table, rows) => {
exports.processFormulas = (
table,
rows,
{ dynamic, contextRows } = { dynamic: true }
) => {
const single = !Array.isArray(rows)
if (single) {
rows = [rows]
contextRows = contextRows ? [contextRows] : contextRows
}
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
const isStatic = schema.formulaType === FormulaTypes.STATIC
if (
schema.type !== FieldTypes.FORMULA ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
rows = rows.map(row => ({
for (let i = 0; i < rows.length; i++) {
if (schema.formula) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
rows[i] = {
...row,
[column]: processStringSync(schema.formula, row),
}))
[column]: processStringSync(schema.formula, context),
}
}
}
}
return single ? rows[0] : rows
}

2
packages/string-templates/src/index.cjs
View File

@ -15,6 +15,8 @@ module.exports.processStringSync = templates.processStringSync
module.exports.processObjectSync = templates.processObjectSync
module.exports.processString = templates.processString
module.exports.processObject = templates.processObject
module.exports.doesContainStrings = templates.doesContainStrings
module.exports.doesContainString = templates.doesContainString
/**
* Use vm2 to run JS scripts in a node env

53
packages/string-templates/src/index.js
View File

@ -3,6 +3,7 @@ const { registerAll } = require("./helpers/index")
const processors = require("./processors")
const { atob, btoa } = require("./utilities")
const manifest = require("../manifest.json")
const { FIND_HBS_REGEX } = require("./utilities")
const hbsInstance = handlebars.create()
registerAll(hbsInstance)
@ -26,7 +27,7 @@ function testObject(object) {
* @param {object|array} object The input structure which is to be recursed, it is important to note that
* if the structure contains any cycles then this will fail.
* @param {object} context The context that handlebars should fill data from.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {Promise<object|array>} The structure input, as fully updated as possible.
*/
module.exports.processObject = async (object, context, opts) => {
@ -57,7 +58,7 @@ module.exports.processObject = async (object, context, opts) => {
* then nothing will occur.
* @param {string} string The template string which is the filled from the context object.
* @param {object} context An object of information which will be used to enrich the string.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
*/
module.exports.processString = async (string, context, opts) => {
@ -71,7 +72,7 @@ module.exports.processString = async (string, context, opts) => {
* @param {object|array} object The input structure which is to be recursed, it is important to note that
* if the structure contains any cycles then this will fail.
* @param {object} context The context that handlebars should fill data from.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {object|array} The structure input, as fully updated as possible.
*/
module.exports.processObjectSync = (object, context, opts) => {
@ -92,7 +93,7 @@ module.exports.processObjectSync = (object, context, opts) => {
* then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
* @param {string} string The template string which is the filled from the context object.
* @param {object} context An object of information which will be used to enrich the string.
* @param {object} opts optional - specify some options for processing.
* @param {object|null} opts optional - specify some options for processing.
* @returns {string} The enriched string, all templates should have been replaced if they can be.
*/
module.exports.processStringSync = (string, context, opts) => {
@ -222,3 +223,47 @@ module.exports.decodeJSBinding = handlebars => {
}
return atob(match[1])
}
/**
* Same as the doesContainString function, but will check for all the strings
* before confirming it contains.
* @param {string} template The template string to search.
* @param {string[]} strings The strings to look for.
* @returns {boolean} Will return true if all strings found in HBS statement.
*/
module.exports.doesContainStrings = (template, strings) => {
let regexp = new RegExp(FIND_HBS_REGEX)
let matches = template.match(regexp)
if (matches == null) {
return false
}
for (let match of matches) {
let hbs = match
if (exports.isJSBinding(match)) {
hbs = exports.decodeJSBinding(match)
}
let allFound = true
for (let string of strings) {
if (!hbs.includes(string)) {
allFound = false
}
}
if (allFound) {
return true
}
}
return false
}
/**
* This function looks in the supplied template for handlebars instances, if they contain
* JS the JS will be decoded and then the supplied string will be looked for. For example
* if the template "Hello, your name is {{ related }}" this function would return that true
* for the string "related" but not for "name" as it is not within the handlebars statement.
* @param {string} template A template string to search for handlebars instances.
* @param {string} string The word or sentence to search for.
* @returns {boolean} The this return true if the string is found, false if not.
*/
module.exports.doesContainString = (template, string) => {
return exports.doesContainStrings(template, [string])
}

2
packages/string-templates/src/index.mjs
View File

@ -15,6 +15,8 @@ export const processStringSync = templates.processStringSync
export const processObjectSync = templates.processObjectSync
export const processString = templates.processString
export const processObject = templates.processObject
export const doesContainStrings = templates.doesContainStrings
export const doesContainString = templates.doesContainString
/**
* Use polyfilled vm to run JS scripts in a browser Env

19
packages/string-templates/test/basic.spec.js
View File

@ -4,6 +4,8 @@ const {
isValid,
makePropSafe,
getManifest,
encodeJSBinding,
doesContainString,
} = require("../src/index.cjs")
describe("Test that the string processing works correctly", () => {
@ -157,3 +159,20 @@ describe("check full stops that are safe", () => {
expect(output).toEqual("1")
})
})
describe("check does contain string function", () => {
it("should work for a simple case", () => {
const hbs = "hello {{ name }}"
expect(doesContainString(hbs, "name")).toEqual(true)
})
it("should reject a case where its in the string, but not the handlebars", () => {
const hbs = "hello {{ name }}"
expect(doesContainString(hbs, "hello")).toEqual(false)
})
it("should handle if its in javascript", () => {
const js = encodeJSBinding(`return $("foo")`)
expect(doesContainString(js, "foo")).toEqual(true)
})
})