MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge and typing

This commit is contained in:
Martin McKeaveney 2024-10-08 17:38:23 +01:00
parent d5324388d6 fda5fdc16e
commit 7b683cfc50
57 changed files with 4160 additions and 3144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.10",
"version": "2.32.11",
"npmClient": "yarn",
"packages": [
"packages/*",

143
packages/backend-core/src/sql/sql.ts
View File

@ -139,29 +139,61 @@ class InternalBuilder {
return this.table.schema[column]
}
// Takes a string like foo and returns a quoted string like [foo] for SQL Server
// and "foo" for Postgres.
private quote(str: string): string {
private quoteChars(): [string, string] {
switch (this.client) {
case SqlClient.SQL_LITE:
case SqlClient.ORACLE:
case SqlClient.POSTGRES:
return `"${str}"`
return ['"', '"']
case SqlClient.MS_SQL:
return `[${str}]`
return ["[", "]"]
case SqlClient.MARIADB:
case SqlClient.MY_SQL:
return `\`${str}\``
case SqlClient.SQL_LITE:
return ["`", "`"]
}
}
// Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
// for SQL Server and `a`.`b`.`c` for MySQL.
private quotedIdentifier(key: string): string {
return key
.split(".")
.map(part => this.quote(part))
.join(".")
// Takes a string like foo and returns a quoted string like [foo] for SQL Server
// and "foo" for Postgres.
private quote(str: string): string {
const [start, end] = this.quoteChars()
return `${start}${str}${end}`
}
private isQuoted(key: string): boolean {
const [start, end] = this.quoteChars()
return key.startsWith(start) && key.endsWith(end)
}
// Takes a string like a.b.c or an array like ["a", "b", "c"] and returns a
// quoted identifier like [a].[b].[c] for SQL Server and `a`.`b`.`c` for
// MySQL.
private quotedIdentifier(key: string | string[]): string {
if (!Array.isArray(key)) {
key = this.splitIdentifier(key)
}
return key.map(part => this.quote(part)).join(".")
}
// Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"]
private splitIdentifier(key: string): string[] {
const [start, end] = this.quoteChars()
if (this.isQuoted(key)) {
return key.slice(1, -1).split(`${end}.${start}`)
}
return key.split(".")
}
private qualifyIdentifier(key: string): string {
const tableName = this.getTableName()
const parts = this.splitIdentifier(key)
if (parts[0] !== tableName) {
parts.unshift(tableName)
}
if (this.isQuoted(key)) {
return this.quotedIdentifier(parts)
}
return parts.join(".")
}
private isFullSelectStatementRequired(): boolean {
@ -231,8 +263,13 @@ class InternalBuilder {
// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
// so when we use them we need to wrap them in to_char(). This function
// converts a field name to the appropriate identifier.
private convertClobs(field: string): string {
const parts = field.split(".")
private convertClobs(field: string, opts?: { forSelect?: boolean }): string {
if (this.client !== SqlClient.ORACLE) {
throw new Error(
"you've called convertClobs on a DB that's not Oracle, this is a mistake"
)
}
const parts = this.splitIdentifier(field)
const col = parts.pop()!
const schema = this.table.schema[col]
let identifier = this.quotedIdentifier(field)
@ -244,7 +281,11 @@ class InternalBuilder {
schema.type === FieldType.OPTIONS ||
schema.type === FieldType.BARCODEQR
) {
identifier = `to_char(${identifier})`
if (opts?.forSelect) {
identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}`
} else {
identifier = `to_char(${identifier})`
}
}
return identifier
}
@ -859,28 +900,58 @@ class InternalBuilder {
const fields = this.query.resource?.fields || []
const tableName = this.getTableName()
if (fields.length > 0) {
query = query.groupBy(fields.map(field => `${tableName}.${field}`))
query = query.select(fields.map(field => `${tableName}.${field}`))
const qualifiedFields = fields.map(field => this.qualifyIdentifier(field))
if (this.client === SqlClient.ORACLE) {
const groupByFields = qualifiedFields.map(field =>
this.convertClobs(field)
)
const selectFields = qualifiedFields.map(field =>
this.convertClobs(field, { forSelect: true })
)
query = query
.groupByRaw(groupByFields.join(", "))
.select(this.knex.raw(selectFields.join(", ")))
} else {
query = query.groupBy(qualifiedFields).select(qualifiedFields)
}
}
for (const aggregation of aggregations) {
const op = aggregation.calculationType
const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
switch (op) {
case CalculationType.COUNT:
query = query.count(field)
break
case CalculationType.SUM:
query = query.sum(field)
break
case CalculationType.AVG:
query = query.avg(field)
break
case CalculationType.MIN:
query = query.min(field)
break
case CalculationType.MAX:
query = query.max(field)
break
if (op === CalculationType.COUNT) {
if ("distinct" in aggregation && aggregation.distinct) {
if (this.client === SqlClient.ORACLE) {
const field = this.convertClobs(`${tableName}.${aggregation.field}`)
query = query.select(
this.knex.raw(
`COUNT(DISTINCT ${field}) as ${this.quotedIdentifier(
aggregation.name
)}`
)
)
} else {
query = query.countDistinct(
`${tableName}.${aggregation.field} as ${aggregation.name}`
)
}
} else {
query = query.count(`* as ${aggregation.name}`)
}
} else {
const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
switch (op) {
case CalculationType.SUM:
query = query.sum(field)
break
case CalculationType.AVG:
query = query.avg(field)
break
case CalculationType.MIN:
query = query.min(field)
break
case CalculationType.MAX:
query = query.max(field)
break
}
}
}
return query

2
packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
View File

@ -95,7 +95,7 @@
{#if isView}
<span>
<Toggle
value={action.allowedViews?.includes(viewId)}
value={action.allowedSources?.includes(viewId)}
on:change={e => toggleAction(action, e.detail)}
/>
</span>

100
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -4,6 +4,7 @@
Button,
Label,
Select,
Multiselect,
Toggle,
Icon,
DatePicker,
@ -142,9 +143,10 @@
}
$: initialiseField(field, savingColumn)
$: checkConstraints(editableColumn)
$: required = hasDefault
? false
: !!editableColumn?.constraints?.presence || primaryDisplay
$: required =
primaryDisplay ||
editableColumn?.constraints?.presence === true ||
editableColumn?.constraints?.presence?.allowEmpty === false
$: uneditable =
$tables.selected?._id === TableNames.USERS &&
UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@ -173,8 +175,8 @@
// used to select what different options can be displayed for column type
$: canBeDisplay =
canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
$: canHaveDefault =
isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type)
$: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
$: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
$: canBeRequired =
editableColumn?.type !== LINK_TYPE &&
!uneditable &&
@ -193,7 +195,6 @@
(originalName &&
SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn)
$: allowedTypes = getAllowedTypes(datasource).map(t => ({
fieldId: makeFieldId(t.type, t.subtype),
...t,
@ -211,6 +212,11 @@
},
...getUserBindings(),
]
$: sanitiseDefaultValue(
editableColumn.type,
editableColumn.constraints?.inclusion || [],
editableColumn.default
)
const fieldDefinitions = Object.values(FIELDS).reduce(
// Storing the fields by complex field id
@ -300,6 +306,22 @@
delete saveColumn.fieldName
}
// Ensure we don't have a default value if we can't have one
if (!canHaveDefault || !defaultValuesEnabled) {
delete saveColumn.default
}
// Ensure primary display columns are always required and don't have default values
if (primaryDisplay) {
saveColumn.constraints.presence = { allowEmpty: false }
delete saveColumn.default
}
// Ensure the field is not required if we have a default value
if (saveColumn.default) {
saveColumn.constraints.presence = false
}
try {
await tables.saveField({
originalName,
@ -547,6 +569,20 @@
return newError
}
const sanitiseDefaultValue = (type, options, defaultValue) => {
if (!defaultValue?.length) {
return
}
// Delete default value for options fields if the option is no longer available
if (type === FieldType.OPTIONS && !options.includes(defaultValue)) {
delete editableColumn.default
}
// Filter array default values to only valid options
if (type === FieldType.ARRAY) {
editableColumn.default = defaultValue.filter(x => options.includes(x))
}
}
onMount(() => {
mounted = true
})
@ -761,9 +797,9 @@
schema={table.schema}
/>
{:else if editableColumn.type === JSON_TYPE}
<Button primary text on:click={openJsonSchemaEditor}
>Open schema editor</Button
>
<Button primary text on:click={openJsonSchemaEditor}>
Open schema editor
</Button>
{/if}
{#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
<Select
@ -792,27 +828,39 @@
</div>
{/if}
{#if canHaveDefault}
<div>
<ModalBindableInput
panel={ServerBindingPanel}
title="Default"
label="Default"
{#if defaultValuesEnabled}
{#if editableColumn.type === FieldType.OPTIONS}
<Select
disabled={!canHaveDefault}
options={editableColumn.constraints?.inclusion || []}
label="Default value"
value={editableColumn.default}
on:change={e => {
editableColumn = {
...editableColumn,
default: e.detail,
}
if (e.detail) {
setRequired(false)
}
}}
on:change={e => (editableColumn.default = e.detail)}
placeholder="None"
/>
{:else if editableColumn.type === FieldType.ARRAY}
<Multiselect
disabled={!canHaveDefault}
options={editableColumn.constraints?.inclusion || []}
label="Default value"
value={editableColumn.default}
on:change={e =>
(editableColumn.default = e.detail?.length ? e.detail : undefined)}
placeholder="None"
/>
{:else}
<ModalBindableInput
disabled={!canHaveDefault}
panel={ServerBindingPanel}
title="Default value"
label="Default value"
placeholder="None"
value={editableColumn.default}
on:change={e => (editableColumn.default = e.detail)}
bindings={defaultValueBindings}
allowJS
/>
</div>
{/if}
{/if}
</Layout>

22
packages/builder/src/components/common/bindings/BindingPanel.svelte
View File

@ -66,6 +66,7 @@
let insertAtPos
let targetMode = null
let expressionResult
let expressionError
let evaluating = false
$: useSnippets = allowSnippets && !$licensing.isFreePlan
@ -142,10 +143,22 @@
}
const debouncedEval = Utils.debounce((expression, context, snippets) => {
expressionResult = processStringSync(expression || "", {
...context,
snippets,
})
try {
expressionError = null
expressionResult = processStringSync(
expression || "",
{
...context,
snippets,
},
{
noThrow: false,
}
)
} catch (err) {
expressionResult = null
expressionError = err
}
evaluating = false
}, 260)
@ -370,6 +383,7 @@
{:else if sidePanel === SidePanels.Evaluation}
<EvaluationSidePanel
{expressionResult}
{expressionError}
{evaluating}
expression={editorValue}
/>

19
packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte
View File

@ -3,26 +3,37 @@
import { Icon, ProgressCircle, notifications } from "@budibase/bbui"
import { copyToClipboard } from "@budibase/bbui/helpers"
import { fade } from "svelte/transition"
import { UserScriptError } from "@budibase/string-templates"
export let expressionResult
export let expressionError
export let evaluating = false
export let expression = null
$: error = expressionResult === "Error while executing JS"
$: error = expressionError != null
$: empty = expression == null || expression?.trim() === ""
$: success = !error && !empty
$: highlightedResult = highlight(expressionResult)
const formatError = err => {
if (err.code === UserScriptError.code) {
return err.userScriptError.toString()
}
return err.toString()
}
const highlight = json => {
if (json == null) {
return ""
}
// Attempt to parse and then stringify, in case this is valid JSON
// Attempt to parse and then stringify, in case this is valid result
try {
json = JSON.stringify(JSON.parse(json), null, 2)
} catch (err) {
// Ignore
}
return formatHighlight(json, {
keyColor: "#e06c75",
numberColor: "#e5c07b",
@ -34,7 +45,7 @@
}
const copy = () => {
let clipboardVal = expressionResult
let clipboardVal = expressionResult.result
if (typeof clipboardVal === "object") {
clipboardVal = JSON.stringify(clipboardVal, null, 2)
}
@ -73,6 +84,8 @@
<div class="body">
{#if empty}
Your expression will be evaluated here
{:else if error}
{formatError(expressionError)}
{:else}
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
{@html highlightedResult}

16
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
View File

@ -63,13 +63,15 @@
$: rowActions.refreshRowActions(id)
const makeRowActionButtons = actions => {
return (actions || []).map(action => ({
text: action.name,
onClick: async row => {
await rowActions.trigger(id, action.id, row._id)
notifications.success("Row action triggered successfully")
},
}))
return (actions || [])
.filter(action => action.allowedSources?.includes(id))
.map(action => ({
text: action.name,
onClick: async row => {
await rowActions.trigger(id, action.id, row._id)
notifications.success("Row action triggered successfully")
},
}))
}
const relationshipSupport = datasource => {

3
packages/builder/src/pages/builder/portal/apps/index.svelte
View File

@ -26,6 +26,7 @@
licensing,
environment,
enrichedApps,
sortBy,
} from "stores/portal"
import { goto } from "@roxi/routify"
import AppRow from "components/start/AppRow.svelte"
@ -247,7 +248,7 @@
<div class="app-actions">
<Select
autoWidth
value={$appsStore.sortBy}
value={$sortBy}
on:change={e => {
appsStore.updateSort(e.detail)
}}

12
packages/builder/src/stores/builder/rowActions.js
View File

@ -129,13 +129,15 @@ const derivedStore = derived(store, $store => {
// Generate an entry for every view as well
Object.keys($store || {}).forEach(tableId => {
// We need to have all the actions for the table in order to be displayed in the crud section
map[tableId] = $store[tableId]
for (let action of $store[tableId]) {
for (let viewId of action.allowedViews || []) {
if (!map[viewId]) {
map[viewId] = []
}
map[viewId].push(action)
const otherSources = (action.allowedSources || []).filter(
sourceId => sourceId !== tableId
)
for (let source of otherSources) {
map[source] ??= []
map[source].push(action)
}
}
})

23
packages/builder/src/stores/builder/viewsV2.js
View File

@ -1,6 +1,24 @@
import { writable, derived, get } from "svelte/store"
import { tables } from "./tables"
import { API } from "api"
import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
}
return view
}
function convertToSearchFilterGroup(view) {
if (view.queryUI) {
view.query = view.queryUI
delete view.queryUI
}
return view
}
export function createViewsV2Store() {
const store = writable({
@ -12,7 +30,7 @@ export function createViewsV2Store() {
const views = Object.values(table?.views || {}).filter(view => {
return view.version === 2
})
list = list.concat(views)
list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
})
return {
...$store,
@ -34,6 +52,7 @@ export function createViewsV2Store() {
}
const create = async view => {
view = convertToSearchFilters(view)
const savedViewResponse = await API.viewV2.create(view)
const savedView = savedViewResponse.data
replaceView(savedView.id, savedView)
@ -41,6 +60,7 @@ export function createViewsV2Store() {
}
const save = async view => {
view = convertToSearchFilters(view)
const res = await API.viewV2.update(view)
const savedView = res?.data
replaceView(view.id, savedView)
@ -51,6 +71,7 @@ export function createViewsV2Store() {
if (!viewId) {
return
}
view = convertToSearchFilterGroup(view)
const existingView = get(derivedStore).list.find(view => view.id === viewId)
const tableIndex = get(tables).list.findIndex(table => {
return table._id === view?.tableId || table._id === existingView?.tableId

89
packages/builder/src/stores/portal/apps.js
View File

@ -9,7 +9,6 @@ const DEV_PROPS = ["updatedBy", "updatedAt"]
export const INITIAL_APPS_STATE = {
apps: [],
sortBy: "name",
}
export class AppsStore extends BudiStore {
@ -53,6 +52,15 @@ export class AppsStore extends BudiStore {
...state,
sortBy,
}))
this.updateUserSort(sortBy)
}
async updateUserSort(sortBy) {
try {
await auth.updateSelf({ appSort: sortBy })
} catch (err) {
console.error("couldn't save user sort: ", err)
}
}
async load() {
@ -140,43 +148,50 @@ export class AppsStore extends BudiStore {
export const appsStore = new AppsStore()
// Centralise any logic that enriches the apps list
export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
const enrichedApps = $store.apps
? $store.apps.map(app => ({
...app,
deployed: app.status === AppStatus.DEPLOYED,
lockedYou: app.lockedBy && app.lockedBy.email === $auth.user?.email,
lockedOther: app.lockedBy && app.lockedBy.email !== $auth.user?.email,
favourite: $auth.user?.appFavourites?.includes(app.appId),
}))
: []
export const sortBy = derived([appsStore, auth], ([$store, $auth]) => {
return $store.sortBy || $auth.user?.appSort || "name"
})
if ($store.sortBy === "status") {
return enrichedApps.sort((a, b) => {
if (a.favourite === b.favourite) {
if (a.status === b.status) {
// Centralise any logic that enriches the apps list
export const enrichedApps = derived(
[appsStore, auth, sortBy],
([$store, $auth, $sortBy]) => {
const enrichedApps = $store.apps
? $store.apps.map(app => ({
...app,
deployed: app.status === AppStatus.DEPLOYED,
lockedYou: app.lockedBy && app.lockedBy.email === $auth.user?.email,
lockedOther: app.lockedBy && app.lockedBy.email !== $auth.user?.email,
favourite: $auth.user?.appFavourites?.includes(app.appId),
}))
: []
if ($sortBy === "status") {
return enrichedApps.sort((a, b) => {
if (a.favourite === b.favourite) {
if (a.status === b.status) {
return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
}
return a.status === AppStatus.DEPLOYED ? -1 : 1
}
return a.favourite ? -1 : 1
})
} else if ($sortBy === "updated") {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
const aUpdated = a.updatedAt || "9999"
const bUpdated = b.updatedAt || "9999"
return aUpdated < bUpdated ? 1 : -1
}
return a.favourite ? -1 : 1
})
} else {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
}
return a.status === AppStatus.DEPLOYED ? -1 : 1
}
return a.favourite ? -1 : 1
})
} else if ($store.sortBy === "updated") {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
const aUpdated = a.updatedAt || "9999"
const bUpdated = b.updatedAt || "9999"
return aUpdated < bUpdated ? 1 : -1
}
return a.favourite ? -1 : 1
})
} else {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
}
return a.favourite ? -1 : 1
})
return a.favourite ? -1 : 1
})
}
}
})
)

2
packages/builder/src/stores/portal/index.js
View File

@ -3,7 +3,7 @@ import { writable } from "svelte/store"
export { organisation } from "./organisation"
export { users } from "./users"
export { admin } from "./admin"
export { appsStore, enrichedApps } from "./apps"
export { appsStore, enrichedApps, sortBy } from "./apps"
export { email } from "./email"
export { auth } from "./auth"
export { oidc } from "./oidc"

3
packages/frontend-core/src/components/grid/cells/AICell.svelte
View File

@ -70,9 +70,6 @@
align-items: flex-start;
overflow: hidden;
}
.long-form-cell.editable:hover {
cursor: text;
}
.value {
display: -webkit-box;
-webkit-line-clamp: var(--content-lines);

18
packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
View File

@ -168,15 +168,15 @@
>
Delete row
</MenuItem>
<MenuItem
icon="MagicWand"
disabled={isNewRow ||
!$focusedRow?._rev ||
!$hasBudibaseIdentifiers}
on:click={generateAIColumns}
>
Generate AI Columns
</MenuItem>
<MenuItem
icon="MagicWand"
disabled={isNewRow ||
!$focusedRow?._rev ||
!$hasBudibaseIdentifiers}
on:click={generateAIColumns}
>
Generate AI Columns
</MenuItem>
{/if}
</Menu>
</GridPopover>

17
packages/frontend-core/src/components/grid/stores/datasource.js
View File

@ -1,6 +1,7 @@
import { derived, get } from "svelte/store"
import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
import { enrichSchemaWithRelColumns, memo } from "../../../utils"
import { cloneDeep } from "lodash"
export const createStores = () => {
const definition = memo(null)
@ -164,10 +165,18 @@ export const createActions = context => {
// Updates the datasources primary display column
const changePrimaryDisplay = async column => {
return await saveDefinition({
...get(definition),
primaryDisplay: column,
})
let newDefinition = cloneDeep(get(definition))
// Update primary display
newDefinition.primaryDisplay = column
// Sanitise schema to ensure field is required and has no default value
if (!newDefinition.schema[column].constraints) {
newDefinition.schema[column].constraints = {}
}
newDefinition.schema[column].constraints.presence = { allowEmpty: false }
delete newDefinition.schema[column].default
return await saveDefinition(newDefinition)
}
// Adds a schema mutation for a single field

14
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -1,4 +1,14 @@
import { get } from "svelte/store"
import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
}
return view
}
const SuppressErrors = true
@ -6,7 +16,7 @@ export const createActions = context => {
const { API, datasource, columns } = context
const saveDefinition = async newDefinition => {
await API.viewV2.update(newDefinition)
await API.viewV2.update(convertToSearchFilters(newDefinition))
}
const saveRow = async row => {
@ -125,7 +135,7 @@ export const initialise = context => {
}
// Only override filter state if we don't have an initial filter
if (!get(initialFilter)) {
filter.set($definition.query)
filter.set($definition.queryUI || $definition.query)
}
})
)

2
packages/pro

@ -1 +1 @@
Subproject commit 0c8f834932670ce1d3b3f639e6e8d0d0086e508d
Subproject commit 1bce30827813e135ee414141fdf51e45236b8f3c

3
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -697,9 +697,8 @@ export class ExternalRequest<T extends Operation> {
const calculationFields = helpers.views.calculationFields(this.source)
for (const [key, field] of Object.entries(calculationFields)) {
aggregations.push({
...field,
name: key,
field: field.field,
calculationType: field.calculationType,
})
}
}

4
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -144,7 +144,9 @@ export async function finaliseRow(
dynamic: false,
contextRows: [enrichedRow],
})
const aiEnabled = await pro.features.isBudibaseAIEnabled() || await pro.features.isAICustomConfigsEnabled()
const aiEnabled =
(await pro.features.isBudibaseAIEnabled()) ||
(await pro.features.isAICustomConfigsEnabled())
if (aiEnabled) {
row = await processAIColumns(table, row, {
contextRows: [enrichedRow],

6
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -125,9 +125,11 @@ export async function buildSqlFieldList(
column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI &&
!existing.find((field: string) => field === columnName)
!existing.find(
(field: string) => field === `${table.name}.${columnName}`
)
)
.map(column => `${table.name}.${column[0]}`)
.map(([columnName]) => `${table.name}.${columnName}`)
}
let fields: string[] = []

38
packages/server/src/api/controllers/row/views.ts
View File

@ -3,6 +3,8 @@ import {
ViewV2,
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
RowSearchParams,
} from "@budibase/types"
import sdk from "../../../sdk"
import { context } from "@budibase/backend-core"
@ -20,30 +22,34 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`)
}
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request
await context.ensureSnippetContext(true)
const result = await sdk.rows.search(
{
viewId: view.id,
tableId: view.tableId,
query: body.query,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
countRows: body.countRows,
},
{
user: sdk.users.getUserContextBindings(ctx.user),
}
)
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<
Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
> = {
tableId: view.tableId,
viewId: view.id,
query: body.query,
fields: viewFields,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
countRows: body.countRows,
}
const result = await sdk.rows.search(searchOptions, {
user: sdk.users.getUserContextBindings(ctx.user),
})
result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result
}
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) {
return {

87
packages/server/src/api/controllers/rowAction/crud.ts
View File

@ -1,6 +1,7 @@
import {
CreateRowActionRequest,
Ctx,
RowActionPermissions,
RowActionResponse,
RowActionsResponse,
UpdateRowActionRequest,
@ -18,25 +19,26 @@ async function getTable(ctx: Ctx) {
export async function find(ctx: Ctx<void, RowActionsResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
if (!(await sdk.rowActions.docExists(table._id!))) {
if (!(await sdk.rowActions.docExists(tableId))) {
ctx.body = {
actions: {},
}
return
}
const { actions } = await sdk.rowActions.getAll(table._id!)
const { actions } = await sdk.rowActions.getAll(tableId)
const result: RowActionsResponse = {
actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
(acc, [key, action]) => ({
...acc,
[key]: {
id: key,
tableId: table._id!,
tableId,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
},
}),
{}
@ -49,17 +51,18 @@ export async function create(
ctx: Ctx<CreateRowActionRequest, RowActionResponse>
) {
const table = await getTable(ctx)
const tableId = table._id!
const createdAction = await sdk.rowActions.create(table._id!, {
const createdAction = await sdk.rowActions.create(tableId, {
name: ctx.request.body.name,
})
ctx.body = {
tableId: table._id!,
tableId,
id: createdAction.id,
name: createdAction.name,
automationId: createdAction.automationId,
allowedViews: undefined,
allowedSources: flattenAllowedSources(tableId, createdAction.permissions),
}
ctx.status = 201
}
@ -68,18 +71,19 @@ export async function update(
ctx: Ctx<UpdateRowActionRequest, RowActionResponse>
) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.update(table._id!, actionId, {
const action = await sdk.rowActions.update(tableId, actionId, {
name: ctx.request.body.name,
})
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: undefined,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
@ -91,52 +95,89 @@ export async function remove(ctx: Ctx<void, void>) {
ctx.status = 204
}
export async function setTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.setTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.unsetTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.setViewPermission(
table._id!,
tableId,
actionId,
viewId
)
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.unsetViewPermission(
table._id!,
tableId,
actionId,
viewId
)
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
function flattenAllowedViews(
permissions: Record<string, { runAllowed: boolean }>
function flattenAllowedSources(
tableId: string,
permissions: RowActionPermissions
) {
const allowedPermissions = Object.entries(permissions || {})
.filter(([_, p]) => p.runAllowed)
.map(([viewId]) => viewId)
if (!allowedPermissions.length) {
return undefined
const allowedPermissions = []
if (permissions.table.runAllowed) {
allowedPermissions.push(tableId)
}
allowedPermissions.push(
...Object.keys(permissions.views || {}).filter(
viewId => permissions.views[viewId].runAllowed
)
)
return allowedPermissions
}

11
packages/server/src/api/controllers/script.ts
View File

@ -1,11 +1,18 @@
import { Ctx } from "@budibase/types"
import { IsolatedVM } from "../../jsRunner/vm"
import { iifeWrapper } from "@budibase/string-templates"
import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body
const vm = new IsolatedVM()
ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
try {
ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
} catch (err: any) {
if (err.code === UserScriptError.code) {
throw err.userScriptError
}
throw err
}
}
export async function save(ctx: Ctx) {

2
packages/server/src/api/controllers/table/utils.ts
View File

@ -33,7 +33,7 @@ import {
} from "@budibase/types"
import sdk from "../../../sdk"
import env from "../../../environment"
import { runAIColumnChecks, runStaticFormulaChecks } from "./bulkFormula"
import { runStaticFormulaChecks } from "./bulkFormula"
export async function clearColumns(table: Table, columnNames: string[]) {
const db = context.getAppDB()

34
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -11,14 +11,40 @@ import {
ViewCalculationFieldMetadata,
RelationSchemaField,
ViewFieldMetadata,
CalculationType,
} from "@budibase/types"
import { builderSocket, gridSocket } from "../../../websockets"
import { helpers } from "@budibase/shared-core"
function stripUnknownFields(
field: BasicViewFieldMetadata
): RequiredKeys<BasicViewFieldMetadata> {
field: ViewFieldMetadata
): RequiredKeys<ViewFieldMetadata> {
if (helpers.views.isCalculationField(field)) {
if (field.calculationType === CalculationType.COUNT) {
if ("distinct" in field && field.distinct) {
return {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
distinct: field.distinct,
calculationType: field.calculationType,
field: field.field,
columns: field.columns,
}
} else {
return {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
calculationType: field.calculationType,
columns: field.columns,
}
}
}
const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
order: field.order,
width: field.width,
@ -103,9 +129,11 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
name: view.name,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.create(tableId, parsedView)
ctx.status = 201
@ -138,9 +166,11 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
version: view.version,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.update(tableId, parsedView)

10
packages/server/src/api/routes/rowAction.ts
View File

@ -51,6 +51,16 @@ router
authorized(BUILDER),
rowActionController.remove
)
.post(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.setTablePermission
)
.delete(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.unsetTablePermission
)
.post(
"/api/tables/:tableId/actions/:actionId/permissions/:viewId",
authorized(BUILDER),

5
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -49,6 +49,7 @@ import * as uuid from "uuid"
import { Knex } from "knex"
import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
@ -3013,7 +3014,7 @@ describe.each([
let i = 0
for (; i < 10; i++) {
const row = rows[i]
if (row.formula !== "Timed out while executing JS") {
if (row.formula !== JsTimeoutError.message) {
break
}
}
@ -3027,7 +3028,7 @@ describe.each([
for (; i < 10; i++) {
const row = rows[i]
expect(row.text).toBe("foo")
expect(row.formula).toBe("Request JS execution limit hit")
expect(row.formula).toStartWith("CPU time limit exceeded ")
}
}
}

145
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -133,6 +133,7 @@ describe("/rowsActions", () => {
id: expect.stringMatching(/^row_action_\w+/),
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual({
@ -142,6 +143,7 @@ describe("/rowsActions", () => {
id: res.id,
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -180,18 +182,21 @@ describe("/rowsActions", () => {
id: responses[0].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
[responses[1].id]: {
name: rowActions[1].name,
id: responses[1].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
[responses[2].id]: {
name: rowActions[2].name,
id: responses[2].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -224,6 +229,7 @@ describe("/rowsActions", () => {
id: expect.any(String),
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual({
@ -233,6 +239,7 @@ describe("/rowsActions", () => {
id: res.id,
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -354,6 +361,7 @@ describe("/rowsActions", () => {
tableId,
name: updatedName,
automationId: actionData.automationId,
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual(
@ -364,6 +372,7 @@ describe("/rowsActions", () => {
id: actionData.id,
tableId: actionData.tableId,
automationId: actionData.automationId,
allowedSources: [tableId],
},
}),
})
@ -515,6 +524,81 @@ describe("/rowsActions", () => {
})
})
describe("set/unsetTablePermission", () => {
describe.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])("unauthorisedTests for %s", (__, delegateTest) => {
unauthorisedTests((expectations, testConfig) =>
delegateTest(tableId, generator.guid(), expectations, testConfig)
)
})
let actionId1: string, actionId2: string
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
const persisted = await config.api.rowAction.find(tableId)
const actions = _.sampleSize(Object.keys(persisted.actions), 2)
actionId1 = actions[0]
actionId2 = actions[1]
})
it("can set table permission", async () => {
await config.api.rowAction.unsetTablePermission(tableId, actionId1)
await config.api.rowAction.unsetTablePermission(tableId, actionId2)
const actionResult = await config.api.rowAction.setTablePermission(
tableId,
actionId1
)
const expectedAction1 = expect.objectContaining({
allowedSources: [tableId],
})
const expectedActions = expect.objectContaining({
[actionId1]: expectedAction1,
[actionId2]: expect.objectContaining({
allowedSources: [],
}),
})
expect(actionResult).toEqual(expectedAction1)
expect((await config.api.rowAction.find(tableId)).actions).toEqual(
expectedActions
)
})
it("can unset table permission", async () => {
const actionResult = await config.api.rowAction.unsetTablePermission(
tableId,
actionId1
)
const expectedAction = expect.objectContaining({
allowedSources: [],
})
expect(actionResult).toEqual(expectedAction)
expect(
(await config.api.rowAction.find(tableId)).actions[actionId1]
).toEqual(expectedAction)
})
it.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])(
"cannot update permission for unexisting tables (%s)",
async (__, delegateTest) => {
const tableId = generator.guid()
await delegateTest(tableId, actionId1, {
status: 404,
})
}
)
})
describe("set/unsetViewPermission", () => {
describe.each([
["setViewPermission", config.api.rowAction.setViewPermission],
@ -531,11 +615,9 @@ describe("/rowsActions", () => {
)
})
let tableIdForDescribe: string
let actionId1: string, actionId2: string
let viewId1: string, viewId2: string
beforeAll(async () => {
tableIdForDescribe = tableId
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
@ -557,11 +639,6 @@ describe("/rowsActions", () => {
).id
})
beforeEach(() => {
// Hack to reuse tables for these given tests
tableId = tableIdForDescribe
})
it("can set permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId1)
const action1Result = await config.api.rowAction.setViewPermission(
@ -576,10 +653,10 @@ describe("/rowsActions", () => {
)
const expectedAction1 = expect.objectContaining({
allowedViews: [viewId1, viewId2],
allowedSources: [tableId, viewId1, viewId2],
})
const expectedAction2 = expect.objectContaining({
allowedViews: [viewId1],
allowedSources: [tableId, viewId1],
})
const expectedActions = expect.objectContaining({
@ -594,6 +671,8 @@ describe("/rowsActions", () => {
})
it("can unset permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId2, actionId1)
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId2)
const actionResult = await config.api.rowAction.unsetViewPermission(
tableId,
viewId1,
@ -601,7 +680,7 @@ describe("/rowsActions", () => {
)
const expectedAction = expect.objectContaining({
allowedViews: [viewId2],
allowedSources: [tableId, viewId2],
})
expect(actionResult).toEqual(expectedAction)
expect(
@ -672,6 +751,7 @@ describe("/rowsActions", () => {
)
await config.publish()
// Travel time in order to "trim" the selected `getAutomationLogs`
tk.travel(Date.now() + 100)
})
@ -698,6 +778,8 @@ describe("/rowsActions", () => {
inputs: null,
outputs: {
fields: {},
id: rowId,
revision: (await config.api.row.get(tableId, rowId))._rev,
row: await config.api.row.get(tableId, rowId),
table: {
...(await config.api.table.get(tableId)),
@ -712,6 +794,38 @@ describe("/rowsActions", () => {
])
})
it("triggers from an allowed table", async () => {
expect(await getAutomationLogs()).toBeEmpty()
await config.api.rowAction.trigger(tableId, rowAction.id, { rowId })
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([
expect.objectContaining({
automationId: rowAction.automationId,
}),
])
})
it("rejects triggering from a non-allowed table", async () => {
await config.api.rowAction.unsetTablePermission(tableId, rowAction.id)
await config.publish()
await config.api.rowAction.trigger(
tableId,
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([])
})
it("rejects triggering from a non-allowed view", async () => {
const viewId = (
await config.api.viewV2.create(
@ -901,7 +1015,7 @@ describe("/rowsActions", () => {
})
it.each(allowedRoleConfig)(
"does not allow running row actions for tables by default even",
"allow running row actions for tables by default",
async (userRole, resourcePermission) => {
await config.api.permission.add({
level: PermissionLevel.READ,
@ -918,15 +1032,12 @@ describe("/rowsActions", () => {
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
status: 200,
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toBeEmpty()
expect(automationLogs).toHaveLength(1)
})
}
)

5479
packages/server/src/api/routes/tests/search.spec.ts
View File

File diff suppressed because it is too large Load Diff

337
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,9 +22,10 @@ import {
RelationshipType,
TableSchema,
RenameColumn,
ViewFieldMetadata,
FeatureFlag,
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -154,7 +155,7 @@ describe.each([
})
it("can persist views with all fields", async () => {
const newView: Required<CreateViewRequest> = {
const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
name: generator.name(),
tableId: table._id!,
primaryDisplay: "id",
@ -176,6 +177,9 @@ describe.each([
visible: true,
},
},
uiMetadata: {
foo: "bar",
},
}
const res = await config.api.viewV2.create(newView)
@ -540,6 +544,33 @@ describe.each([
status: 201,
})
})
it("can create a view with calculation fields", async () => {
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
})
expect(Object.keys(view.schema!)).toHaveLength(1)
let sum = view.schema!.sum as NumericCalculationFieldMetadata
expect(sum).toBeDefined()
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
view = await config.api.viewV2.get(view.id)
sum = view.schema!.sum as NumericCalculationFieldMetadata
expect(sum).toBeDefined()
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
})
})
describe("update", () => {
@ -584,7 +615,7 @@ describe.each([
it("can update all fields", async () => {
const tableId = table._id!
const updatedData: Required<UpdateViewRequest> = {
const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
version: view.version,
id: view.id,
tableId,
@ -612,6 +643,9 @@ describe.each([
readonly: true,
},
},
uiMetadata: {
foo: "bar",
},
}
await config.api.viewV2.update(updatedData)
@ -836,6 +870,185 @@ describe.each([
}
)
})
!isLucene &&
describe("calculation views", () => {
let table: Table
let view: ViewV2
beforeEach(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
country: {
name: "country",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
country: {
visible: true,
},
age: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Steve",
age: 30,
country: "UK",
},
{
name: "Jane",
age: 31,
country: "UK",
},
{
name: "Ruari",
age: 32,
country: "USA",
},
{
name: "Alice",
age: 33,
country: "USA",
},
],
})
})
it("returns the expected rows prior to modification", async () => {
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
},
{
country: "UK",
age: 61,
},
])
)
})
it("can remove a group by field", async () => {
delete view.schema!.country
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows).toEqual(
expect.arrayContaining([
{
age: 126,
},
])
)
})
it("can remove a calculation field", async () => {
delete view.schema!.age
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(4)
// Because the removal of the calculation field actually makes this
// no longer a calculation view, these rows will now have _id and
// _rev fields.
expect(rows).toEqual(
expect.arrayContaining([
expect.objectContaining({ country: "UK" }),
expect.objectContaining({ country: "UK" }),
expect.objectContaining({ country: "USA" }),
expect.objectContaining({ country: "USA" }),
])
)
})
it("can add a new group by field", async () => {
view.schema!.name = { visible: true }
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(4)
expect(rows).toEqual(
expect.arrayContaining([
{
name: "Steve",
age: 30,
country: "UK",
},
{
name: "Jane",
age: 31,
country: "UK",
},
{
name: "Ruari",
age: 32,
country: "USA",
},
{
name: "Alice",
age: 33,
country: "USA",
},
])
)
})
it("can add a new calculation field", async () => {
view.schema!.count = {
visible: true,
calculationType: CalculationType.COUNT,
}
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
count: 2,
},
{
country: "UK",
age: 61,
count: 2,
},
])
)
})
})
})
describe("delete", () => {
@ -1152,10 +1365,7 @@ describe.each([
return table
}
const createView = async (
tableId: string,
schema: Record<string, ViewFieldMetadata>
) =>
const createView = async (tableId: string, schema: ViewV2Schema) =>
await config.api.viewV2.create({
name: generator.guid(),
tableId,
@ -2545,6 +2755,119 @@ describe.each([
expect(actual).toEqual(expected)
}
})
it("should be able to do a COUNT(DISTINCT)", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "name",
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "John",
},
{
name: "John",
},
{
name: "Sue",
},
],
})
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows[0].count).toEqual(2)
})
it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "does not exist oh no",
},
},
},
{
status: 400,
body: {
message:
'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema',
},
}
)
})
})
!isLucene &&
it("should not need required fields to be present", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
await Promise.all([
config.api.row.save(table._id!, { name: "Steve", age: 30 }),
config.api.row.save(table._id!, { name: "Jane", age: 31 }),
])
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].sum).toEqual(61)
})
})

4
packages/server/src/db/linkedRows/index.ts
View File

@ -23,8 +23,8 @@ import {
Row,
Table,
TableSchema,
ViewFieldMetadata,
ViewV2,
ViewV2Schema,
} from "@budibase/types"
import sdk from "../../sdk"
import { helpers } from "@budibase/shared-core"
@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
FeatureFlag.ENRICHED_RELATIONSHIPS
)
let viewSchema: Record<string, ViewFieldMetadata> = {}
let viewSchema: ViewV2Schema = {}
if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) {
return enriched

4
packages/server/src/jsRunner/index.ts
View File

@ -1,7 +1,7 @@
import { serializeError } from "serialize-error"
import env from "../environment"
import {
JsErrorTimeout,
JsTimeoutError,
setJSRunner,
setOnErrorLog,
} from "@budibase/string-templates"
@ -40,7 +40,7 @@ export function init() {
return vm.withContext(rest, () => vm.execute(js))
} catch (error: any) {
if (error.message === "Script execution timed out.") {
throw new JsErrorTimeout()
throw new JsTimeoutError()
}
throw error
}

9
packages/server/src/jsRunner/tests/jsRunner.spec.ts
View File

@ -41,10 +41,11 @@ describe("jsRunner (using isolated-vm)", () => {
})
it("should prevent sandbox escape", async () => {
const output = await processJS(
`return this.constructor.constructor("return process.env")()`
)
expect(output).toBe("Error while executing JS")
expect(
await processJS(
`return this.constructor.constructor("return process.env")()`
)
).toEqual("ReferenceError: process is not defined")
})
describe("helpers", () => {

25
packages/server/src/jsRunner/vm/isolated-vm.ts
View File

@ -7,14 +7,12 @@ import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles"
import { Snippet, VM } from "@budibase/types"
import { iifeWrapper } from "@budibase/string-templates"
import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
import environment from "../../environment"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
export class JsRequestTimeoutError extends Error {
static code = "JS_REQUEST_TIMEOUT_ERROR"
code = JsRequestTimeoutError.code
}
export class IsolatedVM implements VM {
@ -29,6 +27,7 @@ export class IsolatedVM implements VM {
private readonly resultKey = "results"
private runResultKey: string
private runErrorKey: string
constructor({
memoryLimit,
@ -47,6 +46,7 @@ export class IsolatedVM implements VM {
this.jail.setSync("global", this.jail.derefInto())
this.runResultKey = crypto.randomUUID()
this.runErrorKey = crypto.randomUUID()
this.addToContext({
[this.resultKey]: { [this.runResultKey]: "" },
})
@ -210,13 +210,19 @@ export class IsolatedVM implements VM {
if (this.isolateAccumulatedTimeout) {
const cpuMs = Number(this.isolate.cpuTime) / 1e6
if (cpuMs > this.isolateAccumulatedTimeout) {
throw new ExecutionTimeoutError(
throw new JsRequestTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
)
}
}
code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
code = `
try {
results['${this.runResultKey}']=${this.codeWrapper(code)}
} catch (e) {
results['${this.runErrorKey}']=e
}
`
const script = this.isolate.compileScriptSync(code)
@ -227,6 +233,9 @@ export class IsolatedVM implements VM {
// We can't rely on the script run result as it will not work for non-transferable values
const result = this.getFromContext(this.resultKey)
if (result[this.runErrorKey]) {
throw new UserScriptError(result[this.runErrorKey])
}
return result[this.runResultKey]
}

21
packages/server/src/sdk/app/rowActions.ts
View File

@ -75,7 +75,7 @@ export async function create(tableId: string, rowAction: { name: string }) {
name: action.name,
automationId: automation._id!,
permissions: {
table: { runAllowed: false },
table: { runAllowed: true },
views: {},
},
}
@ -163,6 +163,23 @@ async function guardView(tableId: string, viewId: string) {
}
}
export async function setTablePermission(tableId: string, rowActionId: string) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = true
return actionsDoc
})
}
export async function unsetTablePermission(
tableId: string,
rowActionId: string
) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = false
return actionsDoc
})
}
export async function setViewPermission(
tableId: string,
rowActionId: string,
@ -220,6 +237,8 @@ export async function run(tableId: any, rowActionId: any, rowId: string) {
automation,
{
fields: {
id: row._id,
revision: row._rev,
row,
table,
},

50
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,12 +1,10 @@
import {
EmptyFilterOption,
LegacyFilter,
LogicalOperator,
Row,
RowSearchParams,
SearchFilter,
SearchFilterGroup,
SearchFilterKey,
SearchFilters,
SearchResponse,
SortOrder,
Table,
@ -63,7 +61,6 @@ export async function search(
if (options.viewId) {
source = await sdk.views.get(options.viewId)
table = await sdk.views.getTable(source)
options = searchInputMapping(table, options)
} else if (options.tableId) {
source = await sdk.tables.getTable(options.tableId)
table = source
@ -76,7 +73,7 @@ export async function search(
if (options.query) {
const visibleFields = (
options.fields || Object.keys(table.schema)
).filter(field => table.schema[field].visible !== false)
).filter(field => table.schema[field]?.visible !== false)
const queryableFields = await getQueryableFields(table, visibleFields)
options.query = removeInvalidFilters(options.query, queryableFields)
@ -84,40 +81,59 @@ export async function search(
options.query = {}
}
// need to make sure filters in correct shape before checking for view
options = searchInputMapping(table, options)
if (options.viewId) {
const view = await sdk.views.get(options.viewId)
// Delete extraneous search params that cannot be overridden
delete options.query.onEmptyFilter
const view = source as ViewV2
// Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access.
let viewQuery = dataFilters.buildQuery(view.query || [])
let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
delete viewQuery?.onEmptyFilter
if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
// Lucene does not accept conditional filters, so we need to keep the old logic
const query: SearchFilters = viewQuery || {}
const viewFilters = view.query as SearchFilter[]
const sqsEnabled = await features.flags.isEnabled("SQS")
const supportsLogicalOperators =
isExternalTableID(view.tableId) || sqsEnabled
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
delete options.query.onEmptyFilter
// Extract existing fields
const existingFields =
viewFilters
queryFilters
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
viewQuery ??= {}
// Carry over filters for unused fields
Object.keys(options.query || {}).forEach(key => {
Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = options.query[operator]![field]
viewQuery![operator]![field] = options.query[operator]![field]
}
})
})
options.query = query
options.query = viewQuery
} else {
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [viewQuery as SearchFilterGroup, options.query],
conditions: [...conditions, options.query],
},
}
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
}
}
@ -146,8 +162,6 @@ export async function search(
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
options = searchInputMapping(table, options)
let result: SearchResponse<Row>
if (isExternalTable) {
span?.addTags({ searchType: "external" })

27
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -1,5 +1,6 @@
import {
Aggregation,
CalculationType,
Datasource,
DocumentType,
FieldType,
@ -369,11 +370,27 @@ export async function search(
continue
}
aggregations.push({
name: key,
field: mapToUserColumn(field.field),
calculationType: field.calculationType,
})
if (field.calculationType === CalculationType.COUNT) {
if ("distinct" in field && field.distinct) {
aggregations.push({
name: key,
distinct: true,
field: mapToUserColumn(field.field),
calculationType: field.calculationType,
})
} else {
aggregations.push({
name: key,
calculationType: field.calculationType,
})
}
} else {
aggregations.push({
name: key,
field: mapToUserColumn(field.field),
calculationType: field.calculationType,
})
}
}
}

4
packages/server/src/sdk/app/rows/search/utils.ts
View File

@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
}
return dataFilters.recurseLogicalOperators(filters, checkFilters)
}
options.query = checkFilters(options.query)
if (options.query) {
options.query = checkFilters(options.query)
}
return options
}

43
packages/server/src/sdk/app/views/index.ts
View File

@ -1,4 +1,5 @@
import {
CalculationType,
FieldType,
PermissionLevel,
RelationSchemaField,
@ -63,6 +64,15 @@ async function guardCalculationViewSchema(
const calculationFields = helpers.views.calculationFields(view)
for (const calculationFieldName of Object.keys(calculationFields)) {
const schema = calculationFields[calculationFieldName]
const isCount = schema.calculationType === CalculationType.COUNT
const isDistinct = isCount && "distinct" in schema && schema.distinct
// Count fields that aren't distinct don't need to reference another field,
// so we don't validate it.
if (isCount && !isDistinct) {
continue
}
const targetSchema = table.schema[schema.field]
if (!targetSchema) {
throw new HTTPError(
@ -71,7 +81,7 @@ async function guardCalculationViewSchema(
)
}
if (!helpers.schema.isNumeric(targetSchema)) {
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
400
@ -255,19 +265,12 @@ export async function enrichSchema(
view: ViewV2,
tableSchema: TableSchema
): Promise<ViewV2Enriched> {
const tableCache: Record<string, Table> = {}
async function populateRelTableSchema(
tableId: string,
viewFields: Record<string, RelationSchemaField>
) {
if (!tableCache[tableId]) {
tableCache[tableId] = await sdk.tables.getTable(tableId)
}
const relTable = tableCache[tableId]
const relTable = await sdk.tables.getTable(tableId)
const result: Record<string, ViewV2ColumnEnriched> = {}
for (const relTableFieldName of Object.keys(relTable.schema)) {
const relTableField = relTable.schema[relTableFieldName]
if (
@ -300,15 +303,24 @@ export async function enrichSchema(
const viewSchema = view.schema || {}
const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null)
for (const key of Object.keys(tableSchema).filter(
k => tableSchema[k].visible !== false
)) {
const visibleSchemaFields = Object.keys(viewSchema).filter(key => {
if (helpers.views.isCalculationField(viewSchema[key])) {
return viewSchema[key].visible !== false
}
return key in tableSchema && tableSchema[key].visible !== false
})
const visibleTableFields = Object.keys(tableSchema).filter(
key => tableSchema[key].visible !== false
)
const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields])
for (const key of visibleFields) {
// if nothing specified in view, then it is not visible
const ui = viewSchema[key] || { visible: false }
schema[key] = {
...tableSchema[key],
...ui,
order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order,
order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
columns: undefined,
}
@ -320,10 +332,7 @@ export async function enrichSchema(
}
}
return {
...view,
schema: schema,
}
return { ...view, schema }
}
export function syncSchema(

4
packages/server/src/tests/utilities/api/row.ts
View File

@ -7,11 +7,11 @@ import {
BulkImportRequest,
BulkImportResponse,
SearchRowResponse,
RowSearchParams,
DeleteRows,
DeleteRow,
PaginatedSearchRowResponse,
RowExportFormat,
SearchRowRequest,
} from "@budibase/types"
import { Expectations, TestAPI } from "./base"
@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
)
}
search = async <T extends RowSearchParams>(
search = async <T extends SearchRowRequest>(
sourceId: string,
params?: T,
expectations?: Expectations

36
packages/server/src/tests/utilities/api/rowAction.ts
View File

@ -72,6 +72,42 @@ export class RowActionAPI extends TestAPI {
)
}
setTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._post<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
unsetTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._delete<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
setViewPermission = async (
tableId: string,
viewId: string,

21
packages/shared-core/src/filters.ts
View File

@ -3,7 +3,7 @@ import {
BBReferenceFieldSubType,
FieldType,
FormulaType,
SearchFilter,
LegacyFilter,
SearchFilters,
SearchQueryFields,
ArrayOperator,
@ -129,7 +129,7 @@ export function recurseLogicalOperators(
fn: (f: SearchFilters) => SearchFilters
) {
for (const logical of LOGICAL_OPERATORS) {
if (filters?.[logical]) {
if (filters[logical]) {
filters[logical]!.conditions = filters[logical]!.conditions.map(
condition => fn(condition)
)
@ -165,9 +165,6 @@ export function recurseSearchFilters(
* https://github.com/Budibase/budibase/issues/10118
*/
export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) {
if (!query[filterField]) {
continue
@ -313,7 +310,7 @@ export class ColumnSplitter {
* @param filter the builder filter structure
*/
const buildCondition = (expression: SearchFilter) => {
const buildCondition = (expression: LegacyFilter) => {
// Filter body
let query: SearchFilters = {
string: {},
@ -439,8 +436,13 @@ const buildCondition = (expression: SearchFilter) => {
}
export const buildQueryLegacy = (
filter?: SearchFilterGroup | SearchFilter[]
filter?: LegacyFilter[] | SearchFilters
): SearchFilters | undefined => {
// this is of type SearchFilters or is undefined
if (!Array.isArray(filter)) {
return filter
}
let query: SearchFilters = {
string: {},
fuzzy: {},
@ -574,7 +576,7 @@ export const buildQueryLegacy = (
*/
export const buildQuery = (
filter?: SearchFilterGroup | SearchFilter[]
filter?: SearchFilterGroup | LegacyFilter[]
): SearchFilters | undefined => {
const parsedFilter: SearchFilterGroup | undefined =
processSearchFilters(filter)
@ -596,7 +598,7 @@ export const buildQuery = (
const globalOperator: LogicalOperator =
operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
const coreRequest: SearchFilters = {
return {
...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
[globalOperator]: {
conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@ -610,7 +612,6 @@ export const buildQuery = (
}),
},
}
return coreRequest
}
// The frontend can send single values for array fields sometimes, so to handle

201
packages/shared-core/src/utils.ts
View File

@ -1,5 +1,5 @@
import {
SearchFilter,
LegacyFilter,
SearchFilterGroup,
FilterGroupLogicalOperator,
SearchFilters,
@ -9,6 +9,10 @@ import {
import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
// an array of keys from filter type to properties that are in the type
// this can then be converted using .fromEntries to an object
type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
export function unreachable(
value: never,
message = `No such case in exhaustive switch: ${value}`
@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
return result
}
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {SearchFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: SearchFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterWhitelistKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
const migratedSetting: SearchFilterGroup = filters.reduce(
(acc: SearchFilterGroup, filter: SearchFilter) => {
// Sort the properties for easier debugging
const filterEntries = Object.entries(filter)
.sort((a, b) => {
return a[0].localeCompare(b[0])
})
.filter(x => x[1] ?? false)
if (filterEntries.length == 1) {
const [key, value] = filterEntries[0]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
(acc: [string, any][], entry: [string, any]) => {
const [key, value] = entry
if (filterWhitelistKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: SearchFilter = Object.fromEntries(
whiteListedFilterSettings
) as SearchFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
},
defaultCfg
)
return migratedSetting
} else if (!filters?.groups) {
return
}
return filters
}
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
{ op: BasicOperator.STRING, key: "email" },
@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
}
return true
}
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {LegacyFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: LegacyFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterAllowedKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
// Sort the properties for easier debugging
const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
.sort((a, b) => {
return a.localeCompare(b)
})
.filter(key => key in filter)
if (filterPropertyKeys.length == 1) {
const key = filterPropertyKeys[0],
value = filter[key]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
(acc: AllowedFilters, key) => {
const value = filter[key]
if (filterAllowedKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: LegacyFilter = Object.fromEntries(
allowedFilterSettings
) as LegacyFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
}, defaultCfg)
} else if (!filters?.groups) {
return
}
return filters
}

21
packages/string-templates/src/errors.ts
View File

@ -1,3 +1,20 @@
export class JsErrorTimeout extends Error {
code = "ERR_SCRIPT_EXECUTION_TIMEOUT"
export class JsTimeoutError extends Error {
static message = "Timed out while executing JS"
static code = "JS_TIMEOUT_ERROR"
code: string = JsTimeoutError.code
constructor() {
super(JsTimeoutError.message)
}
}
export class UserScriptError extends Error {
static code = "USER_SCRIPT_ERROR"
code: string = UserScriptError.code
constructor(readonly userScriptError: Error) {
super(
`error while running user-supplied JavaScript: ${userScriptError.toString()}`
)
}
}

42
packages/string-templates/src/helpers/javascript.ts
View File

@ -3,6 +3,7 @@ import cloneDeep from "lodash/fp/cloneDeep"
import { LITERAL_MARKER } from "../helpers/constants"
import { getJsHelperList } from "./list"
import { iifeWrapper } from "../iife"
import { JsTimeoutError, UserScriptError } from "../errors"
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.js or index.mjs).
@ -94,12 +95,49 @@ export function processJS(handlebars: string, context: any) {
} catch (error: any) {
onErrorLog && onErrorLog(error)
const { noThrow = true } = context.__opts || {}
// The error handling below is quite messy, because it has fallen to
// string-templates to handle a variety of types of error specific to usages
// above it in the stack. It would be nice some day to refactor this to
// allow each user of processStringSync to handle errors in the way they see
// fit.
// This is to catch the error vm.runInNewContext() throws when the timeout
// is exceeded.
if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
return "Timed out while executing JS"
}
if (error.name === "ExecutionTimeoutError") {
return "Request JS execution limit hit"
// This is to catch the JsRequestTimeoutError we throw when we detect a
// timeout across an entire request in the backend. We use a magic string
// because we can't import from the backend into string-templates.
if (error.code === "JS_REQUEST_TIMEOUT_ERROR") {
return error.message
}
// This is to catch the JsTimeoutError we throw when we detect a timeout in
// a single JS execution.
if (error.code === JsTimeoutError.code) {
return JsTimeoutError.message
}
// This is to catch the error that happens if a user-supplied JS script
// throws for reasons introduced by the user.
if (error.code === UserScriptError.code) {
if (noThrow) {
return error.userScriptError.toString()
}
throw error
}
if (error.name === "SyntaxError") {
if (noThrow) {
return error.toString()
}
throw error
}
return "Error while executing JS"
}
}

37
packages/string-templates/src/index.ts
View File

@ -16,6 +16,7 @@ import { removeJSRunner, setJSRunner } from "./helpers/javascript"
import manifest from "./manifest.json"
import { ProcessOptions } from "./types"
import { UserScriptError } from "./errors"
export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list"
export { FIND_ANY_HBS_REGEX } from "./utilities"
@ -122,7 +123,7 @@ function createTemplate(
export async function processObject<T extends Record<string, any>>(
object: T,
context: object,
opts?: { noHelpers?: boolean; escapeNewlines?: boolean; onlyFound?: boolean }
opts?: ProcessOptions
): Promise<T> {
testObject(object)
@ -172,7 +173,7 @@ export async function processString(
export function processObjectSync(
object: { [x: string]: any },
context: any,
opts: any
opts?: ProcessOptions
): object | Array<any> {
testObject(object)
for (let key of Object.keys(object || {})) {
@ -229,8 +230,12 @@ export function processStringSync(
} else {
return process(string)
}
} catch (err) {
return input
} catch (err: any) {
const { noThrow = true } = opts || {}
if (noThrow) {
return input
}
throw err
}
}
@ -448,7 +453,7 @@ export function convertToJS(hbs: string) {
return `${varBlock}${js}`
}
export { JsErrorTimeout } from "./errors"
export { JsTimeoutError, UserScriptError } from "./errors"
export function defaultJSSetup() {
if (!isBackendService()) {
@ -463,7 +468,27 @@ export function defaultJSSetup() {
setTimeout: undefined,
}
createContext(context)
return runInNewContext(js, context, { timeout: 1000 })
const wrappedJs = `
result = {
result: null,
error: null,
};
try {
result.result = ${js};
} catch (e) {
result.error = e;
}
result;
`
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
} else {
removeJSRunner()

1
packages/string-templates/src/types.ts
View File

@ -3,6 +3,7 @@ export interface ProcessOptions {
noEscaping?: boolean
noHelpers?: boolean
noFinalise?: boolean
noThrow?: boolean
escapeNewlines?: boolean
onlyFound?: boolean
disabledHelpers?: string[]

2
packages/types/src/api/web/app/rowAction.ts
View File

@ -8,7 +8,7 @@ export interface RowActionResponse extends RowActionData {
id: string
tableId: string
automationId: string
allowedViews: string[] | undefined
allowedSources: string[] | undefined
}
export interface RowActionsResponse {

1
packages/types/src/api/web/auth.ts
View File

@ -21,6 +21,7 @@ export interface UpdateSelfRequest {
freeTrialConfirmedAt?: string
appFavourites?: string[]
tours?: Record<string, Date>
appSort?: string
}
export interface UpdateSelfResponse {

5
packages/types/src/api/web/searchFilter.ts
View File

@ -5,7 +5,7 @@ import {
SearchFilters,
} from "../../sdk"
export type SearchFilter = {
export type LegacyFilter = {
operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
onEmptyFilter?: EmptyFilterOption
field: string
@ -14,9 +14,10 @@ export type SearchFilter = {
externalType?: string
}
// this is a type purely used by the UI
export type SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator
onEmptyFilter?: EmptyFilterOption
groups?: SearchFilterGroup[]
filters?: SearchFilter[]
filters?: LegacyFilter[]
}

10
packages/types/src/documents/app/rowAction.ts
View File

@ -8,8 +8,10 @@ export interface TableRowActions extends Document {
export interface RowActionData {
name: string
automationId: string
permissions: {
table: { runAllowed: boolean }
views: Record<string, { runAllowed: boolean }>
}
permissions: RowActionPermissions
}
export interface RowActionPermissions {
table: { runAllowed: boolean }
views: Record<string, { runAllowed: boolean }>
}

37
packages/types/src/documents/app/view.ts
View File

@ -1,7 +1,7 @@
import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table"
import { Document } from "../document"
import { DBView } from "../../sdk"
import { DBView, SearchFilters } from "../../sdk"
export type ViewTemplateOpts = {
field: string
@ -42,11 +42,31 @@ export interface RelationSchemaField extends UIFieldMetadata {
readonly?: boolean
}
export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata {
calculationType: CalculationType
export interface NumericCalculationFieldMetadata
extends BasicViewFieldMetadata {
calculationType:
| CalculationType.MIN
| CalculationType.MAX
| CalculationType.SUM
| CalculationType.AVG
field: string
}
export interface CountCalculationFieldMetadata extends BasicViewFieldMetadata {
calculationType: CalculationType.COUNT
}
export interface CountDistinctCalculationFieldMetadata
extends CountCalculationFieldMetadata {
distinct: true
field: string
}
export type ViewCalculationFieldMetadata =
| NumericCalculationFieldMetadata
| CountCalculationFieldMetadata
| CountDistinctCalculationFieldMetadata
export type ViewFieldMetadata =
| BasicViewFieldMetadata
| ViewCalculationFieldMetadata
@ -65,15 +85,20 @@ export interface ViewV2 {
name: string
primaryDisplay?: string
tableId: string
query?: SearchFilter[] | SearchFilterGroup
query?: LegacyFilter[] | SearchFilters
// duplicate to store UI information about filters
queryUI?: SearchFilterGroup
sort?: {
field: string
order?: SortOrder
type?: SortType
}
schema?: Record<string, ViewFieldMetadata>
schema?: ViewV2Schema
uiMetadata?: Record<string, any>
}
export type ViewV2Schema = Record<string, ViewFieldMetadata>
export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
export interface ViewCountOrSumSchema {

1
packages/types/src/documents/global/user.ts
View File

@ -67,6 +67,7 @@ export interface User extends Document {
scimInfo?: { isSync: true } & Record<string, any>
appFavourites?: string[]
ssoId?: string
appSort?: string
}
export enum UserStatus {

2
packages/types/src/sdk/ai.ts
View File

@ -63,7 +63,7 @@ type TranslateSchema = BaseSchema & {
type CategoriseTextSchema = BaseSchema & {
operation: AIOperationEnum.CATEGORISE_TEXT
columns: string[]
categories: string
categories: string[]
}
type SentimentAnalysisSchema = BaseSchema & {

25
packages/types/src/sdk/row.ts
View File

@ -3,12 +3,33 @@ import { SearchFilters } from "./search"
import { CalculationType, Row } from "../documents"
import { WithRequired } from "../shared"
export interface Aggregation {
export interface BaseAggregation {
name: string
calculationType: CalculationType
}
export interface NumericAggregation extends BaseAggregation {
calculationType:
| CalculationType.AVG
| CalculationType.MAX
| CalculationType.MIN
| CalculationType.SUM
field: string
}
export interface CountAggregation extends BaseAggregation {
calculationType: CalculationType.COUNT
}
export interface CountDistinctAggregation extends CountAggregation {
distinct: true
field: string
}
export type Aggregation =
| NumericAggregation
| CountAggregation
| CountDistinctAggregation
export interface SearchParams {
tableId?: string
viewId?: string

1
packages/worker/src/api/routes/validation/users.ts
View File

@ -29,6 +29,7 @@ export const buildSelfSaveValidation = () => {
freeTrialConfirmedAt: Joi.string().optional(),
appFavourites: Joi.array().optional(),
tours: Joi.object().optional(),
appSort: Joi.string().optional(),
}
return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
}

22
yarn.lock
View File

@ -2051,7 +2051,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.10":
"@budibase/backend-core@2.32.11":
version "0.0.0"
dependencies:
"@budibase/nano" "10.1.5"
@ -2132,15 +2132,15 @@
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "2.32.10"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.10.tgz#19fcbb3ced74791a7e96dfdc5a1270165792eea5"
integrity sha512-TbVp2bjmA0rHK+TKi9NVW06+O23fhDm7IJ/FlpWPHIBIZW7xDkCYu6LUOhSwSWMbOTcWzaJFuMbpN1HoTc/YjQ==
version "2.32.11"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.11.tgz#c94d534f829ca0ef252677757e157a7e58b87b4d"
integrity sha512-mOkqJpqHKWsfTWZwWcvBCYFUIluSUHltQNinc1ZRsg9rC3OKoHSDop6gzm744++H/GzGRN8V86kLhCgtNIlkpA==
dependencies:
"@anthropic-ai/sdk" "^0.27.3"
"@budibase/backend-core" "2.32.10"
"@budibase/shared-core" "2.32.10"
"@budibase/string-templates" "2.32.10"
"@budibase/types" "2.32.10"
"@budibase/backend-core" "2.32.11"
"@budibase/shared-core" "2.32.11"
"@budibase/string-templates" "2.32.11"
"@budibase/types" "2.32.11"
"@koa/router" "8.0.8"
bull "4.10.1"
dd-trace "5.2.0"
@ -2153,13 +2153,13 @@
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.10":
"@budibase/shared-core@2.32.11":
version "0.0.0"
dependencies:
"@budibase/types" "0.0.0"
cron-validate "1.4.5"
"@budibase/string-templates@2.32.10":
"@budibase/string-templates@2.32.11":
version "0.0.0"
dependencies:
"@budibase/handlebars-helpers" "^0.13.2"
@ -2167,7 +2167,7 @@
handlebars "^4.7.8"
lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.10":
"@budibase/types@2.32.11":
version "0.0.0"
dependencies:
scim-patch "^0.8.1"