MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge and typing

This commit is contained in:
Martin McKeaveney 2024-10-08 17:38:23 +01:00
parent d5324388d6 fda5fdc16e
commit 7b683cfc50
57 changed files with 4160 additions and 3144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{ {
"$schema": "node_modules/lerna/schemas/lerna-schema.json", "$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.10", "version": "2.32.11",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [ "packages": [
"packages/*", "packages/*",

113
packages/backend-core/src/sql/sql.ts
View File

@ -139,29 +139,61 @@ class InternalBuilder {
return this.table.schema[column] return this.table.schema[column]
} }
// Takes a string like foo and returns a quoted string like [foo] for SQL Server private quoteChars(): [string, string] {
// and "foo" for Postgres.
private quote(str: string): string {
switch (this.client) { switch (this.client) {
case SqlClient.SQL_LITE:
case SqlClient.ORACLE: case SqlClient.ORACLE:
case SqlClient.POSTGRES: case SqlClient.POSTGRES:
return `"${str}"` return ['"', '"']
case SqlClient.MS_SQL: case SqlClient.MS_SQL:
return `[${str}]` return ["[", "]"]
case SqlClient.MARIADB: case SqlClient.MARIADB:
case SqlClient.MY_SQL: case SqlClient.MY_SQL:
return `\`${str}\`` case SqlClient.SQL_LITE:
return ["`", "`"]
} }
} }
// Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c] // Takes a string like foo and returns a quoted string like [foo] for SQL Server
// for SQL Server and `a`.`b`.`c` for MySQL. // and "foo" for Postgres.
private quotedIdentifier(key: string): string { private quote(str: string): string {
return key const [start, end] = this.quoteChars()
.split(".") return `${start}${str}${end}`
.map(part => this.quote(part)) }
.join(".")
private isQuoted(key: string): boolean {
const [start, end] = this.quoteChars()
return key.startsWith(start) && key.endsWith(end)
}
// Takes a string like a.b.c or an array like ["a", "b", "c"] and returns a
// quoted identifier like [a].[b].[c] for SQL Server and `a`.`b`.`c` for
// MySQL.
private quotedIdentifier(key: string | string[]): string {
if (!Array.isArray(key)) {
key = this.splitIdentifier(key)
}
return key.map(part => this.quote(part)).join(".")
}
// Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"]
private splitIdentifier(key: string): string[] {
const [start, end] = this.quoteChars()
if (this.isQuoted(key)) {
return key.slice(1, -1).split(`${end}.${start}`)
}
return key.split(".")
}
private qualifyIdentifier(key: string): string {
const tableName = this.getTableName()
const parts = this.splitIdentifier(key)
if (parts[0] !== tableName) {
parts.unshift(tableName)
}
if (this.isQuoted(key)) {
return this.quotedIdentifier(parts)
}
return parts.join(".")
} }
private isFullSelectStatementRequired(): boolean { private isFullSelectStatementRequired(): boolean {
@ -231,8 +263,13 @@ class InternalBuilder {
// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
// so when we use them we need to wrap them in to_char(). This function // so when we use them we need to wrap them in to_char(). This function
// converts a field name to the appropriate identifier. // converts a field name to the appropriate identifier.
private convertClobs(field: string): string { private convertClobs(field: string, opts?: { forSelect?: boolean }): string {
const parts = field.split(".") if (this.client !== SqlClient.ORACLE) {
throw new Error(
"you've called convertClobs on a DB that's not Oracle, this is a mistake"
)
}
const parts = this.splitIdentifier(field)
const col = parts.pop()! const col = parts.pop()!
const schema = this.table.schema[col] const schema = this.table.schema[col]
let identifier = this.quotedIdentifier(field) let identifier = this.quotedIdentifier(field)
@ -244,8 +281,12 @@ class InternalBuilder {
schema.type === FieldType.OPTIONS || schema.type === FieldType.OPTIONS ||
schema.type === FieldType.BARCODEQR schema.type === FieldType.BARCODEQR
) { ) {
if (opts?.forSelect) {
identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}`
} else {
identifier = `to_char(${identifier})` identifier = `to_char(${identifier})`
} }
}
return identifier return identifier
} }
@ -859,16 +900,45 @@ class InternalBuilder {
const fields = this.query.resource?.fields || [] const fields = this.query.resource?.fields || []
const tableName = this.getTableName() const tableName = this.getTableName()
if (fields.length > 0) { if (fields.length > 0) {
query = query.groupBy(fields.map(field => `${tableName}.${field}`)) const qualifiedFields = fields.map(field => this.qualifyIdentifier(field))
query = query.select(fields.map(field => `${tableName}.${field}`)) if (this.client === SqlClient.ORACLE) {
const groupByFields = qualifiedFields.map(field =>
this.convertClobs(field)
)
const selectFields = qualifiedFields.map(field =>
this.convertClobs(field, { forSelect: true })
)
query = query
.groupByRaw(groupByFields.join(", "))
.select(this.knex.raw(selectFields.join(", ")))
} else {
query = query.groupBy(qualifiedFields).select(qualifiedFields)
}
} }
for (const aggregation of aggregations) { for (const aggregation of aggregations) {
const op = aggregation.calculationType const op = aggregation.calculationType
if (op === CalculationType.COUNT) {
if ("distinct" in aggregation && aggregation.distinct) {
if (this.client === SqlClient.ORACLE) {
const field = this.convertClobs(`${tableName}.${aggregation.field}`)
query = query.select(
this.knex.raw(
`COUNT(DISTINCT ${field}) as ${this.quotedIdentifier(
aggregation.name
)}`
)
)
} else {
query = query.countDistinct(
`${tableName}.${aggregation.field} as ${aggregation.name}`
)
}
} else {
query = query.count(`* as ${aggregation.name}`)
}
} else {
const field = `${tableName}.${aggregation.field} as ${aggregation.name}` const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
switch (op) { switch (op) {
case CalculationType.COUNT:
query = query.count(field)
break
case CalculationType.SUM: case CalculationType.SUM:
query = query.sum(field) query = query.sum(field)
break break
@ -883,6 +953,7 @@ class InternalBuilder {
break break
} }
} }
}
return query return query
} }

2
packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
View File

@ -95,7 +95,7 @@
{#if isView} {#if isView}
<span> <span>
<Toggle <Toggle
value={action.allowedViews?.includes(viewId)} value={action.allowedSources?.includes(viewId)}
on:change={e => toggleAction(action, e.detail)} on:change={e => toggleAction(action, e.detail)}
/> />
</span> </span>

100
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -4,6 +4,7 @@
Button, Button,
Label, Label,
Select, Select,
Multiselect,
Toggle, Toggle,
Icon, Icon,
DatePicker, DatePicker,
@ -142,9 +143,10 @@
} }
$: initialiseField(field, savingColumn) $: initialiseField(field, savingColumn)
$: checkConstraints(editableColumn) $: checkConstraints(editableColumn)
$: required = hasDefault $: required =
? false primaryDisplay ||
: !!editableColumn?.constraints?.presence || primaryDisplay editableColumn?.constraints?.presence === true ||
editableColumn?.constraints?.presence?.allowEmpty === false
$: uneditable = $: uneditable =
$tables.selected?._id === TableNames.USERS && $tables.selected?._id === TableNames.USERS &&
UNEDITABLE_USER_FIELDS.includes(editableColumn.name) UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@ -173,8 +175,8 @@
// used to select what different options can be displayed for column type // used to select what different options can be displayed for column type
$: canBeDisplay = $: canBeDisplay =
canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
$: canHaveDefault = $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type) $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
$: canBeRequired = $: canBeRequired =
editableColumn?.type !== LINK_TYPE && editableColumn?.type !== LINK_TYPE &&
!uneditable && !uneditable &&
@ -193,7 +195,6 @@
(originalName && (originalName &&
SWITCHABLE_TYPES[field.type] && SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn) !editableColumn?.autocolumn)
$: allowedTypes = getAllowedTypes(datasource).map(t => ({ $: allowedTypes = getAllowedTypes(datasource).map(t => ({
fieldId: makeFieldId(t.type, t.subtype), fieldId: makeFieldId(t.type, t.subtype),
...t, ...t,
@ -211,6 +212,11 @@
}, },
...getUserBindings(), ...getUserBindings(),
] ]
$: sanitiseDefaultValue(
editableColumn.type,
editableColumn.constraints?.inclusion || [],
editableColumn.default
)
const fieldDefinitions = Object.values(FIELDS).reduce( const fieldDefinitions = Object.values(FIELDS).reduce(
// Storing the fields by complex field id // Storing the fields by complex field id
@ -300,6 +306,22 @@
delete saveColumn.fieldName delete saveColumn.fieldName
} }
// Ensure we don't have a default value if we can't have one
if (!canHaveDefault || !defaultValuesEnabled) {
delete saveColumn.default
}
// Ensure primary display columns are always required and don't have default values
if (primaryDisplay) {
saveColumn.constraints.presence = { allowEmpty: false }
delete saveColumn.default
}
// Ensure the field is not required if we have a default value
if (saveColumn.default) {
saveColumn.constraints.presence = false
}
try { try {
await tables.saveField({ await tables.saveField({
originalName, originalName,
@ -547,6 +569,20 @@
return newError return newError
} }
const sanitiseDefaultValue = (type, options, defaultValue) => {
if (!defaultValue?.length) {
return
}
// Delete default value for options fields if the option is no longer available
if (type === FieldType.OPTIONS && !options.includes(defaultValue)) {
delete editableColumn.default
}
// Filter array default values to only valid options
if (type === FieldType.ARRAY) {
editableColumn.default = defaultValue.filter(x => options.includes(x))
}
}
onMount(() => { onMount(() => {
mounted = true mounted = true
}) })
@ -761,9 +797,9 @@
schema={table.schema} schema={table.schema}
/> />
{:else if editableColumn.type === JSON_TYPE} {:else if editableColumn.type === JSON_TYPE}
<Button primary text on:click={openJsonSchemaEditor} <Button primary text on:click={openJsonSchemaEditor}>
>Open schema editor</Button Open schema editor
> </Button>
{/if} {/if}
{#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn} {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
<Select <Select
@ -792,27 +828,39 @@
</div> </div>
{/if} {/if}
{#if canHaveDefault} {#if defaultValuesEnabled}
<div> {#if editableColumn.type === FieldType.OPTIONS}
<ModalBindableInput <Select
panel={ServerBindingPanel} disabled={!canHaveDefault}
title="Default" options={editableColumn.constraints?.inclusion || []}
label="Default" label="Default value"
value={editableColumn.default} value={editableColumn.default}
on:change={e => { on:change={e => (editableColumn.default = e.detail)}
editableColumn = { placeholder="None"
...editableColumn, />
default: e.detail, {:else if editableColumn.type === FieldType.ARRAY}
} <Multiselect
disabled={!canHaveDefault}
if (e.detail) { options={editableColumn.constraints?.inclusion || []}
setRequired(false) label="Default value"
} value={editableColumn.default}
}} on:change={e =>
(editableColumn.default = e.detail?.length ? e.detail : undefined)}
placeholder="None"
/>
{:else}
<ModalBindableInput
disabled={!canHaveDefault}
panel={ServerBindingPanel}
title="Default value"
label="Default value"
placeholder="None"
value={editableColumn.default}
on:change={e => (editableColumn.default = e.detail)}
bindings={defaultValueBindings} bindings={defaultValueBindings}
allowJS allowJS
/> />
</div> {/if}
{/if} {/if}
</Layout> </Layout>

18
packages/builder/src/components/common/bindings/BindingPanel.svelte
View File

@ -66,6 +66,7 @@
let insertAtPos let insertAtPos
let targetMode = null let targetMode = null
let expressionResult let expressionResult
let expressionError
let evaluating = false let evaluating = false
$: useSnippets = allowSnippets && !$licensing.isFreePlan $: useSnippets = allowSnippets && !$licensing.isFreePlan
@ -142,10 +143,22 @@
} }
const debouncedEval = Utils.debounce((expression, context, snippets) => { const debouncedEval = Utils.debounce((expression, context, snippets) => {
expressionResult = processStringSync(expression || "", { try {
expressionError = null
expressionResult = processStringSync(
expression || "",
{
...context, ...context,
snippets, snippets,
}) },
{
noThrow: false,
}
)
} catch (err) {
expressionResult = null
expressionError = err
}
evaluating = false evaluating = false
}, 260) }, 260)
@ -370,6 +383,7 @@
{:else if sidePanel === SidePanels.Evaluation} {:else if sidePanel === SidePanels.Evaluation}
<EvaluationSidePanel <EvaluationSidePanel
{expressionResult} {expressionResult}
{expressionError}
{evaluating} {evaluating}
expression={editorValue} expression={editorValue}
/> />

19
packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte
View File

@ -3,26 +3,37 @@
import { Icon, ProgressCircle, notifications } from "@budibase/bbui" import { Icon, ProgressCircle, notifications } from "@budibase/bbui"
import { copyToClipboard } from "@budibase/bbui/helpers" import { copyToClipboard } from "@budibase/bbui/helpers"
import { fade } from "svelte/transition" import { fade } from "svelte/transition"
import { UserScriptError } from "@budibase/string-templates"
export let expressionResult export let expressionResult
export let expressionError
export let evaluating = false export let evaluating = false
export let expression = null export let expression = null
$: error = expressionResult === "Error while executing JS" $: error = expressionError != null
$: empty = expression == null || expression?.trim() === "" $: empty = expression == null || expression?.trim() === ""
$: success = !error && !empty $: success = !error && !empty
$: highlightedResult = highlight(expressionResult) $: highlightedResult = highlight(expressionResult)
const formatError = err => {
if (err.code === UserScriptError.code) {
return err.userScriptError.toString()
}
return err.toString()
}
const highlight = json => { const highlight = json => {
if (json == null) { if (json == null) {
return "" return ""
} }
// Attempt to parse and then stringify, in case this is valid JSON
// Attempt to parse and then stringify, in case this is valid result
try { try {
json = JSON.stringify(JSON.parse(json), null, 2) json = JSON.stringify(JSON.parse(json), null, 2)
} catch (err) { } catch (err) {
// Ignore // Ignore
} }
return formatHighlight(json, { return formatHighlight(json, {
keyColor: "#e06c75", keyColor: "#e06c75",
numberColor: "#e5c07b", numberColor: "#e5c07b",
@ -34,7 +45,7 @@
} }
const copy = () => { const copy = () => {
let clipboardVal = expressionResult let clipboardVal = expressionResult.result
if (typeof clipboardVal === "object") { if (typeof clipboardVal === "object") {
clipboardVal = JSON.stringify(clipboardVal, null, 2) clipboardVal = JSON.stringify(clipboardVal, null, 2)
} }
@ -73,6 +84,8 @@
<div class="body"> <div class="body">
{#if empty} {#if empty}
Your expression will be evaluated here Your expression will be evaluated here
{:else if error}
{formatError(expressionError)}
{:else} {:else}
<!-- eslint-disable-next-line svelte/no-at-html-tags--> <!-- eslint-disable-next-line svelte/no-at-html-tags-->
{@html highlightedResult} {@html highlightedResult}

4
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
View File

@ -63,7 +63,9 @@
$: rowActions.refreshRowActions(id) $: rowActions.refreshRowActions(id)
const makeRowActionButtons = actions => { const makeRowActionButtons = actions => {
return (actions || []).map(action => ({ return (actions || [])
.filter(action => action.allowedSources?.includes(id))
.map(action => ({
text: action.name, text: action.name,
onClick: async row => { onClick: async row => {
await rowActions.trigger(id, action.id, row._id) await rowActions.trigger(id, action.id, row._id)

3
packages/builder/src/pages/builder/portal/apps/index.svelte
View File

@ -26,6 +26,7 @@
licensing, licensing,
environment, environment,
enrichedApps, enrichedApps,
sortBy,
} from "stores/portal" } from "stores/portal"
import { goto } from "@roxi/routify" import { goto } from "@roxi/routify"
import AppRow from "components/start/AppRow.svelte" import AppRow from "components/start/AppRow.svelte"
@ -247,7 +248,7 @@
<div class="app-actions"> <div class="app-actions">
<Select <Select
autoWidth autoWidth
value={$appsStore.sortBy} value={$sortBy}
on:change={e => { on:change={e => {
appsStore.updateSort(e.detail) appsStore.updateSort(e.detail)
}} }}

12
packages/builder/src/stores/builder/rowActions.js
View File

@ -129,13 +129,15 @@ const derivedStore = derived(store, $store => {
// Generate an entry for every view as well // Generate an entry for every view as well
Object.keys($store || {}).forEach(tableId => { Object.keys($store || {}).forEach(tableId => {
// We need to have all the actions for the table in order to be displayed in the crud section
map[tableId] = $store[tableId] map[tableId] = $store[tableId]
for (let action of $store[tableId]) { for (let action of $store[tableId]) {
for (let viewId of action.allowedViews || []) { const otherSources = (action.allowedSources || []).filter(
if (!map[viewId]) { sourceId => sourceId !== tableId
map[viewId] = [] )
} for (let source of otherSources) {
map[viewId].push(action) map[source] ??= []
map[source].push(action)
} }
} }
}) })

23
packages/builder/src/stores/builder/viewsV2.js
View File

@ -1,6 +1,24 @@
import { writable, derived, get } from "svelte/store" import { writable, derived, get } from "svelte/store"
import { tables } from "./tables" import { tables } from "./tables"
import { API } from "api" import { API } from "api"
import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
}
return view
}
function convertToSearchFilterGroup(view) {
if (view.queryUI) {
view.query = view.queryUI
delete view.queryUI
}
return view
}
export function createViewsV2Store() { export function createViewsV2Store() {
const store = writable({ const store = writable({
@ -12,7 +30,7 @@ export function createViewsV2Store() {
const views = Object.values(table?.views || {}).filter(view => { const views = Object.values(table?.views || {}).filter(view => {
return view.version === 2 return view.version === 2
}) })
list = list.concat(views) list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
}) })
return { return {
...$store, ...$store,
@ -34,6 +52,7 @@ export function createViewsV2Store() {
} }
const create = async view => { const create = async view => {
view = convertToSearchFilters(view)
const savedViewResponse = await API.viewV2.create(view) const savedViewResponse = await API.viewV2.create(view)
const savedView = savedViewResponse.data const savedView = savedViewResponse.data
replaceView(savedView.id, savedView) replaceView(savedView.id, savedView)
@ -41,6 +60,7 @@ export function createViewsV2Store() {
} }
const save = async view => { const save = async view => {
view = convertToSearchFilters(view)
const res = await API.viewV2.update(view) const res = await API.viewV2.update(view)
const savedView = res?.data const savedView = res?.data
replaceView(view.id, savedView) replaceView(view.id, savedView)
@ -51,6 +71,7 @@ export function createViewsV2Store() {
if (!viewId) { if (!viewId) {
return return
} }
view = convertToSearchFilterGroup(view)
const existingView = get(derivedStore).list.find(view => view.id === viewId) const existingView = get(derivedStore).list.find(view => view.id === viewId)
const tableIndex = get(tables).list.findIndex(table => { const tableIndex = get(tables).list.findIndex(table => {
return table._id === view?.tableId || table._id === existingView?.tableId return table._id === view?.tableId || table._id === existingView?.tableId

25
packages/builder/src/stores/portal/apps.js
View File

@ -9,7 +9,6 @@ const DEV_PROPS = ["updatedBy", "updatedAt"]
export const INITIAL_APPS_STATE = { export const INITIAL_APPS_STATE = {
apps: [], apps: [],
sortBy: "name",
} }
export class AppsStore extends BudiStore { export class AppsStore extends BudiStore {
@ -53,6 +52,15 @@ export class AppsStore extends BudiStore {
...state, ...state,
sortBy, sortBy,
})) }))
this.updateUserSort(sortBy)
}
async updateUserSort(sortBy) {
try {
await auth.updateSelf({ appSort: sortBy })
} catch (err) {
console.error("couldn't save user sort: ", err)
}
} }
async load() { async load() {
@ -140,8 +148,14 @@ export class AppsStore extends BudiStore {
export const appsStore = new AppsStore() export const appsStore = new AppsStore()
export const sortBy = derived([appsStore, auth], ([$store, $auth]) => {
return $store.sortBy || $auth.user?.appSort || "name"
})
// Centralise any logic that enriches the apps list // Centralise any logic that enriches the apps list
export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => { export const enrichedApps = derived(
[appsStore, auth, sortBy],
([$store, $auth, $sortBy]) => {
const enrichedApps = $store.apps const enrichedApps = $store.apps
? $store.apps.map(app => ({ ? $store.apps.map(app => ({
...app, ...app,
@ -152,7 +166,7 @@ export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
})) }))
: [] : []
if ($store.sortBy === "status") { if ($sortBy === "status") {
return enrichedApps.sort((a, b) => { return enrichedApps.sort((a, b) => {
if (a.favourite === b.favourite) { if (a.favourite === b.favourite) {
if (a.status === b.status) { if (a.status === b.status) {
@ -162,7 +176,7 @@ export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
} }
return a.favourite ? -1 : 1 return a.favourite ? -1 : 1
}) })
} else if ($store.sortBy === "updated") { } else if ($sortBy === "updated") {
return enrichedApps?.sort((a, b) => { return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) { if (a.favourite === b.favourite) {
const aUpdated = a.updatedAt || "9999" const aUpdated = a.updatedAt || "9999"
@ -179,4 +193,5 @@ export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
return a.favourite ? -1 : 1 return a.favourite ? -1 : 1
}) })
} }
}) }
)

2
packages/builder/src/stores/portal/index.js
View File

@ -3,7 +3,7 @@ import { writable } from "svelte/store"
export { organisation } from "./organisation" export { organisation } from "./organisation"
export { users } from "./users" export { users } from "./users"
export { admin } from "./admin" export { admin } from "./admin"
export { appsStore, enrichedApps } from "./apps" export { appsStore, enrichedApps, sortBy } from "./apps"
export { email } from "./email" export { email } from "./email"
export { auth } from "./auth" export { auth } from "./auth"
export { oidc } from "./oidc" export { oidc } from "./oidc"

3
packages/frontend-core/src/components/grid/cells/AICell.svelte
View File

@ -70,9 +70,6 @@
align-items: flex-start; align-items: flex-start;
overflow: hidden; overflow: hidden;
} }
.long-form-cell.editable:hover {
cursor: text;
}
.value { .value {
display: -webkit-box; display: -webkit-box;
-webkit-line-clamp: var(--content-lines); -webkit-line-clamp: var(--content-lines);

17
packages/frontend-core/src/components/grid/stores/datasource.js
View File

@ -1,6 +1,7 @@
import { derived, get } from "svelte/store" import { derived, get } from "svelte/store"
import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
import { enrichSchemaWithRelColumns, memo } from "../../../utils" import { enrichSchemaWithRelColumns, memo } from "../../../utils"
import { cloneDeep } from "lodash"
export const createStores = () => { export const createStores = () => {
const definition = memo(null) const definition = memo(null)
@ -164,10 +165,18 @@ export const createActions = context => {
// Updates the datasources primary display column // Updates the datasources primary display column
const changePrimaryDisplay = async column => { const changePrimaryDisplay = async column => {
return await saveDefinition({ let newDefinition = cloneDeep(get(definition))
...get(definition),
primaryDisplay: column, // Update primary display
}) newDefinition.primaryDisplay = column
// Sanitise schema to ensure field is required and has no default value
if (!newDefinition.schema[column].constraints) {
newDefinition.schema[column].constraints = {}
}
newDefinition.schema[column].constraints.presence = { allowEmpty: false }
delete newDefinition.schema[column].default
return await saveDefinition(newDefinition)
} }
// Adds a schema mutation for a single field // Adds a schema mutation for a single field

14
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -1,4 +1,14 @@
import { get } from "svelte/store" import { get } from "svelte/store"
import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
}
return view
}
const SuppressErrors = true const SuppressErrors = true
@ -6,7 +16,7 @@ export const createActions = context => {
const { API, datasource, columns } = context const { API, datasource, columns } = context
const saveDefinition = async newDefinition => { const saveDefinition = async newDefinition => {
await API.viewV2.update(newDefinition) await API.viewV2.update(convertToSearchFilters(newDefinition))
} }
const saveRow = async row => { const saveRow = async row => {
@ -125,7 +135,7 @@ export const initialise = context => {
} }
// Only override filter state if we don't have an initial filter // Only override filter state if we don't have an initial filter
if (!get(initialFilter)) { if (!get(initialFilter)) {
filter.set($definition.query) filter.set($definition.queryUI || $definition.query)
} }
}) })
) )

2
packages/pro

@ -1 +1 @@
Subproject commit 0c8f834932670ce1d3b3f639e6e8d0d0086e508d Subproject commit 1bce30827813e135ee414141fdf51e45236b8f3c

3
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -697,9 +697,8 @@ export class ExternalRequest<T extends Operation> {
const calculationFields = helpers.views.calculationFields(this.source) const calculationFields = helpers.views.calculationFields(this.source)
for (const [key, field] of Object.entries(calculationFields)) { for (const [key, field] of Object.entries(calculationFields)) {
aggregations.push({ aggregations.push({
...field,
name: key, name: key,
field: field.field,
calculationType: field.calculationType,
}) })
} }
} }

4
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -144,7 +144,9 @@ export async function finaliseRow(
dynamic: false, dynamic: false,
contextRows: [enrichedRow], contextRows: [enrichedRow],
}) })
const aiEnabled = await pro.features.isBudibaseAIEnabled() || await pro.features.isAICustomConfigsEnabled() const aiEnabled =
(await pro.features.isBudibaseAIEnabled()) ||
(await pro.features.isAICustomConfigsEnabled())
if (aiEnabled) { if (aiEnabled) {
row = await processAIColumns(table, row, { row = await processAIColumns(table, row, {
contextRows: [enrichedRow], contextRows: [enrichedRow],

6
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -125,9 +125,11 @@ export async function buildSqlFieldList(
column.type !== FieldType.LINK && column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA && column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI && column.type !== FieldType.AI &&
!existing.find((field: string) => field === columnName) !existing.find(
(field: string) => field === `${table.name}.${columnName}`
) )
.map(column => `${table.name}.${column[0]}`) )
.map(([columnName]) => `${table.name}.${columnName}`)
} }
let fields: string[] = [] let fields: string[] = []

22
packages/server/src/api/controllers/row/views.ts
View File

@ -3,6 +3,8 @@ import {
ViewV2, ViewV2,
SearchRowResponse, SearchRowResponse,
SearchViewRowRequest, SearchViewRowRequest,
RequiredKeys,
RowSearchParams,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
@ -20,30 +22,34 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`) ctx.throw(400, `This method only supports viewsV2`)
} }
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request const { body } = ctx.request
await context.ensureSnippetContext(true) await context.ensureSnippetContext(true)
const result = await sdk.rows.search( const searchOptions: RequiredKeys<SearchViewRowRequest> &
{ RequiredKeys<
viewId: view.id, Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
> = {
tableId: view.tableId, tableId: view.tableId,
viewId: view.id,
query: body.query, query: body.query,
fields: viewFields,
...getSortOptions(body, view), ...getSortOptions(body, view),
limit: body.limit, limit: body.limit,
bookmark: body.bookmark, bookmark: body.bookmark,
paginate: body.paginate, paginate: body.paginate,
countRows: body.countRows, countRows: body.countRows,
},
{
user: sdk.users.getUserContextBindings(ctx.user),
} }
)
const result = await sdk.rows.search(searchOptions, {
user: sdk.users.getUserContextBindings(ctx.user),
})
result.rows.forEach(r => (r._viewId = view.id)) result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result ctx.body = result
} }
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) { if (request.sort) {
return { return {

87
packages/server/src/api/controllers/rowAction/crud.ts
View File

@ -1,6 +1,7 @@
import { import {
CreateRowActionRequest, CreateRowActionRequest,
Ctx, Ctx,
RowActionPermissions,
RowActionResponse, RowActionResponse,
RowActionsResponse, RowActionsResponse,
UpdateRowActionRequest, UpdateRowActionRequest,
@ -18,25 +19,26 @@ async function getTable(ctx: Ctx) {
export async function find(ctx: Ctx<void, RowActionsResponse>) { export async function find(ctx: Ctx<void, RowActionsResponse>) {
const table = await getTable(ctx) const table = await getTable(ctx)
const tableId = table._id!
if (!(await sdk.rowActions.docExists(table._id!))) { if (!(await sdk.rowActions.docExists(tableId))) {
ctx.body = { ctx.body = {
actions: {}, actions: {},
} }
return return
} }
const { actions } = await sdk.rowActions.getAll(table._id!) const { actions } = await sdk.rowActions.getAll(tableId)
const result: RowActionsResponse = { const result: RowActionsResponse = {
actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>( actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
(acc, [key, action]) => ({ (acc, [key, action]) => ({
...acc, ...acc,
[key]: { [key]: {
id: key, id: key,
tableId: table._id!, tableId,
name: action.name, name: action.name,
automationId: action.automationId, automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views), allowedSources: flattenAllowedSources(tableId, action.permissions),
}, },
}), }),
{} {}
@ -49,17 +51,18 @@ export async function create(
ctx: Ctx<CreateRowActionRequest, RowActionResponse> ctx: Ctx<CreateRowActionRequest, RowActionResponse>
) { ) {
const table = await getTable(ctx) const table = await getTable(ctx)
const tableId = table._id!
const createdAction = await sdk.rowActions.create(table._id!, { const createdAction = await sdk.rowActions.create(tableId, {
name: ctx.request.body.name, name: ctx.request.body.name,
}) })
ctx.body = { ctx.body = {
tableId: table._id!, tableId,
id: createdAction.id, id: createdAction.id,
name: createdAction.name, name: createdAction.name,
automationId: createdAction.automationId, automationId: createdAction.automationId,
allowedViews: undefined, allowedSources: flattenAllowedSources(tableId, createdAction.permissions),
} }
ctx.status = 201 ctx.status = 201
} }
@ -68,18 +71,19 @@ export async function update(
ctx: Ctx<UpdateRowActionRequest, RowActionResponse> ctx: Ctx<UpdateRowActionRequest, RowActionResponse>
) { ) {
const table = await getTable(ctx) const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params const { actionId } = ctx.params
const action = await sdk.rowActions.update(table._id!, actionId, { const action = await sdk.rowActions.update(tableId, actionId, {
name: ctx.request.body.name, name: ctx.request.body.name,
}) })
ctx.body = { ctx.body = {
tableId: table._id!, tableId,
id: action.id, id: action.id,
name: action.name, name: action.name,
automationId: action.automationId, automationId: action.automationId,
allowedViews: undefined, allowedSources: flattenAllowedSources(tableId, action.permissions),
} }
} }
@ -91,52 +95,89 @@ export async function remove(ctx: Ctx<void, void>) {
ctx.status = 204 ctx.status = 204
} }
export async function setTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.setTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.unsetTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) { export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx) const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.setViewPermission( const action = await sdk.rowActions.setViewPermission(
table._id!, tableId,
actionId, actionId,
viewId viewId
) )
ctx.body = { ctx.body = {
tableId: table._id!, tableId,
id: action.id, id: action.id,
name: action.name, name: action.name,
automationId: action.automationId, automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views), allowedSources: flattenAllowedSources(tableId, action.permissions),
} }
} }
export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) { export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx) const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.unsetViewPermission( const action = await sdk.rowActions.unsetViewPermission(
table._id!, tableId,
actionId, actionId,
viewId viewId
) )
ctx.body = { ctx.body = {
tableId: table._id!, tableId,
id: action.id, id: action.id,
name: action.name, name: action.name,
automationId: action.automationId, automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views), allowedSources: flattenAllowedSources(tableId, action.permissions),
} }
} }
function flattenAllowedViews( function flattenAllowedSources(
permissions: Record<string, { runAllowed: boolean }> tableId: string,
permissions: RowActionPermissions
) { ) {
const allowedPermissions = Object.entries(permissions || {}) const allowedPermissions = []
.filter(([_, p]) => p.runAllowed) if (permissions.table.runAllowed) {
.map(([viewId]) => viewId) allowedPermissions.push(tableId)
if (!allowedPermissions.length) {
return undefined
} }
allowedPermissions.push(
...Object.keys(permissions.views || {}).filter(
viewId => permissions.views[viewId].runAllowed
)
)
return allowedPermissions return allowedPermissions
} }

9
packages/server/src/api/controllers/script.ts
View File

@ -1,11 +1,18 @@
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import { IsolatedVM } from "../../jsRunner/vm" import { IsolatedVM } from "../../jsRunner/vm"
import { iifeWrapper } from "@budibase/string-templates" import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
export async function execute(ctx: Ctx) { export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body const { script, context } = ctx.request.body
const vm = new IsolatedVM() const vm = new IsolatedVM()
try {
ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script))) ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
} catch (err: any) {
if (err.code === UserScriptError.code) {
throw err.userScriptError
}
throw err
}
} }
export async function save(ctx: Ctx) { export async function save(ctx: Ctx) {

2
packages/server/src/api/controllers/table/utils.ts
View File

@ -33,7 +33,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import env from "../../../environment" import env from "../../../environment"
import { runAIColumnChecks, runStaticFormulaChecks } from "./bulkFormula" import { runStaticFormulaChecks } from "./bulkFormula"
export async function clearColumns(table: Table, columnNames: string[]) { export async function clearColumns(table: Table, columnNames: string[]) {
const db = context.getAppDB() const db = context.getAppDB()

34
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -11,14 +11,40 @@ import {
ViewCalculationFieldMetadata, ViewCalculationFieldMetadata,
RelationSchemaField, RelationSchemaField,
ViewFieldMetadata, ViewFieldMetadata,
CalculationType,
} from "@budibase/types" } from "@budibase/types"
import { builderSocket, gridSocket } from "../../../websockets" import { builderSocket, gridSocket } from "../../../websockets"
import { helpers } from "@budibase/shared-core" import { helpers } from "@budibase/shared-core"
function stripUnknownFields( function stripUnknownFields(
field: BasicViewFieldMetadata field: ViewFieldMetadata
): RequiredKeys<BasicViewFieldMetadata> { ): RequiredKeys<ViewFieldMetadata> {
if (helpers.views.isCalculationField(field)) { if (helpers.views.isCalculationField(field)) {
if (field.calculationType === CalculationType.COUNT) {
if ("distinct" in field && field.distinct) {
return {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
distinct: field.distinct,
calculationType: field.calculationType,
field: field.field,
columns: field.columns,
}
} else {
return {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
calculationType: field.calculationType,
columns: field.columns,
}
}
}
const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = { const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
order: field.order, order: field.order,
width: field.width, width: field.width,
@ -103,9 +129,11 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
name: view.name, name: view.name,
tableId: view.tableId, tableId: view.tableId,
query: view.query, query: view.query,
queryUI: view.queryUI,
sort: view.sort, sort: view.sort,
schema, schema,
primaryDisplay: view.primaryDisplay, primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
} }
const result = await sdk.views.create(tableId, parsedView) const result = await sdk.views.create(tableId, parsedView)
ctx.status = 201 ctx.status = 201
@ -138,9 +166,11 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
version: view.version, version: view.version,
tableId: view.tableId, tableId: view.tableId,
query: view.query, query: view.query,
queryUI: view.queryUI,
sort: view.sort, sort: view.sort,
schema, schema,
primaryDisplay: view.primaryDisplay, primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
} }
const result = await sdk.views.update(tableId, parsedView) const result = await sdk.views.update(tableId, parsedView)

10
packages/server/src/api/routes/rowAction.ts
View File

@ -51,6 +51,16 @@ router
authorized(BUILDER), authorized(BUILDER),
rowActionController.remove rowActionController.remove
) )
.post(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.setTablePermission
)
.delete(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.unsetTablePermission
)
.post( .post(
"/api/tables/:tableId/actions/:actionId/permissions/:viewId", "/api/tables/:tableId/actions/:actionId/permissions/:viewId",
authorized(BUILDER), authorized(BUILDER),

5
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -49,6 +49,7 @@ import * as uuid from "uuid"
import { Knex } from "knex" import { Knex } from "knex"
import { InternalTables } from "../../../db/utils" import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment" import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp) tk.freeze(timestamp)
@ -3013,7 +3014,7 @@ describe.each([
let i = 0 let i = 0
for (; i < 10; i++) { for (; i < 10; i++) {
const row = rows[i] const row = rows[i]
if (row.formula !== "Timed out while executing JS") { if (row.formula !== JsTimeoutError.message) {
break break
} }
} }
@ -3027,7 +3028,7 @@ describe.each([
for (; i < 10; i++) { for (; i < 10; i++) {
const row = rows[i] const row = rows[i]
expect(row.text).toBe("foo") expect(row.text).toBe("foo")
expect(row.formula).toBe("Request JS execution limit hit") expect(row.formula).toStartWith("CPU time limit exceeded ")
} }
} }
} }

145
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -133,6 +133,7 @@ describe("/rowsActions", () => {
id: expect.stringMatching(/^row_action_\w+/), id: expect.stringMatching(/^row_action_\w+/),
tableId: tableId, tableId: tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}) })
expect(await config.api.rowAction.find(tableId)).toEqual({ expect(await config.api.rowAction.find(tableId)).toEqual({
@ -142,6 +143,7 @@ describe("/rowsActions", () => {
id: res.id, id: res.id,
tableId: tableId, tableId: tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}, },
}, },
}) })
@ -180,18 +182,21 @@ describe("/rowsActions", () => {
id: responses[0].id, id: responses[0].id,
tableId, tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}, },
[responses[1].id]: { [responses[1].id]: {
name: rowActions[1].name, name: rowActions[1].name,
id: responses[1].id, id: responses[1].id,
tableId, tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}, },
[responses[2].id]: { [responses[2].id]: {
name: rowActions[2].name, name: rowActions[2].name,
id: responses[2].id, id: responses[2].id,
tableId, tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}, },
}, },
}) })
@ -224,6 +229,7 @@ describe("/rowsActions", () => {
id: expect.any(String), id: expect.any(String),
tableId, tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}) })
expect(await config.api.rowAction.find(tableId)).toEqual({ expect(await config.api.rowAction.find(tableId)).toEqual({
@ -233,6 +239,7 @@ describe("/rowsActions", () => {
id: res.id, id: res.id,
tableId: tableId, tableId: tableId,
automationId: expectAutomationId(), automationId: expectAutomationId(),
allowedSources: [tableId],
}, },
}, },
}) })
@ -354,6 +361,7 @@ describe("/rowsActions", () => {
tableId, tableId,
name: updatedName, name: updatedName,
automationId: actionData.automationId, automationId: actionData.automationId,
allowedSources: [tableId],
}) })
expect(await config.api.rowAction.find(tableId)).toEqual( expect(await config.api.rowAction.find(tableId)).toEqual(
@ -364,6 +372,7 @@ describe("/rowsActions", () => {
id: actionData.id, id: actionData.id,
tableId: actionData.tableId, tableId: actionData.tableId,
automationId: actionData.automationId, automationId: actionData.automationId,
allowedSources: [tableId],
}, },
}), }),
}) })
@ -515,6 +524,81 @@ describe("/rowsActions", () => {
}) })
}) })
describe("set/unsetTablePermission", () => {
describe.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])("unauthorisedTests for %s", (__, delegateTest) => {
unauthorisedTests((expectations, testConfig) =>
delegateTest(tableId, generator.guid(), expectations, testConfig)
)
})
let actionId1: string, actionId2: string
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
const persisted = await config.api.rowAction.find(tableId)
const actions = _.sampleSize(Object.keys(persisted.actions), 2)
actionId1 = actions[0]
actionId2 = actions[1]
})
it("can set table permission", async () => {
await config.api.rowAction.unsetTablePermission(tableId, actionId1)
await config.api.rowAction.unsetTablePermission(tableId, actionId2)
const actionResult = await config.api.rowAction.setTablePermission(
tableId,
actionId1
)
const expectedAction1 = expect.objectContaining({
allowedSources: [tableId],
})
const expectedActions = expect.objectContaining({
[actionId1]: expectedAction1,
[actionId2]: expect.objectContaining({
allowedSources: [],
}),
})
expect(actionResult).toEqual(expectedAction1)
expect((await config.api.rowAction.find(tableId)).actions).toEqual(
expectedActions
)
})
it("can unset table permission", async () => {
const actionResult = await config.api.rowAction.unsetTablePermission(
tableId,
actionId1
)
const expectedAction = expect.objectContaining({
allowedSources: [],
})
expect(actionResult).toEqual(expectedAction)
expect(
(await config.api.rowAction.find(tableId)).actions[actionId1]
).toEqual(expectedAction)
})
it.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])(
"cannot update permission for unexisting tables (%s)",
async (__, delegateTest) => {
const tableId = generator.guid()
await delegateTest(tableId, actionId1, {
status: 404,
})
}
)
})
describe("set/unsetViewPermission", () => { describe("set/unsetViewPermission", () => {
describe.each([ describe.each([
["setViewPermission", config.api.rowAction.setViewPermission], ["setViewPermission", config.api.rowAction.setViewPermission],
@ -531,11 +615,9 @@ describe("/rowsActions", () => {
) )
}) })
let tableIdForDescribe: string
let actionId1: string, actionId2: string let actionId1: string, actionId2: string
let viewId1: string, viewId2: string let viewId1: string, viewId2: string
beforeAll(async () => { beforeEach(async () => {
tableIdForDescribe = tableId
for (const rowAction of createRowActionRequests(3)) { for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction) await createRowAction(tableId, rowAction)
} }
@ -557,11 +639,6 @@ describe("/rowsActions", () => {
).id ).id
}) })
beforeEach(() => {
// Hack to reuse tables for these given tests
tableId = tableIdForDescribe
})
it("can set permission views", async () => { it("can set permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId1) await config.api.rowAction.setViewPermission(tableId, viewId1, actionId1)
const action1Result = await config.api.rowAction.setViewPermission( const action1Result = await config.api.rowAction.setViewPermission(
@ -576,10 +653,10 @@ describe("/rowsActions", () => {
) )
const expectedAction1 = expect.objectContaining({ const expectedAction1 = expect.objectContaining({
allowedViews: [viewId1, viewId2], allowedSources: [tableId, viewId1, viewId2],
}) })
const expectedAction2 = expect.objectContaining({ const expectedAction2 = expect.objectContaining({
allowedViews: [viewId1], allowedSources: [tableId, viewId1],
}) })
const expectedActions = expect.objectContaining({ const expectedActions = expect.objectContaining({
@ -594,6 +671,8 @@ describe("/rowsActions", () => {
}) })
it("can unset permission views", async () => { it("can unset permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId2, actionId1)
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId2)
const actionResult = await config.api.rowAction.unsetViewPermission( const actionResult = await config.api.rowAction.unsetViewPermission(
tableId, tableId,
viewId1, viewId1,
@ -601,7 +680,7 @@ describe("/rowsActions", () => {
) )
const expectedAction = expect.objectContaining({ const expectedAction = expect.objectContaining({
allowedViews: [viewId2], allowedSources: [tableId, viewId2],
}) })
expect(actionResult).toEqual(expectedAction) expect(actionResult).toEqual(expectedAction)
expect( expect(
@ -672,6 +751,7 @@ describe("/rowsActions", () => {
) )
await config.publish() await config.publish()
// Travel time in order to "trim" the selected `getAutomationLogs`
tk.travel(Date.now() + 100) tk.travel(Date.now() + 100)
}) })
@ -698,6 +778,8 @@ describe("/rowsActions", () => {
inputs: null, inputs: null,
outputs: { outputs: {
fields: {}, fields: {},
id: rowId,
revision: (await config.api.row.get(tableId, rowId))._rev,
row: await config.api.row.get(tableId, rowId), row: await config.api.row.get(tableId, rowId),
table: { table: {
...(await config.api.table.get(tableId)), ...(await config.api.table.get(tableId)),
@ -712,6 +794,38 @@ describe("/rowsActions", () => {
]) ])
}) })
it("triggers from an allowed table", async () => {
expect(await getAutomationLogs()).toBeEmpty()
await config.api.rowAction.trigger(tableId, rowAction.id, { rowId })
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([
expect.objectContaining({
automationId: rowAction.automationId,
}),
])
})
it("rejects triggering from a non-allowed table", async () => {
await config.api.rowAction.unsetTablePermission(tableId, rowAction.id)
await config.publish()
await config.api.rowAction.trigger(
tableId,
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([])
})
it("rejects triggering from a non-allowed view", async () => { it("rejects triggering from a non-allowed view", async () => {
const viewId = ( const viewId = (
await config.api.viewV2.create( await config.api.viewV2.create(
@ -901,7 +1015,7 @@ describe("/rowsActions", () => {
}) })
it.each(allowedRoleConfig)( it.each(allowedRoleConfig)(
"does not allow running row actions for tables by default even", "allow running row actions for tables by default",
async (userRole, resourcePermission) => { async (userRole, resourcePermission) => {
await config.api.permission.add({ await config.api.permission.add({
level: PermissionLevel.READ, level: PermissionLevel.READ,
@ -918,15 +1032,12 @@ describe("/rowsActions", () => {
rowAction.id, rowAction.id,
{ rowId }, { rowId },
{ {
status: 403, status: 200,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
} }
) )
const automationLogs = await getAutomationLogs() const automationLogs = await getAutomationLogs()
expect(automationLogs).toBeEmpty() expect(automationLogs).toHaveLength(1)
}) })
} }
) )

371
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -28,11 +28,13 @@ import {
RowSearchParams, RowSearchParams,
SearchFilters, SearchFilters,
SearchResponse, SearchResponse,
SearchRowRequest,
SortOrder, SortOrder,
SortType, SortType,
Table, Table,
TableSchema, TableSchema,
User, User,
ViewV2Schema,
} from "@budibase/types" } from "@budibase/types"
import _ from "lodash" import _ from "lodash"
import tk from "timekeeper" import tk from "timekeeper"
@ -64,18 +66,14 @@ describe.each([
let envCleanup: (() => void) | undefined let envCleanup: (() => void) | undefined
let datasource: Datasource | undefined let datasource: Datasource | undefined
let client: Knex | undefined let client: Knex | undefined
let table: Table let tableOrViewId: string
let rows: Row[] let rows: Row[]
async function basicRelationshipTables(type: RelationshipType) { async function basicRelationshipTables(type: RelationshipType) {
const relatedTable = await createTable( const relatedTable = await createTable({
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}, })
generator.guid().substring(0, 10) const tableId = await createTable({
)
table = await createTable(
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
//@ts-ignore - API accepts this structure, will build out rest of definition //@ts-ignore - API accepts this structure, will build out rest of definition
productCat: { productCat: {
@ -83,17 +81,15 @@ describe.each([
relationshipType: type, relationshipType: type,
name: "productCat", name: "productCat",
fieldName: "product", fieldName: "product",
tableId: relatedTable._id!, tableId: relatedTable,
constraints: { constraints: {
type: "array", type: "array",
}, },
}, },
}, })
generator.guid().substring(0, 10)
)
return { return {
relatedTable: await config.api.table.get(relatedTable._id!), relatedTable: await config.api.table.get(relatedTable),
table, tableId,
} }
} }
@ -136,32 +132,69 @@ describe.each([
} }
}) })
async function createTable(schema: TableSchema, name?: string) { async function createTable(schema: TableSchema) {
return await config.api.table.save( const table = await config.api.table.save(
tableForDatasource(datasource, { schema, name }) tableForDatasource(datasource, { schema })
) )
return table._id!
}
async function createView(tableId: string, schema: ViewV2Schema) {
const view = await config.api.viewV2.create({
tableId: tableId,
name: generator.guid(),
schema,
})
return view.id
} }
async function createRows(arr: Record<string, any>[]) { async function createRows(arr: Record<string, any>[]) {
// Shuffling to avoid false positives given a fixed order // Shuffling to avoid false positives given a fixed order
await config.api.row.bulkImport(table._id!, { for (const row of _.shuffle(arr)) {
rows: _.shuffle(arr), await config.api.row.save(tableOrViewId, row)
}) }
rows = await config.api.row.fetch(table._id!) rows = await config.api.row.fetch(tableOrViewId)
}
describe.each([
["table", createTable],
[
"view",
async (schema: TableSchema) => {
const tableId = await createTable(schema)
const viewId = await createView(
tableId,
Object.keys(schema).reduce<ViewV2Schema>((viewSchema, fieldName) => {
const field = schema[fieldName]
viewSchema[fieldName] = {
visible: field.visible ?? true,
readonly: false,
}
return viewSchema
}, {})
)
return viewId
},
],
])("from %s", (sourceType, createTableOrView) => {
const isView = sourceType === "view"
if (isView && isLucene) {
// Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests
return
} }
class SearchAssertion { class SearchAssertion {
constructor(private readonly query: RowSearchParams) {} constructor(private readonly query: SearchRowRequest) {}
private async performSearch(): Promise<SearchResponse<Row>> { private async performSearch(): Promise<SearchResponse<Row>> {
if (isInMemory) { if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), this.query) return dataFilters.search(_.cloneDeep(rows), {
...this.query,
tableId: tableOrViewId,
})
} else { } else {
const sourceId = this.query.viewId || this.query.tableId return config.api.row.search(tableOrViewId, this.query)
if (!sourceId) {
throw new Error("No source ID provided")
}
return config.api.row.search(sourceId, this.query)
} }
} }
@ -331,8 +364,8 @@ describe.each([
} }
} }
function expectSearch(query: Omit<RowSearchParams, "tableId">) { function expectSearch(query: SearchRowRequest) {
return new SearchAssertion({ ...query, tableId: table._id! }) return new SearchAssertion(query)
} }
function expectQuery(query: SearchFilters) { function expectQuery(query: SearchFilters) {
@ -341,7 +374,7 @@ describe.each([
describe("boolean", () => { describe("boolean", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
}) })
await createRows([{ isTrue: true }, { isTrue: false }]) await createRows([{ isTrue: true }, { isTrue: false }])
@ -429,38 +462,35 @@ describe.each([
{ name: "serverDate", appointment: serverTime.toISOString() }, { name: "serverDate", appointment: serverTime.toISOString() },
{ {
name: "single user, session user", name: "single user, session user",
single_user: JSON.stringify(currentUser), single_user: currentUser,
}, },
{ {
name: "single user", name: "single user",
single_user: JSON.stringify(globalUsers[0]), single_user: globalUsers[0],
}, },
{ {
name: "deprecated single user, session user", name: "deprecated single user, session user",
deprecated_single_user: JSON.stringify([currentUser]), deprecated_single_user: [currentUser],
}, },
{ {
name: "deprecated single user", name: "deprecated single user",
deprecated_single_user: JSON.stringify([globalUsers[0]]), deprecated_single_user: [globalUsers[0]],
}, },
{ {
name: "multi user", name: "multi user",
multi_user: JSON.stringify(globalUsers), multi_user: globalUsers,
}, },
{ {
name: "multi user with session user", name: "multi user with session user",
multi_user: JSON.stringify([...globalUsers, currentUser]), multi_user: [...globalUsers, currentUser],
}, },
{ {
name: "deprecated multi user", name: "deprecated multi user",
deprecated_multi_user: JSON.stringify(globalUsers), deprecated_multi_user: globalUsers,
}, },
{ {
name: "deprecated multi user with session user", name: "deprecated multi user with session user",
deprecated_multi_user: JSON.stringify([ deprecated_multi_user: [...globalUsers, currentUser],
...globalUsers,
currentUser,
]),
}, },
] ]
} }
@ -482,7 +512,7 @@ describe.each([
}) })
) )
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
appointment: { name: "appointment", type: FieldType.DATETIME }, appointment: { name: "appointment", type: FieldType.DATETIME },
single_user: { single_user: {
@ -632,9 +662,11 @@ describe.each([
}) })
it("should match the session user id in a multi user field", async () => { it("should match the session user id in a multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => { const allUsers = [...globalUsers, config.getUser()].map(
(user: any) => {
return { _id: user._id } return { _id: user._id }
}) }
)
await expectQuery({ await expectQuery({
contains: { multi_user: ["{{ [user]._id }}"] }, contains: { multi_user: ["{{ [user]._id }}"] },
@ -647,9 +679,11 @@ describe.each([
}) })
it("should match the session user id in a deprecated multi user field", async () => { it("should match the session user id in a deprecated multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => { const allUsers = [...globalUsers, config.getUser()].map(
(user: any) => {
return { _id: user._id } return { _id: user._id }
}) }
)
await expectQuery({ await expectQuery({
contains: { deprecated_multi_user: ["{{ [user]._id }}"] }, contains: { deprecated_multi_user: ["{{ [user]._id }}"] },
@ -764,7 +798,7 @@ describe.each([
describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}) })
await createRows([{ name: "foo" }, { name: "bar" }]) await createRows([{ name: "foo" }, { name: "bar" }])
@ -791,6 +825,8 @@ describe.each([
}).toContainExactly([{ name: "foo" }, { name: "bar" }]) }).toContainExactly([{ name: "foo" }, { name: "bar" }])
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { it("should return nothing if onEmptyFilter is RETURN_NONE", async () => {
await expectQuery({ await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE, onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@ -891,6 +927,8 @@ describe.each([
}).toContainExactly([{ name: "foo" }, { name: "bar" }]) }).toContainExactly([{ name: "foo" }, { name: "bar" }])
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => { it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
await expectQuery({ await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE, onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@ -1055,7 +1093,7 @@ describe.each([
datasourceId: datasource!._id!, datasourceId: datasource!._id!,
}) })
table = resp.datasource.entities![tableName] tableOrViewId = resp.datasource.entities![tableName]._id!
await createRows([{ name: "foo" }, { name: "bar" }]) await createRows([{ name: "foo" }, { name: "bar" }])
}) })
@ -1079,7 +1117,7 @@ describe.each([
describe("numbers", () => { describe("numbers", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER }, age: { name: "age", type: FieldType.NUMBER },
}) })
await createRows([{ age: 1 }, { age: 10 }]) await createRows([{ age: 1 }, { age: 10 }])
@ -1087,7 +1125,9 @@ describe.each([
describe("equal", () => { describe("equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]) await expectQuery({ equal: { age: 1 } }).toContainExactly([
{ age: 1 },
])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -1252,7 +1292,7 @@ describe.each([
const JAN_10TH = "2020-01-10T00:00:00.000Z" const JAN_10TH = "2020-01-10T00:00:00.000Z"
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
dob: { name: "dob", type: FieldType.DATETIME }, dob: { name: "dob", type: FieldType.DATETIME },
}) })
@ -1324,25 +1364,33 @@ describe.each([
it("greater than equal to", async () => { it("greater than equal to", async () => {
await expectQuery({ await expectQuery({
range: { dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() } }, range: {
dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_10TH }]) }).toContainExactly([{ dob: JAN_10TH }])
}) })
it("greater than", async () => { it("greater than", async () => {
await expectQuery({ await expectQuery({
range: { dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() } }, range: {
dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_10TH }]) }).toContainExactly([{ dob: JAN_10TH }])
}) })
it("less than equal to", async () => { it("less than equal to", async () => {
await expectQuery({ await expectQuery({
range: { dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() } }, range: {
dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_1ST }]) }).toContainExactly([{ dob: JAN_1ST }])
}) })
it("less than", async () => { it("less than", async () => {
await expectQuery({ await expectQuery({
range: { dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() } }, range: {
dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_1ST }]) }).toContainExactly([{ dob: JAN_1ST }])
}) })
}) })
@ -1399,7 +1447,7 @@ describe.each([
const NULL_TIME__ID = `null_time__id` const NULL_TIME__ID = `null_time__id`
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
timeid: { name: "timeid", type: FieldType.STRING }, timeid: { name: "timeid", type: FieldType.STRING },
time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
}) })
@ -1560,7 +1608,7 @@ describe.each([
describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
numbers: { numbers: {
name: "numbers", name: "numbers",
type: FieldType.ARRAY, type: FieldType.ARRAY,
@ -1575,9 +1623,9 @@ describe.each([
describe("contains", () => { describe("contains", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ await expectQuery({
{ numbers: ["one", "two"] }, contains: { numbers: ["one"] },
]) }).toContainExactly([{ numbers: ["one", "two"] }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -1657,7 +1705,7 @@ describe.each([
let BIG = "9223372036854775807" let BIG = "9223372036854775807"
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
num: { name: "num", type: FieldType.BIGINT }, num: { name: "num", type: FieldType.BIGINT },
}) })
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@ -1762,7 +1810,7 @@ describe.each([
isInternal && isInternal &&
describe("auto", () => { describe("auto", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
auto: { auto: {
name: "auto", name: "auto",
type: FieldType.AUTO, type: FieldType.AUTO,
@ -1913,9 +1961,9 @@ describe.each([
// to specify an order for pagination to work. // to specify an order for pagination to work.
it("is stable without a sort specified", async () => { it("is stable without a sort specified", async () => {
let { rows: fullRowList } = await config.api.row.search( let { rows: fullRowList } = await config.api.row.search(
table._id!, tableOrViewId,
{ {
tableId: table._id!, tableId: tableOrViewId,
query: {}, query: {},
} }
) )
@ -1925,8 +1973,8 @@ describe.each([
hasNextPage: boolean | undefined = true, hasNextPage: boolean | undefined = true,
rowCount: number = 0 rowCount: number = 0
do { do {
const response = await config.api.row.search(table._id!, { const response = await config.api.row.search(tableOrViewId, {
tableId: table._id!, tableId: tableOrViewId,
limit: 1, limit: 1,
paginate: true, paginate: true,
query: {}, query: {},
@ -1949,8 +1997,8 @@ describe.each([
// eslint-disable-next-line no-constant-condition // eslint-disable-next-line no-constant-condition
while (true) { while (true) {
const response = await config.api.row.search(table._id!, { const response = await config.api.row.search(tableOrViewId, {
tableId: table._id!, tableId: tableOrViewId,
limit: 3, limit: 3,
query: {}, query: {},
bookmark, bookmark,
@ -1973,7 +2021,7 @@ describe.each([
describe("field name 1:name", () => { describe("field name 1:name", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
"1:name": { name: "1:name", type: FieldType.STRING }, "1:name": { name: "1:name", type: FieldType.STRING },
}) })
await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
@ -1993,8 +2041,7 @@ describe.each([
isSql && isSql &&
describe("related formulas", () => { describe("related formulas", () => {
beforeAll(async () => { beforeAll(async () => {
const arrayTable = await createTable( const arrayTable = await createTable({
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
array: { array: {
name: "array", name: "array",
@ -2004,17 +2051,14 @@ describe.each([
inclusion: ["option 1", "option 2"], inclusion: ["option 1", "option 2"],
}, },
}, },
}, })
"array" tableOrViewId = await createTableOrView({
)
table = await createTable(
{
relationship: { relationship: {
type: FieldType.LINK, type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE, relationshipType: RelationshipType.MANY_TO_ONE,
name: "relationship", name: "relationship",
fieldName: "relate", fieldName: "relate",
tableId: arrayTable._id!, tableId: arrayTable,
constraints: { constraints: {
type: "array", type: "array",
}, },
@ -2026,21 +2070,19 @@ describe.each([
`let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
), ),
}, },
}, })
"main"
)
const arrayRows = await Promise.all([ const arrayRows = await Promise.all([
config.api.row.save(arrayTable._id!, { config.api.row.save(arrayTable, {
name: "foo", name: "foo",
array: ["option 1"], array: ["option 1"],
}), }),
config.api.row.save(arrayTable._id!, { config.api.row.save(arrayTable, {
name: "bar", name: "bar",
array: ["option 2"], array: ["option 2"],
}), }),
]) ])
await Promise.all([ await Promise.all([
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
relationship: [arrayRows[0]._id, arrayRows[1]._id], relationship: [arrayRows[0]._id, arrayRows[1]._id],
}), }),
]) ])
@ -2059,7 +2101,7 @@ describe.each([
user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user1 = await config.createUser({ _id: `us_${utils.newid()}` })
user2 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` })
table = await createTable({ tableOrViewId = await createTableOrView({
user: { user: {
name: "user", name: "user",
type: FieldType.BB_REFERENCE_SINGLE, type: FieldType.BB_REFERENCE_SINGLE,
@ -2067,11 +2109,7 @@ describe.each([
}, },
}) })
await createRows([ await createRows([{ user: user1 }, { user: user2 }, { user: null }])
{ user: JSON.stringify(user1) },
{ user: JSON.stringify(user2) },
{ user: null },
])
}) })
describe("equal", () => { describe("equal", () => {
@ -2088,18 +2126,15 @@ describe.each([
describe("notEqual", () => { describe("notEqual", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ await expectQuery({ notEqual: { user: user1._id } }).toContainExactly(
{ user: { _id: user2._id } }, [{ user: { _id: user2._id } }, {}]
{}, )
])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly(
{ user: { _id: user1._id } }, [{ user: { _id: user1._id } }, { user: { _id: user2._id } }, {}]
{ user: { _id: user2._id } }, )
{},
])
}) })
}) })
@ -2139,7 +2174,7 @@ describe.each([
user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user1 = await config.createUser({ _id: `us_${utils.newid()}` })
user2 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` })
table = await createTable({ tableOrViewId = await createTableOrView({
users: { users: {
name: "users", name: "users",
type: FieldType.BB_REFERENCE, type: FieldType.BB_REFERENCE,
@ -2153,10 +2188,10 @@ describe.each([
}) })
await createRows([ await createRows([
{ number: 1, users: JSON.stringify([user1]) }, { number: 1, users: [user1] },
{ number: 2, users: JSON.stringify([user2]) }, { number: 2, users: [user2] },
{ number: 3, users: JSON.stringify([user1, user2]) }, { number: 3, users: [user1, user2] },
{ number: 4, users: JSON.stringify([]) }, { number: 4, users: [] },
]) ])
}) })
@ -2182,7 +2217,9 @@ describe.each([
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing() await expectQuery({
contains: { users: ["us_none"] },
}).toFindNothing()
}) })
}) })
@ -2249,9 +2286,10 @@ describe.each([
let productCategoryTable: Table, productCatRows: Row[] let productCategoryTable: Table, productCatRows: Row[]
beforeAll(async () => { beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables( const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.ONE_TO_MANY RelationshipType.ONE_TO_MANY
) )
tableOrViewId = tableId
productCategoryTable = relatedTable productCategoryTable = relatedTable
productCatRows = await Promise.all([ productCatRows = await Promise.all([
@ -2260,15 +2298,15 @@ describe.each([
]) ])
await Promise.all([ await Promise.all([
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
productCat: [productCatRows[0]._id], productCat: [productCatRows[0]._id],
}), }),
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "bar", name: "bar",
productCat: [productCatRows[1]._id], productCat: [productCatRows[1]._id],
}), }),
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "baz", name: "baz",
productCat: [], productCat: [],
}), }),
@ -2301,10 +2339,11 @@ describe.each([
isSql && isSql &&
describe("big relations", () => { describe("big relations", () => {
beforeAll(async () => { beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables( const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.MANY_TO_ONE RelationshipType.MANY_TO_ONE
) )
const mainRow = await config.api.row.save(table._id!, { tableOrViewId = tableId
const mainRow = await config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
}) })
for (let i = 0; i < 11; i++) { for (let i = 0; i < 11; i++) {
@ -2329,45 +2368,42 @@ describe.each([
}) })
;(isSqs || isLucene) && ;(isSqs || isLucene) &&
describe("relations to same table", () => { describe("relations to same table", () => {
let relatedTable: Table, relatedRows: Row[] let relatedTable: string, relatedRows: Row[]
beforeAll(async () => { beforeAll(async () => {
relatedTable = await createTable( relatedTable = await createTable({
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}, })
"productCategory" tableOrViewId = await createTableOrView({
)
table = await createTable({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
related1: { related1: {
type: FieldType.LINK, type: FieldType.LINK,
name: "related1", name: "related1",
fieldName: "main1", fieldName: "main1",
tableId: relatedTable._id!, tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
}, },
related2: { related2: {
type: FieldType.LINK, type: FieldType.LINK,
name: "related2", name: "related2",
fieldName: "main2", fieldName: "main2",
tableId: relatedTable._id!, tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
}, },
}) })
relatedRows = await Promise.all([ relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, { name: "foo" }), config.api.row.save(relatedTable, { name: "foo" }),
config.api.row.save(relatedTable._id!, { name: "bar" }), config.api.row.save(relatedTable, { name: "bar" }),
config.api.row.save(relatedTable._id!, { name: "baz" }), config.api.row.save(relatedTable, { name: "baz" }),
config.api.row.save(relatedTable._id!, { name: "boo" }), config.api.row.save(relatedTable, { name: "boo" }),
]) ])
await Promise.all([ await Promise.all([
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "test", name: "test",
related1: [relatedRows[0]._id!], related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!], related2: [relatedRows[1]._id!],
}), }),
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "test2", name: "test2",
related1: [relatedRows[2]._id!], related1: [relatedRows[2]._id!],
related2: [relatedRows[3]._id!], related2: [relatedRows[3]._id!],
@ -2430,7 +2466,7 @@ describe.each([
isInternal && isInternal &&
describe("no column error backwards compat", () => { describe("no column error backwards compat", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2453,7 +2489,7 @@ describe.each([
!isLucene && !isLucene &&
describe("row counting", () => { describe("row counting", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2488,7 +2524,7 @@ describe.each([
describe("Invalid column definitions", () => { describe("Invalid column definitions", () => {
beforeAll(async () => { beforeAll(async () => {
// need to create an invalid table - means ignoring typescript // need to create an invalid table - means ignoring typescript
table = await createTable({ tableOrViewId = await createTableOrView({
// @ts-ignore // @ts-ignore
invalid: { invalid: {
type: FieldType.STRING, type: FieldType.STRING,
@ -2518,7 +2554,7 @@ describe.each([
"special (%s) case", "special (%s) case",
column => { column => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
[column]: { [column]: {
name: column, name: column,
type: FieldType.STRING, type: FieldType.STRING,
@ -2543,8 +2579,8 @@ describe.each([
describe("sample data", () => { describe("sample data", () => {
beforeAll(async () => { beforeAll(async () => {
await config.api.application.addSampleData(config.appId!) await config.api.application.addSampleData(config.appId!)
table = DEFAULT_EMPLOYEE_TABLE_SCHEMA tableOrViewId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id!
rows = await config.api.row.fetch(table._id!) rows = await config.api.row.fetch(tableOrViewId)
}) })
it("should be able to search sample data", async () => { it("should be able to search sample data", async () => {
@ -2567,7 +2603,7 @@ describe.each([
const earlyDate = "2024-07-03T10:00:00.000Z", const earlyDate = "2024-07-03T10:00:00.000Z",
laterDate = "2024-07-03T11:00:00.000Z" laterDate = "2024-07-03T11:00:00.000Z"
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
date: { date: {
name: "date", name: "date",
type: FieldType.DATETIME, type: FieldType.DATETIME,
@ -2610,7 +2646,7 @@ describe.each([
"ชื่อผู้ใช้", // Thai for "username" "ชื่อผู้ใช้", // Thai for "username"
])("non-ascii column name: %s", name => { ])("non-ascii column name: %s", name => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
[name]: { [name]: {
name, name,
type: FieldType.STRING, type: FieldType.STRING,
@ -2637,7 +2673,7 @@ describe.each([
isInternal && isInternal &&
describe("space at end of column name", () => { describe("space at end of column name", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
"name ": { "name ": {
name: "name ", name: "name ",
type: FieldType.STRING, type: FieldType.STRING,
@ -2672,7 +2708,7 @@ describe.each([
;(isSqs || isInMemory) && ;(isSqs || isInMemory) &&
describe("space at start of column name", () => { describe("space at start of column name", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
" name": { " name": {
name: " name", name: " name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2703,9 +2739,10 @@ describe.each([
}) })
isSqs && isSqs &&
!isView &&
describe("duplicate columns", () => { describe("duplicate columns", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2713,7 +2750,7 @@ describe.each([
}) })
await context.doInAppContext(config.getAppId(), async () => { await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB() const db = context.getAppDB()
const tableDoc = await db.get<Table>(table._id!) const tableDoc = await db.get<Table>(tableOrViewId)
tableDoc.schema.Name = { tableDoc.schema.Name = {
name: "Name", name: "Name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2747,7 +2784,7 @@ describe.each([
type: FieldType.STRING, type: FieldType.STRING,
}, },
}) })
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2756,15 +2793,15 @@ describe.each([
name: "rel", name: "rel",
type: FieldType.LINK, type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
tableId: toRelateTable._id!, tableId: toRelateTable,
fieldName: "rel", fieldName: "rel",
}, },
}) })
const [row1, row2] = await Promise.all([ const [row1, row2] = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "tag 1" }), config.api.row.save(toRelateTable, { name: "tag 1" }),
config.api.row.save(toRelateTable._id!, { name: "tag 2" }), config.api.row.save(toRelateTable, { name: "tag 2" }),
]) ])
row = await config.api.row.save(table._id!, { row = await config.api.row.save(tableOrViewId, {
name: "product 1", name: "product 1",
rel: [row1._id, row2._id], rel: [row1._id, row2._id],
}) })
@ -2783,7 +2820,7 @@ describe.each([
!isInternal && !isInternal &&
describe("search by composite key", () => { describe("search by composite key", () => {
beforeAll(async () => { beforeAll(async () => {
table = await config.api.table.save( const table = await config.api.table.save(
tableForDatasource(datasource, { tableForDatasource(datasource, {
schema: { schema: {
idColumn1: { idColumn1: {
@ -2798,6 +2835,7 @@ describe.each([
primary: ["idColumn1", "idColumn2"], primary: ["idColumn1", "idColumn2"],
}) })
) )
tableOrViewId = table._id!
await createRows([{ idColumn1: 1, idColumn2: 2 }]) await createRows([{ idColumn1: 1, idColumn2: 2 }])
}) })
@ -2819,15 +2857,13 @@ describe.each([
isSql && isSql &&
describe("primaryDisplay", () => { describe("primaryDisplay", () => {
beforeAll(async () => { beforeAll(async () => {
let toRelateTable = await createTable({ let toRelateTableId = await createTable({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
}, },
}) })
table = await config.api.table.save( tableOrViewId = await createTableOrView({
tableForDatasource(datasource, {
schema: {
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2836,33 +2872,30 @@ describe.each([
name: "link", name: "link",
type: FieldType.LINK, type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE, relationshipType: RelationshipType.MANY_TO_ONE,
tableId: toRelateTable._id!, tableId: toRelateTableId,
fieldName: "link", fieldName: "link",
}, },
},
}) })
)
toRelateTable = await config.api.table.get(toRelateTable._id!) const toRelateTable = await config.api.table.get(toRelateTableId)
await config.api.table.save({ await config.api.table.save({
...toRelateTable, ...toRelateTable,
primaryDisplay: "link", primaryDisplay: "link",
}) })
const relatedRows = await Promise.all([ const relatedRows = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "test" }), config.api.row.save(toRelateTable._id!, { name: "related" }),
]) ])
await Promise.all([ await config.api.row.save(tableOrViewId, {
config.api.row.save(table._id!, {
name: "test", name: "test",
link: relatedRows.map(row => row._id), link: relatedRows.map(row => row._id),
}), })
])
}) })
it("should be able to query, primary display on related table shouldn't be used", async () => { it("should be able to query, primary display on related table shouldn't be used", async () => {
// this test makes sure that if a relationship has been specified as the primary display on a table // this test makes sure that if a relationship has been specified as the primary display on a table
// it is ignored and another column is used instead // it is ignored and another column is used instead
await expectQuery({}).toContain([ await expectQuery({}).toContain([
{ name: "test", link: [{ primaryDisplay: "test" }] }, { name: "test", link: [{ primaryDisplay: "related" }] },
]) ])
}) })
}) })
@ -2870,7 +2903,7 @@ describe.each([
!isLucene && !isLucene &&
describe("$and", () => { describe("$and", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER }, age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}) })
@ -2945,7 +2978,10 @@ describe.each([
await expect( await expect(
expectQuery({ expectQuery({
$and: { $and: {
conditions: [{ equal: { age: 10 } }, "invalidCondition" as any], conditions: [
{ equal: { age: 10 } },
"invalidCondition" as any,
],
}, },
}).toFindNothing() }).toFindNothing()
).rejects.toThrow( ).rejects.toThrow(
@ -2973,6 +3009,8 @@ describe.each([
) )
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("returns no rows when onEmptyFilter set to none", async () => { it("returns no rows when onEmptyFilter set to none", async () => {
await expectSearch({ await expectSearch({
query: { query: {
@ -2999,7 +3037,7 @@ describe.each([
!isLucene && !isLucene &&
describe("$or", () => { describe("$or", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER }, age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}) })
@ -3123,6 +3161,8 @@ describe.each([
}).toContainExactly([{ age: 1, name: "Jane" }]) }).toContainExactly([{ age: 1, name: "Jane" }])
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("returns no rows when onEmptyFilter set to none", async () => { it("returns no rows when onEmptyFilter set to none", async () => {
await expectSearch({ await expectSearch({
query: { query: {
@ -3159,20 +3199,20 @@ describe.each([
row[name] = i row[name] = i
} }
const relatedTable = await createTable(relatedSchema) const relatedTable = await createTable(relatedSchema)
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
related1: { related1: {
type: FieldType.LINK, type: FieldType.LINK,
name: "related1", name: "related1",
fieldName: "main1", fieldName: "main1",
tableId: relatedTable._id!, tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
}, },
}) })
relatedRows = await Promise.all([ relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, row), config.api.row.save(relatedTable, row),
]) ])
await config.api.row.save(table._id!, { await config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
related1: [relatedRows[0]._id], related1: [relatedRows[0]._id],
}) })
@ -3188,3 +3228,4 @@ describe.each([
}) })
}) })
}) })
})

337
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,9 +22,10 @@ import {
RelationshipType, RelationshipType,
TableSchema, TableSchema,
RenameColumn, RenameColumn,
ViewFieldMetadata,
FeatureFlag, FeatureFlag,
BBReferenceFieldSubType, BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
} from "@budibase/types" } from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests" import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -154,7 +155,7 @@ describe.each([
}) })
it("can persist views with all fields", async () => { it("can persist views with all fields", async () => {
const newView: Required<CreateViewRequest> = { const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
name: generator.name(), name: generator.name(),
tableId: table._id!, tableId: table._id!,
primaryDisplay: "id", primaryDisplay: "id",
@ -176,6 +177,9 @@ describe.each([
visible: true, visible: true,
}, },
}, },
uiMetadata: {
foo: "bar",
},
} }
const res = await config.api.viewV2.create(newView) const res = await config.api.viewV2.create(newView)
@ -540,6 +544,33 @@ describe.each([
status: 201, status: 201,
}) })
}) })
it("can create a view with calculation fields", async () => {
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
})
expect(Object.keys(view.schema!)).toHaveLength(1)
let sum = view.schema!.sum as NumericCalculationFieldMetadata
expect(sum).toBeDefined()
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
view = await config.api.viewV2.get(view.id)
sum = view.schema!.sum as NumericCalculationFieldMetadata
expect(sum).toBeDefined()
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
})
}) })
describe("update", () => { describe("update", () => {
@ -584,7 +615,7 @@ describe.each([
it("can update all fields", async () => { it("can update all fields", async () => {
const tableId = table._id! const tableId = table._id!
const updatedData: Required<UpdateViewRequest> = { const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
version: view.version, version: view.version,
id: view.id, id: view.id,
tableId, tableId,
@ -612,6 +643,9 @@ describe.each([
readonly: true, readonly: true,
}, },
}, },
uiMetadata: {
foo: "bar",
},
} }
await config.api.viewV2.update(updatedData) await config.api.viewV2.update(updatedData)
@ -836,6 +870,185 @@ describe.each([
} }
) )
}) })
!isLucene &&
describe("calculation views", () => {
let table: Table
let view: ViewV2
beforeEach(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
country: {
name: "country",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
country: {
visible: true,
},
age: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Steve",
age: 30,
country: "UK",
},
{
name: "Jane",
age: 31,
country: "UK",
},
{
name: "Ruari",
age: 32,
country: "USA",
},
{
name: "Alice",
age: 33,
country: "USA",
},
],
})
})
it("returns the expected rows prior to modification", async () => {
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
},
{
country: "UK",
age: 61,
},
])
)
})
it("can remove a group by field", async () => {
delete view.schema!.country
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows).toEqual(
expect.arrayContaining([
{
age: 126,
},
])
)
})
it("can remove a calculation field", async () => {
delete view.schema!.age
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(4)
// Because the removal of the calculation field actually makes this
// no longer a calculation view, these rows will now have _id and
// _rev fields.
expect(rows).toEqual(
expect.arrayContaining([
expect.objectContaining({ country: "UK" }),
expect.objectContaining({ country: "UK" }),
expect.objectContaining({ country: "USA" }),
expect.objectContaining({ country: "USA" }),
])
)
})
it("can add a new group by field", async () => {
view.schema!.name = { visible: true }
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(4)
expect(rows).toEqual(
expect.arrayContaining([
{
name: "Steve",
age: 30,
country: "UK",
},
{
name: "Jane",
age: 31,
country: "UK",
},
{
name: "Ruari",
age: 32,
country: "USA",
},
{
name: "Alice",
age: 33,
country: "USA",
},
])
)
})
it("can add a new calculation field", async () => {
view.schema!.count = {
visible: true,
calculationType: CalculationType.COUNT,
}
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
count: 2,
},
{
country: "UK",
age: 61,
count: 2,
},
])
)
})
})
}) })
describe("delete", () => { describe("delete", () => {
@ -1152,10 +1365,7 @@ describe.each([
return table return table
} }
const createView = async ( const createView = async (tableId: string, schema: ViewV2Schema) =>
tableId: string,
schema: Record<string, ViewFieldMetadata>
) =>
await config.api.viewV2.create({ await config.api.viewV2.create({
name: generator.guid(), name: generator.guid(),
tableId, tableId,
@ -2545,6 +2755,119 @@ describe.each([
expect(actual).toEqual(expected) expect(actual).toEqual(expected)
} }
}) })
it("should be able to do a COUNT(DISTINCT)", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "name",
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "John",
},
{
name: "John",
},
{
name: "Sue",
},
],
})
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows[0].count).toEqual(2)
})
it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "does not exist oh no",
},
},
},
{
status: 400,
body: {
message:
'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema',
},
}
)
})
})
!isLucene &&
it("should not need required fields to be present", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
await Promise.all([
config.api.row.save(table._id!, { name: "Steve", age: 30 }),
config.api.row.save(table._id!, { name: "Jane", age: 31 }),
])
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].sum).toEqual(61)
}) })
}) })

4
packages/server/src/db/linkedRows/index.ts
View File

@ -23,8 +23,8 @@ import {
Row, Row,
Table, Table,
TableSchema, TableSchema,
ViewFieldMetadata,
ViewV2, ViewV2,
ViewV2Schema,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../sdk" import sdk from "../../sdk"
import { helpers } from "@budibase/shared-core" import { helpers } from "@budibase/shared-core"
@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
FeatureFlag.ENRICHED_RELATIONSHIPS FeatureFlag.ENRICHED_RELATIONSHIPS
) )
let viewSchema: Record<string, ViewFieldMetadata> = {} let viewSchema: ViewV2Schema = {}
if (sdk.views.isView(source)) { if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) { if (helpers.views.isCalculationView(source)) {
return enriched return enriched

4
packages/server/src/jsRunner/index.ts
View File

@ -1,7 +1,7 @@
import { serializeError } from "serialize-error" import { serializeError } from "serialize-error"
import env from "../environment" import env from "../environment"
import { import {
JsErrorTimeout, JsTimeoutError,
setJSRunner, setJSRunner,
setOnErrorLog, setOnErrorLog,
} from "@budibase/string-templates" } from "@budibase/string-templates"
@ -40,7 +40,7 @@ export function init() {
return vm.withContext(rest, () => vm.execute(js)) return vm.withContext(rest, () => vm.execute(js))
} catch (error: any) { } catch (error: any) {
if (error.message === "Script execution timed out.") { if (error.message === "Script execution timed out.") {
throw new JsErrorTimeout() throw new JsTimeoutError()
} }
throw error throw error
} }

5
packages/server/src/jsRunner/tests/jsRunner.spec.ts
View File

@ -41,10 +41,11 @@ describe("jsRunner (using isolated-vm)", () => {
}) })
it("should prevent sandbox escape", async () => { it("should prevent sandbox escape", async () => {
const output = await processJS( expect(
await processJS(
`return this.constructor.constructor("return process.env")()` `return this.constructor.constructor("return process.env")()`
) )
expect(output).toBe("Error while executing JS") ).toEqual("ReferenceError: process is not defined")
}) })
describe("helpers", () => { describe("helpers", () => {

25
packages/server/src/jsRunner/vm/isolated-vm.ts
View File

@ -7,14 +7,12 @@ import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles" import { BundleType, loadBundle } from "../bundles"
import { Snippet, VM } from "@budibase/types" import { Snippet, VM } from "@budibase/types"
import { iifeWrapper } from "@budibase/string-templates" import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
import environment from "../../environment" import environment from "../../environment"
class ExecutionTimeoutError extends Error { export class JsRequestTimeoutError extends Error {
constructor(message: string) { static code = "JS_REQUEST_TIMEOUT_ERROR"
super(message) code = JsRequestTimeoutError.code
this.name = "ExecutionTimeoutError"
}
} }
export class IsolatedVM implements VM { export class IsolatedVM implements VM {
@ -29,6 +27,7 @@ export class IsolatedVM implements VM {
private readonly resultKey = "results" private readonly resultKey = "results"
private runResultKey: string private runResultKey: string
private runErrorKey: string
constructor({ constructor({
memoryLimit, memoryLimit,
@ -47,6 +46,7 @@ export class IsolatedVM implements VM {
this.jail.setSync("global", this.jail.derefInto()) this.jail.setSync("global", this.jail.derefInto())
this.runResultKey = crypto.randomUUID() this.runResultKey = crypto.randomUUID()
this.runErrorKey = crypto.randomUUID()
this.addToContext({ this.addToContext({
[this.resultKey]: { [this.runResultKey]: "" }, [this.resultKey]: { [this.runResultKey]: "" },
}) })
@ -210,13 +210,19 @@ export class IsolatedVM implements VM {
if (this.isolateAccumulatedTimeout) { if (this.isolateAccumulatedTimeout) {
const cpuMs = Number(this.isolate.cpuTime) / 1e6 const cpuMs = Number(this.isolate.cpuTime) / 1e6
if (cpuMs > this.isolateAccumulatedTimeout) { if (cpuMs > this.isolateAccumulatedTimeout) {
throw new ExecutionTimeoutError( throw new JsRequestTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)` `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
) )
} }
} }
code = `results['${this.runResultKey}']=${this.codeWrapper(code)}` code = `
try {
results['${this.runResultKey}']=${this.codeWrapper(code)}
} catch (e) {
results['${this.runErrorKey}']=e
}
`
const script = this.isolate.compileScriptSync(code) const script = this.isolate.compileScriptSync(code)
@ -227,6 +233,9 @@ export class IsolatedVM implements VM {
// We can't rely on the script run result as it will not work for non-transferable values // We can't rely on the script run result as it will not work for non-transferable values
const result = this.getFromContext(this.resultKey) const result = this.getFromContext(this.resultKey)
if (result[this.runErrorKey]) {
throw new UserScriptError(result[this.runErrorKey])
}
return result[this.runResultKey] return result[this.runResultKey]
} }

21
packages/server/src/sdk/app/rowActions.ts
View File

@ -75,7 +75,7 @@ export async function create(tableId: string, rowAction: { name: string }) {
name: action.name, name: action.name,
automationId: automation._id!, automationId: automation._id!,
permissions: { permissions: {
table: { runAllowed: false }, table: { runAllowed: true },
views: {}, views: {},
}, },
} }
@ -163,6 +163,23 @@ async function guardView(tableId: string, viewId: string) {
} }
} }
export async function setTablePermission(tableId: string, rowActionId: string) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = true
return actionsDoc
})
}
export async function unsetTablePermission(
tableId: string,
rowActionId: string
) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = false
return actionsDoc
})
}
export async function setViewPermission( export async function setViewPermission(
tableId: string, tableId: string,
rowActionId: string, rowActionId: string,
@ -220,6 +237,8 @@ export async function run(tableId: any, rowActionId: any, rowId: string) {
automation, automation,
{ {
fields: { fields: {
id: row._id,
revision: row._rev,
row, row,
table, table,
}, },

50
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,12 +1,10 @@
import { import {
EmptyFilterOption, EmptyFilterOption,
LegacyFilter,
LogicalOperator, LogicalOperator,
Row, Row,
RowSearchParams, RowSearchParams,
SearchFilter,
SearchFilterGroup,
SearchFilterKey, SearchFilterKey,
SearchFilters,
SearchResponse, SearchResponse,
SortOrder, SortOrder,
Table, Table,
@ -63,7 +61,6 @@ export async function search(
if (options.viewId) { if (options.viewId) {
source = await sdk.views.get(options.viewId) source = await sdk.views.get(options.viewId)
table = await sdk.views.getTable(source) table = await sdk.views.getTable(source)
options = searchInputMapping(table, options)
} else if (options.tableId) { } else if (options.tableId) {
source = await sdk.tables.getTable(options.tableId) source = await sdk.tables.getTable(options.tableId)
table = source table = source
@ -76,7 +73,7 @@ export async function search(
if (options.query) { if (options.query) {
const visibleFields = ( const visibleFields = (
options.fields || Object.keys(table.schema) options.fields || Object.keys(table.schema)
).filter(field => table.schema[field].visible !== false) ).filter(field => table.schema[field]?.visible !== false)
const queryableFields = await getQueryableFields(table, visibleFields) const queryableFields = await getQueryableFields(table, visibleFields)
options.query = removeInvalidFilters(options.query, queryableFields) options.query = removeInvalidFilters(options.query, queryableFields)
@ -84,40 +81,59 @@ export async function search(
options.query = {} options.query = {}
} }
// need to make sure filters in correct shape before checking for view
options = searchInputMapping(table, options)
if (options.viewId) { if (options.viewId) {
const view = await sdk.views.get(options.viewId) // Delete extraneous search params that cannot be overridden
delete options.query.onEmptyFilter
const view = source as ViewV2
// Enrich saved query with ephemeral query params. // Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as // We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access. // that could let users find rows they should not be allowed to access.
let viewQuery = dataFilters.buildQuery(view.query || []) let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
delete viewQuery?.onEmptyFilter
if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { const sqsEnabled = await features.flags.isEnabled("SQS")
// Lucene does not accept conditional filters, so we need to keep the old logic const supportsLogicalOperators =
const query: SearchFilters = viewQuery || {} isExternalTableID(view.tableId) || sqsEnabled
const viewFilters = view.query as SearchFilter[] if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
delete options.query.onEmptyFilter
// Extract existing fields // Extract existing fields
const existingFields = const existingFields =
viewFilters queryFilters
?.filter(filter => filter.field) ?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || [] .map(filter => db.removeKeyNumbering(filter.field)) || []
viewQuery ??= {}
// Carry over filters for unused fields // Carry over filters for unused fields
Object.keys(options.query || {}).forEach(key => { Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator> const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => { Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) { if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = options.query[operator]![field] viewQuery![operator]![field] = options.query[operator]![field]
} }
}) })
}) })
options.query = query options.query = viewQuery
} else { } else {
const conditions = viewQuery ? [viewQuery] : []
options.query = { options.query = {
$and: { $and: {
conditions: [viewQuery as SearchFilterGroup, options.query], conditions: [...conditions, options.query],
}, },
} }
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
} }
} }
@ -146,8 +162,6 @@ export async function search(
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
} }
options = searchInputMapping(table, options)
let result: SearchResponse<Row> let result: SearchResponse<Row>
if (isExternalTable) { if (isExternalTable) {
span?.addTags({ searchType: "external" }) span?.addTags({ searchType: "external" })

17
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -1,5 +1,6 @@
import { import {
Aggregation, Aggregation,
CalculationType,
Datasource, Datasource,
DocumentType, DocumentType,
FieldType, FieldType,
@ -369,6 +370,21 @@ export async function search(
continue continue
} }
if (field.calculationType === CalculationType.COUNT) {
if ("distinct" in field && field.distinct) {
aggregations.push({
name: key,
distinct: true,
field: mapToUserColumn(field.field),
calculationType: field.calculationType,
})
} else {
aggregations.push({
name: key,
calculationType: field.calculationType,
})
}
} else {
aggregations.push({ aggregations.push({
name: key, name: key,
field: mapToUserColumn(field.field), field: mapToUserColumn(field.field),
@ -376,6 +392,7 @@ export async function search(
}) })
} }
} }
}
const request: QueryJson = { const request: QueryJson = {
endpoint: { endpoint: {

2
packages/server/src/sdk/app/rows/search/utils.ts
View File

@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
} }
return dataFilters.recurseLogicalOperators(filters, checkFilters) return dataFilters.recurseLogicalOperators(filters, checkFilters)
} }
if (options.query) {
options.query = checkFilters(options.query) options.query = checkFilters(options.query)
}
return options return options
} }

43
packages/server/src/sdk/app/views/index.ts
View File

@ -1,4 +1,5 @@
import { import {
CalculationType,
FieldType, FieldType,
PermissionLevel, PermissionLevel,
RelationSchemaField, RelationSchemaField,
@ -63,6 +64,15 @@ async function guardCalculationViewSchema(
const calculationFields = helpers.views.calculationFields(view) const calculationFields = helpers.views.calculationFields(view)
for (const calculationFieldName of Object.keys(calculationFields)) { for (const calculationFieldName of Object.keys(calculationFields)) {
const schema = calculationFields[calculationFieldName] const schema = calculationFields[calculationFieldName]
const isCount = schema.calculationType === CalculationType.COUNT
const isDistinct = isCount && "distinct" in schema && schema.distinct
// Count fields that aren't distinct don't need to reference another field,
// so we don't validate it.
if (isCount && !isDistinct) {
continue
}
const targetSchema = table.schema[schema.field] const targetSchema = table.schema[schema.field]
if (!targetSchema) { if (!targetSchema) {
throw new HTTPError( throw new HTTPError(
@ -71,7 +81,7 @@ async function guardCalculationViewSchema(
) )
} }
if (!helpers.schema.isNumeric(targetSchema)) { if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
throw new HTTPError( throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`, `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
400 400
@ -255,19 +265,12 @@ export async function enrichSchema(
view: ViewV2, view: ViewV2,
tableSchema: TableSchema tableSchema: TableSchema
): Promise<ViewV2Enriched> { ): Promise<ViewV2Enriched> {
const tableCache: Record<string, Table> = {}
async function populateRelTableSchema( async function populateRelTableSchema(
tableId: string, tableId: string,
viewFields: Record<string, RelationSchemaField> viewFields: Record<string, RelationSchemaField>
) { ) {
if (!tableCache[tableId]) { const relTable = await sdk.tables.getTable(tableId)
tableCache[tableId] = await sdk.tables.getTable(tableId)
}
const relTable = tableCache[tableId]
const result: Record<string, ViewV2ColumnEnriched> = {} const result: Record<string, ViewV2ColumnEnriched> = {}
for (const relTableFieldName of Object.keys(relTable.schema)) { for (const relTableFieldName of Object.keys(relTable.schema)) {
const relTableField = relTable.schema[relTableFieldName] const relTableField = relTable.schema[relTableFieldName]
if ( if (
@ -300,15 +303,24 @@ export async function enrichSchema(
const viewSchema = view.schema || {} const viewSchema = view.schema || {}
const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null) const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null)
for (const key of Object.keys(tableSchema).filter(
k => tableSchema[k].visible !== false const visibleSchemaFields = Object.keys(viewSchema).filter(key => {
)) { if (helpers.views.isCalculationField(viewSchema[key])) {
return viewSchema[key].visible !== false
}
return key in tableSchema && tableSchema[key].visible !== false
})
const visibleTableFields = Object.keys(tableSchema).filter(
key => tableSchema[key].visible !== false
)
const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields])
for (const key of visibleFields) {
// if nothing specified in view, then it is not visible // if nothing specified in view, then it is not visible
const ui = viewSchema[key] || { visible: false } const ui = viewSchema[key] || { visible: false }
schema[key] = { schema[key] = {
...tableSchema[key], ...tableSchema[key],
...ui, ...ui,
order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order, order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
columns: undefined, columns: undefined,
} }
@ -320,10 +332,7 @@ export async function enrichSchema(
} }
} }
return { return { ...view, schema }
...view,
schema: schema,
}
} }
export function syncSchema( export function syncSchema(

4
packages/server/src/tests/utilities/api/row.ts
View File

@ -7,11 +7,11 @@ import {
BulkImportRequest, BulkImportRequest,
BulkImportResponse, BulkImportResponse,
SearchRowResponse, SearchRowResponse,
RowSearchParams,
DeleteRows, DeleteRows,
DeleteRow, DeleteRow,
PaginatedSearchRowResponse, PaginatedSearchRowResponse,
RowExportFormat, RowExportFormat,
SearchRowRequest,
} from "@budibase/types" } from "@budibase/types"
import { Expectations, TestAPI } from "./base" import { Expectations, TestAPI } from "./base"
@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
) )
} }
search = async <T extends RowSearchParams>( search = async <T extends SearchRowRequest>(
sourceId: string, sourceId: string,
params?: T, params?: T,
expectations?: Expectations expectations?: Expectations

36
packages/server/src/tests/utilities/api/rowAction.ts
View File

@ -72,6 +72,42 @@ export class RowActionAPI extends TestAPI {
) )
} }
setTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._post<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
unsetTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._delete<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
setViewPermission = async ( setViewPermission = async (
tableId: string, tableId: string,
viewId: string, viewId: string,

21
packages/shared-core/src/filters.ts
View File

@ -3,7 +3,7 @@ import {
BBReferenceFieldSubType, BBReferenceFieldSubType,
FieldType, FieldType,
FormulaType, FormulaType,
SearchFilter, LegacyFilter,
SearchFilters, SearchFilters,
SearchQueryFields, SearchQueryFields,
ArrayOperator, ArrayOperator,
@ -129,7 +129,7 @@ export function recurseLogicalOperators(
fn: (f: SearchFilters) => SearchFilters fn: (f: SearchFilters) => SearchFilters
) { ) {
for (const logical of LOGICAL_OPERATORS) { for (const logical of LOGICAL_OPERATORS) {
if (filters?.[logical]) { if (filters[logical]) {
filters[logical]!.conditions = filters[logical]!.conditions.map( filters[logical]!.conditions = filters[logical]!.conditions.map(
condition => fn(condition) condition => fn(condition)
) )
@ -165,9 +165,6 @@ export function recurseSearchFilters(
* https://github.com/Budibase/budibase/issues/10118 * https://github.com/Budibase/budibase/issues/10118
*/ */
export const cleanupQuery = (query: SearchFilters) => { export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) { for (let filterField of NoEmptyFilterStrings) {
if (!query[filterField]) { if (!query[filterField]) {
continue continue
@ -313,7 +310,7 @@ export class ColumnSplitter {
* @param filter the builder filter structure * @param filter the builder filter structure
*/ */
const buildCondition = (expression: SearchFilter) => { const buildCondition = (expression: LegacyFilter) => {
// Filter body // Filter body
let query: SearchFilters = { let query: SearchFilters = {
string: {}, string: {},
@ -439,8 +436,13 @@ const buildCondition = (expression: SearchFilter) => {
} }
export const buildQueryLegacy = ( export const buildQueryLegacy = (
filter?: SearchFilterGroup | SearchFilter[] filter?: LegacyFilter[] | SearchFilters
): SearchFilters | undefined => { ): SearchFilters | undefined => {
// this is of type SearchFilters or is undefined
if (!Array.isArray(filter)) {
return filter
}
let query: SearchFilters = { let query: SearchFilters = {
string: {}, string: {},
fuzzy: {}, fuzzy: {},
@ -574,7 +576,7 @@ export const buildQueryLegacy = (
*/ */
export const buildQuery = ( export const buildQuery = (
filter?: SearchFilterGroup | SearchFilter[] filter?: SearchFilterGroup | LegacyFilter[]
): SearchFilters | undefined => { ): SearchFilters | undefined => {
const parsedFilter: SearchFilterGroup | undefined = const parsedFilter: SearchFilterGroup | undefined =
processSearchFilters(filter) processSearchFilters(filter)
@ -596,7 +598,7 @@ export const buildQuery = (
const globalOperator: LogicalOperator = const globalOperator: LogicalOperator =
operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
const coreRequest: SearchFilters = { return {
...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
[globalOperator]: { [globalOperator]: {
conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@ -610,7 +612,6 @@ export const buildQuery = (
}), }),
}, },
} }
return coreRequest
} }
// The frontend can send single values for array fields sometimes, so to handle // The frontend can send single values for array fields sometimes, so to handle

201
packages/shared-core/src/utils.ts
View File

@ -1,5 +1,5 @@
import { import {
SearchFilter, LegacyFilter,
SearchFilterGroup, SearchFilterGroup,
FilterGroupLogicalOperator, FilterGroupLogicalOperator,
SearchFilters, SearchFilters,
@ -9,6 +9,10 @@ import {
import * as Constants from "./constants" import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters" import { removeKeyNumbering } from "./filters"
// an array of keys from filter type to properties that are in the type
// this can then be converted using .fromEntries to an object
type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
export function unreachable( export function unreachable(
value: never, value: never,
message = `No such case in exhaustive switch: ${value}` message = `No such case in exhaustive switch: ${value}`
@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
return result return result
} }
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {SearchFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: SearchFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterWhitelistKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
const migratedSetting: SearchFilterGroup = filters.reduce(
(acc: SearchFilterGroup, filter: SearchFilter) => {
// Sort the properties for easier debugging
const filterEntries = Object.entries(filter)
.sort((a, b) => {
return a[0].localeCompare(b[0])
})
.filter(x => x[1] ?? false)
if (filterEntries.length == 1) {
const [key, value] = filterEntries[0]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
(acc: [string, any][], entry: [string, any]) => {
const [key, value] = entry
if (filterWhitelistKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: SearchFilter = Object.fromEntries(
whiteListedFilterSettings
) as SearchFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
},
defaultCfg
)
return migratedSetting
} else if (!filters?.groups) {
return
}
return filters
}
export function isSupportedUserSearch(query: SearchFilters) { export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [ const allowed = [
{ op: BasicOperator.STRING, key: "email" }, { op: BasicOperator.STRING, key: "email" },
@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
} }
return true return true
} }
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {LegacyFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: LegacyFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterAllowedKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
// Sort the properties for easier debugging
const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
.sort((a, b) => {
return a.localeCompare(b)
})
.filter(key => key in filter)
if (filterPropertyKeys.length == 1) {
const key = filterPropertyKeys[0],
value = filter[key]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
(acc: AllowedFilters, key) => {
const value = filter[key]
if (filterAllowedKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: LegacyFilter = Object.fromEntries(
allowedFilterSettings
) as LegacyFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
}, defaultCfg)
} else if (!filters?.groups) {
return
}
return filters
}

21
packages/string-templates/src/errors.ts
View File

@ -1,3 +1,20 @@
export class JsErrorTimeout extends Error { export class JsTimeoutError extends Error {
code = "ERR_SCRIPT_EXECUTION_TIMEOUT" static message = "Timed out while executing JS"
static code = "JS_TIMEOUT_ERROR"
code: string = JsTimeoutError.code
constructor() {
super(JsTimeoutError.message)
}
}
export class UserScriptError extends Error {
static code = "USER_SCRIPT_ERROR"
code: string = UserScriptError.code
constructor(readonly userScriptError: Error) {
super(
`error while running user-supplied JavaScript: ${userScriptError.toString()}`
)
}
} }

42
packages/string-templates/src/helpers/javascript.ts
View File

@ -3,6 +3,7 @@ import cloneDeep from "lodash/fp/cloneDeep"
import { LITERAL_MARKER } from "../helpers/constants" import { LITERAL_MARKER } from "../helpers/constants"
import { getJsHelperList } from "./list" import { getJsHelperList } from "./list"
import { iifeWrapper } from "../iife" import { iifeWrapper } from "../iife"
import { JsTimeoutError, UserScriptError } from "../errors"
// The method of executing JS scripts depends on the bundle being built. // The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.js or index.mjs). // This setter is used in the entrypoint (either index.js or index.mjs).
@ -94,12 +95,49 @@ export function processJS(handlebars: string, context: any) {
} catch (error: any) { } catch (error: any) {
onErrorLog && onErrorLog(error) onErrorLog && onErrorLog(error)
const { noThrow = true } = context.__opts || {}
// The error handling below is quite messy, because it has fallen to
// string-templates to handle a variety of types of error specific to usages
// above it in the stack. It would be nice some day to refactor this to
// allow each user of processStringSync to handle errors in the way they see
// fit.
// This is to catch the error vm.runInNewContext() throws when the timeout
// is exceeded.
if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") { if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
return "Timed out while executing JS" return "Timed out while executing JS"
} }
if (error.name === "ExecutionTimeoutError") {
return "Request JS execution limit hit" // This is to catch the JsRequestTimeoutError we throw when we detect a
// timeout across an entire request in the backend. We use a magic string
// because we can't import from the backend into string-templates.
if (error.code === "JS_REQUEST_TIMEOUT_ERROR") {
return error.message
} }
// This is to catch the JsTimeoutError we throw when we detect a timeout in
// a single JS execution.
if (error.code === JsTimeoutError.code) {
return JsTimeoutError.message
}
// This is to catch the error that happens if a user-supplied JS script
// throws for reasons introduced by the user.
if (error.code === UserScriptError.code) {
if (noThrow) {
return error.userScriptError.toString()
}
throw error
}
if (error.name === "SyntaxError") {
if (noThrow) {
return error.toString()
}
throw error
}
return "Error while executing JS" return "Error while executing JS"
} }
} }

35
packages/string-templates/src/index.ts
View File

@ -16,6 +16,7 @@ import { removeJSRunner, setJSRunner } from "./helpers/javascript"
import manifest from "./manifest.json" import manifest from "./manifest.json"
import { ProcessOptions } from "./types" import { ProcessOptions } from "./types"
import { UserScriptError } from "./errors"
export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list"
export { FIND_ANY_HBS_REGEX } from "./utilities" export { FIND_ANY_HBS_REGEX } from "./utilities"
@ -122,7 +123,7 @@ function createTemplate(
export async function processObject<T extends Record<string, any>>( export async function processObject<T extends Record<string, any>>(
object: T, object: T,
context: object, context: object,
opts?: { noHelpers?: boolean; escapeNewlines?: boolean; onlyFound?: boolean } opts?: ProcessOptions
): Promise<T> { ): Promise<T> {
testObject(object) testObject(object)
@ -172,7 +173,7 @@ export async function processString(
export function processObjectSync( export function processObjectSync(
object: { [x: string]: any }, object: { [x: string]: any },
context: any, context: any,
opts: any opts?: ProcessOptions
): object | Array<any> { ): object | Array<any> {
testObject(object) testObject(object)
for (let key of Object.keys(object || {})) { for (let key of Object.keys(object || {})) {
@ -229,9 +230,13 @@ export function processStringSync(
} else { } else {
return process(string) return process(string)
} }
} catch (err) { } catch (err: any) {
const { noThrow = true } = opts || {}
if (noThrow) {
return input return input
} }
throw err
}
} }
/** /**
@ -448,7 +453,7 @@ export function convertToJS(hbs: string) {
return `${varBlock}${js}` return `${varBlock}${js}`
} }
export { JsErrorTimeout } from "./errors" export { JsTimeoutError, UserScriptError } from "./errors"
export function defaultJSSetup() { export function defaultJSSetup() {
if (!isBackendService()) { if (!isBackendService()) {
@ -463,7 +468,27 @@ export function defaultJSSetup() {
setTimeout: undefined, setTimeout: undefined,
} }
createContext(context) createContext(context)
return runInNewContext(js, context, { timeout: 1000 })
const wrappedJs = `
result = {
result: null,
error: null,
};
try {
result.result = ${js};
} catch (e) {
result.error = e;
}
result;
`
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
}) })
} else { } else {
removeJSRunner() removeJSRunner()

1
packages/string-templates/src/types.ts
View File

@ -3,6 +3,7 @@ export interface ProcessOptions {
noEscaping?: boolean noEscaping?: boolean
noHelpers?: boolean noHelpers?: boolean
noFinalise?: boolean noFinalise?: boolean
noThrow?: boolean
escapeNewlines?: boolean escapeNewlines?: boolean
onlyFound?: boolean onlyFound?: boolean
disabledHelpers?: string[] disabledHelpers?: string[]

2
packages/types/src/api/web/app/rowAction.ts
View File

@ -8,7 +8,7 @@ export interface RowActionResponse extends RowActionData {
id: string id: string
tableId: string tableId: string
automationId: string automationId: string
allowedViews: string[] | undefined allowedSources: string[] | undefined
} }
export interface RowActionsResponse { export interface RowActionsResponse {

1
packages/types/src/api/web/auth.ts
View File

@ -21,6 +21,7 @@ export interface UpdateSelfRequest {
freeTrialConfirmedAt?: string freeTrialConfirmedAt?: string
appFavourites?: string[] appFavourites?: string[]
tours?: Record<string, Date> tours?: Record<string, Date>
appSort?: string
} }
export interface UpdateSelfResponse { export interface UpdateSelfResponse {

5
packages/types/src/api/web/searchFilter.ts
View File

@ -5,7 +5,7 @@ import {
SearchFilters, SearchFilters,
} from "../../sdk" } from "../../sdk"
export type SearchFilter = { export type LegacyFilter = {
operator: keyof SearchFilters | "rangeLow" | "rangeHigh" operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
onEmptyFilter?: EmptyFilterOption onEmptyFilter?: EmptyFilterOption
field: string field: string
@ -14,9 +14,10 @@ export type SearchFilter = {
externalType?: string externalType?: string
} }
// this is a type purely used by the UI
export type SearchFilterGroup = { export type SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator logicalOperator: FilterGroupLogicalOperator
onEmptyFilter?: EmptyFilterOption onEmptyFilter?: EmptyFilterOption
groups?: SearchFilterGroup[] groups?: SearchFilterGroup[]
filters?: SearchFilter[] filters?: LegacyFilter[]
} }

6
packages/types/src/documents/app/rowAction.ts
View File

@ -8,8 +8,10 @@ export interface TableRowActions extends Document {
export interface RowActionData { export interface RowActionData {
name: string name: string
automationId: string automationId: string
permissions: { permissions: RowActionPermissions
}
export interface RowActionPermissions {
table: { runAllowed: boolean } table: { runAllowed: boolean }
views: Record<string, { runAllowed: boolean }> views: Record<string, { runAllowed: boolean }>
} }
}

37
packages/types/src/documents/app/view.ts
View File

@ -1,7 +1,7 @@
import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table" import { UIFieldMetadata } from "./table"
import { Document } from "../document" import { Document } from "../document"
import { DBView } from "../../sdk" import { DBView, SearchFilters } from "../../sdk"
export type ViewTemplateOpts = { export type ViewTemplateOpts = {
field: string field: string
@ -42,11 +42,31 @@ export interface RelationSchemaField extends UIFieldMetadata {
readonly?: boolean readonly?: boolean
} }
export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata { export interface NumericCalculationFieldMetadata
calculationType: CalculationType extends BasicViewFieldMetadata {
calculationType:
| CalculationType.MIN
| CalculationType.MAX
| CalculationType.SUM
| CalculationType.AVG
field: string field: string
} }
export interface CountCalculationFieldMetadata extends BasicViewFieldMetadata {
calculationType: CalculationType.COUNT
}
export interface CountDistinctCalculationFieldMetadata
extends CountCalculationFieldMetadata {
distinct: true
field: string
}
export type ViewCalculationFieldMetadata =
| NumericCalculationFieldMetadata
| CountCalculationFieldMetadata
| CountDistinctCalculationFieldMetadata
export type ViewFieldMetadata = export type ViewFieldMetadata =
| BasicViewFieldMetadata | BasicViewFieldMetadata
| ViewCalculationFieldMetadata | ViewCalculationFieldMetadata
@ -65,15 +85,20 @@ export interface ViewV2 {
name: string name: string
primaryDisplay?: string primaryDisplay?: string
tableId: string tableId: string
query?: SearchFilter[] | SearchFilterGroup query?: LegacyFilter[] | SearchFilters
// duplicate to store UI information about filters
queryUI?: SearchFilterGroup
sort?: { sort?: {
field: string field: string
order?: SortOrder order?: SortOrder
type?: SortType type?: SortType
} }
schema?: Record<string, ViewFieldMetadata> schema?: ViewV2Schema
uiMetadata?: Record<string, any>
} }
export type ViewV2Schema = Record<string, ViewFieldMetadata>
export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
export interface ViewCountOrSumSchema { export interface ViewCountOrSumSchema {

1
packages/types/src/documents/global/user.ts
View File

@ -67,6 +67,7 @@ export interface User extends Document {
scimInfo?: { isSync: true } & Record<string, any> scimInfo?: { isSync: true } & Record<string, any>
appFavourites?: string[] appFavourites?: string[]
ssoId?: string ssoId?: string
appSort?: string
} }
export enum UserStatus { export enum UserStatus {

2
packages/types/src/sdk/ai.ts
View File

@ -63,7 +63,7 @@ type TranslateSchema = BaseSchema & {
type CategoriseTextSchema = BaseSchema & { type CategoriseTextSchema = BaseSchema & {
operation: AIOperationEnum.CATEGORISE_TEXT operation: AIOperationEnum.CATEGORISE_TEXT
columns: string[] columns: string[]
categories: string categories: string[]
} }
type SentimentAnalysisSchema = BaseSchema & { type SentimentAnalysisSchema = BaseSchema & {

25
packages/types/src/sdk/row.ts
View File

@ -3,12 +3,33 @@ import { SearchFilters } from "./search"
import { CalculationType, Row } from "../documents" import { CalculationType, Row } from "../documents"
import { WithRequired } from "../shared" import { WithRequired } from "../shared"
export interface Aggregation { export interface BaseAggregation {
name: string name: string
calculationType: CalculationType }
export interface NumericAggregation extends BaseAggregation {
calculationType:
| CalculationType.AVG
| CalculationType.MAX
| CalculationType.MIN
| CalculationType.SUM
field: string field: string
} }
export interface CountAggregation extends BaseAggregation {
calculationType: CalculationType.COUNT
}
export interface CountDistinctAggregation extends CountAggregation {
distinct: true
field: string
}
export type Aggregation =
| NumericAggregation
| CountAggregation
| CountDistinctAggregation
export interface SearchParams { export interface SearchParams {
tableId?: string tableId?: string
viewId?: string viewId?: string

1
packages/worker/src/api/routes/validation/users.ts
View File

@ -29,6 +29,7 @@ export const buildSelfSaveValidation = () => {
freeTrialConfirmedAt: Joi.string().optional(), freeTrialConfirmedAt: Joi.string().optional(),
appFavourites: Joi.array().optional(), appFavourites: Joi.array().optional(),
tours: Joi.object().optional(), tours: Joi.object().optional(),
appSort: Joi.string().optional(),
} }
return auth.joiValidator.body(Joi.object(schema).required().unknown(false)) return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
} }

22
yarn.lock
View File

@ -2051,7 +2051,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.10": "@budibase/backend-core@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
"@budibase/nano" "10.1.5" "@budibase/nano" "10.1.5"
@ -2132,15 +2132,15 @@
through2 "^2.0.0" through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest": "@budibase/pro@npm:@budibase/pro@latest":
version "2.32.10" version "2.32.11"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.10.tgz#19fcbb3ced74791a7e96dfdc5a1270165792eea5" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.11.tgz#c94d534f829ca0ef252677757e157a7e58b87b4d"
integrity sha512-TbVp2bjmA0rHK+TKi9NVW06+O23fhDm7IJ/FlpWPHIBIZW7xDkCYu6LUOhSwSWMbOTcWzaJFuMbpN1HoTc/YjQ== integrity sha512-mOkqJpqHKWsfTWZwWcvBCYFUIluSUHltQNinc1ZRsg9rC3OKoHSDop6gzm744++H/GzGRN8V86kLhCgtNIlkpA==
dependencies: dependencies:
"@anthropic-ai/sdk" "^0.27.3" "@anthropic-ai/sdk" "^0.27.3"
"@budibase/backend-core" "2.32.10" "@budibase/backend-core" "2.32.11"
"@budibase/shared-core" "2.32.10" "@budibase/shared-core" "2.32.11"
"@budibase/string-templates" "2.32.10" "@budibase/string-templates" "2.32.11"
"@budibase/types" "2.32.10" "@budibase/types" "2.32.11"
"@koa/router" "8.0.8" "@koa/router" "8.0.8"
bull "4.10.1" bull "4.10.1"
dd-trace "5.2.0" dd-trace "5.2.0"
@ -2153,13 +2153,13 @@
scim-patch "^0.8.1" scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8" scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.10": "@budibase/shared-core@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
"@budibase/types" "0.0.0" "@budibase/types" "0.0.0"
cron-validate "1.4.5" cron-validate "1.4.5"
"@budibase/string-templates@2.32.10": "@budibase/string-templates@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
"@budibase/handlebars-helpers" "^0.13.2" "@budibase/handlebars-helpers" "^0.13.2"
@ -2167,7 +2167,7 @@
handlebars "^4.7.8" handlebars "^4.7.8"
lodash.clonedeep "^4.5.0" lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.10": "@budibase/types@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
scim-patch "^0.8.1" scim-patch "^0.8.1"