MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge and typing

This commit is contained in:
Martin McKeaveney 2024-10-08 17:38:23 +01:00
parent d5324388d6 fda5fdc16e
commit 7b683cfc50
57 changed files with 4160 additions and 3144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.10",
"version": "2.32.11",
"npmClient": "yarn",
"packages": [
"packages/*",

113
packages/backend-core/src/sql/sql.ts
View File

@ -139,29 +139,61 @@ class InternalBuilder {
return this.table.schema[column]
}
// Takes a string like foo and returns a quoted string like [foo] for SQL Server
// and "foo" for Postgres.
private quote(str: string): string {
private quoteChars(): [string, string] {
switch (this.client) {
case SqlClient.SQL_LITE:
case SqlClient.ORACLE:
case SqlClient.POSTGRES:
return `"${str}"`
return ['"', '"']
case SqlClient.MS_SQL:
return `[${str}]`
return ["[", "]"]
case SqlClient.MARIADB:
case SqlClient.MY_SQL:
return `\`${str}\``
case SqlClient.SQL_LITE:
return ["`", "`"]
}
}
// Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
// for SQL Server and `a`.`b`.`c` for MySQL.
private quotedIdentifier(key: string): string {
return key
.split(".")
.map(part => this.quote(part))
.join(".")
// Takes a string like foo and returns a quoted string like [foo] for SQL Server
// and "foo" for Postgres.
private quote(str: string): string {
const [start, end] = this.quoteChars()
return `${start}${str}${end}`
}
private isQuoted(key: string): boolean {
const [start, end] = this.quoteChars()
return key.startsWith(start) && key.endsWith(end)
}
// Takes a string like a.b.c or an array like ["a", "b", "c"] and returns a
// quoted identifier like [a].[b].[c] for SQL Server and `a`.`b`.`c` for
// MySQL.
private quotedIdentifier(key: string | string[]): string {
if (!Array.isArray(key)) {
key = this.splitIdentifier(key)
}
return key.map(part => this.quote(part)).join(".")
}
// Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"]
private splitIdentifier(key: string): string[] {
const [start, end] = this.quoteChars()
if (this.isQuoted(key)) {
return key.slice(1, -1).split(`${end}.${start}`)
}
return key.split(".")
}
private qualifyIdentifier(key: string): string {
const tableName = this.getTableName()
const parts = this.splitIdentifier(key)
if (parts[0] !== tableName) {
parts.unshift(tableName)
}
if (this.isQuoted(key)) {
return this.quotedIdentifier(parts)
}
return parts.join(".")
}
private isFullSelectStatementRequired(): boolean {
@ -231,8 +263,13 @@ class InternalBuilder {
// OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
// so when we use them we need to wrap them in to_char(). This function
// converts a field name to the appropriate identifier.
private convertClobs(field: string): string {
const parts = field.split(".")
private convertClobs(field: string, opts?: { forSelect?: boolean }): string {
if (this.client !== SqlClient.ORACLE) {
throw new Error(
"you've called convertClobs on a DB that's not Oracle, this is a mistake"
)
}
const parts = this.splitIdentifier(field)
const col = parts.pop()!
const schema = this.table.schema[col]
let identifier = this.quotedIdentifier(field)
@ -244,8 +281,12 @@ class InternalBuilder {
schema.type === FieldType.OPTIONS ||
schema.type === FieldType.BARCODEQR
) {
if (opts?.forSelect) {
identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}`
} else {
identifier = `to_char(${identifier})`
}
}
return identifier
}
@ -859,16 +900,45 @@ class InternalBuilder {
const fields = this.query.resource?.fields || []
const tableName = this.getTableName()
if (fields.length > 0) {
query = query.groupBy(fields.map(field => `${tableName}.${field}`))
query = query.select(fields.map(field => `${tableName}.${field}`))
const qualifiedFields = fields.map(field => this.qualifyIdentifier(field))
if (this.client === SqlClient.ORACLE) {
const groupByFields = qualifiedFields.map(field =>
this.convertClobs(field)
)
const selectFields = qualifiedFields.map(field =>
this.convertClobs(field, { forSelect: true })
)
query = query
.groupByRaw(groupByFields.join(", "))
.select(this.knex.raw(selectFields.join(", ")))
} else {
query = query.groupBy(qualifiedFields).select(qualifiedFields)
}
}
for (const aggregation of aggregations) {
const op = aggregation.calculationType
if (op === CalculationType.COUNT) {
if ("distinct" in aggregation && aggregation.distinct) {
if (this.client === SqlClient.ORACLE) {
const field = this.convertClobs(`${tableName}.${aggregation.field}`)
query = query.select(
this.knex.raw(
`COUNT(DISTINCT ${field}) as ${this.quotedIdentifier(
aggregation.name
)}`
)
)
} else {
query = query.countDistinct(
`${tableName}.${aggregation.field} as ${aggregation.name}`
)
}
} else {
query = query.count(`* as ${aggregation.name}`)
}
} else {
const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
switch (op) {
case CalculationType.COUNT:
query = query.count(field)
break
case CalculationType.SUM:
query = query.sum(field)
break
@ -883,6 +953,7 @@ class InternalBuilder {
break
}
}
}
return query
}

2
packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte
View File

@ -95,7 +95,7 @@
{#if isView}
<span>
<Toggle
value={action.allowedViews?.includes(viewId)}
value={action.allowedSources?.includes(viewId)}
on:change={e => toggleAction(action, e.detail)}
/>
</span>

100
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -4,6 +4,7 @@
Button,
Label,
Select,
Multiselect,
Toggle,
Icon,
DatePicker,
@ -142,9 +143,10 @@
}
$: initialiseField(field, savingColumn)
$: checkConstraints(editableColumn)
$: required = hasDefault
? false
: !!editableColumn?.constraints?.presence || primaryDisplay
$: required =
primaryDisplay ||
editableColumn?.constraints?.presence === true ||
editableColumn?.constraints?.presence?.allowEmpty === false
$: uneditable =
$tables.selected?._id === TableNames.USERS &&
UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@ -173,8 +175,8 @@
// used to select what different options can be displayed for column type
$: canBeDisplay =
canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
$: canHaveDefault =
isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type)
$: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
$: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
$: canBeRequired =
editableColumn?.type !== LINK_TYPE &&
!uneditable &&
@ -193,7 +195,6 @@
(originalName &&
SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn)
$: allowedTypes = getAllowedTypes(datasource).map(t => ({
fieldId: makeFieldId(t.type, t.subtype),
...t,
@ -211,6 +212,11 @@
},
...getUserBindings(),
]
$: sanitiseDefaultValue(
editableColumn.type,
editableColumn.constraints?.inclusion || [],
editableColumn.default
)
const fieldDefinitions = Object.values(FIELDS).reduce(
// Storing the fields by complex field id
@ -300,6 +306,22 @@
delete saveColumn.fieldName
}
// Ensure we don't have a default value if we can't have one
if (!canHaveDefault || !defaultValuesEnabled) {
delete saveColumn.default
}
// Ensure primary display columns are always required and don't have default values
if (primaryDisplay) {
saveColumn.constraints.presence = { allowEmpty: false }
delete saveColumn.default
}
// Ensure the field is not required if we have a default value
if (saveColumn.default) {
saveColumn.constraints.presence = false
}
try {
await tables.saveField({
originalName,
@ -547,6 +569,20 @@
return newError
}
const sanitiseDefaultValue = (type, options, defaultValue) => {
if (!defaultValue?.length) {
return
}
// Delete default value for options fields if the option is no longer available
if (type === FieldType.OPTIONS && !options.includes(defaultValue)) {
delete editableColumn.default
}
// Filter array default values to only valid options
if (type === FieldType.ARRAY) {
editableColumn.default = defaultValue.filter(x => options.includes(x))
}
}
onMount(() => {
mounted = true
})
@ -761,9 +797,9 @@
schema={table.schema}
/>
{:else if editableColumn.type === JSON_TYPE}
<Button primary text on:click={openJsonSchemaEditor}
>Open schema editor</Button
>
<Button primary text on:click={openJsonSchemaEditor}>
Open schema editor
</Button>
{/if}
{#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
<Select
@ -792,27 +828,39 @@
</div>
{/if}
{#if canHaveDefault}
<div>
<ModalBindableInput
panel={ServerBindingPanel}
title="Default"
label="Default"
{#if defaultValuesEnabled}
{#if editableColumn.type === FieldType.OPTIONS}
<Select
disabled={!canHaveDefault}
options={editableColumn.constraints?.inclusion || []}
label="Default value"
value={editableColumn.default}
on:change={e => {
editableColumn = {
...editableColumn,
default: e.detail,
}
if (e.detail) {
setRequired(false)
}
}}
on:change={e => (editableColumn.default = e.detail)}
placeholder="None"
/>
{:else if editableColumn.type === FieldType.ARRAY}
<Multiselect
disabled={!canHaveDefault}
options={editableColumn.constraints?.inclusion || []}
label="Default value"
value={editableColumn.default}
on:change={e =>
(editableColumn.default = e.detail?.length ? e.detail : undefined)}
placeholder="None"
/>
{:else}
<ModalBindableInput
disabled={!canHaveDefault}
panel={ServerBindingPanel}
title="Default value"
label="Default value"
placeholder="None"
value={editableColumn.default}
on:change={e => (editableColumn.default = e.detail)}
bindings={defaultValueBindings}
allowJS
/>
</div>
{/if}
{/if}
</Layout>

18
packages/builder/src/components/common/bindings/BindingPanel.svelte
View File

@ -66,6 +66,7 @@
let insertAtPos
let targetMode = null
let expressionResult
let expressionError
let evaluating = false
$: useSnippets = allowSnippets && !$licensing.isFreePlan
@ -142,10 +143,22 @@
}
const debouncedEval = Utils.debounce((expression, context, snippets) => {
expressionResult = processStringSync(expression || "", {
try {
expressionError = null
expressionResult = processStringSync(
expression || "",
{
...context,
snippets,
})
},
{
noThrow: false,
}
)
} catch (err) {
expressionResult = null
expressionError = err
}
evaluating = false
}, 260)
@ -370,6 +383,7 @@
{:else if sidePanel === SidePanels.Evaluation}
<EvaluationSidePanel
{expressionResult}
{expressionError}
{evaluating}
expression={editorValue}
/>

19
packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte
View File

@ -3,26 +3,37 @@
import { Icon, ProgressCircle, notifications } from "@budibase/bbui"
import { copyToClipboard } from "@budibase/bbui/helpers"
import { fade } from "svelte/transition"
import { UserScriptError } from "@budibase/string-templates"
export let expressionResult
export let expressionError
export let evaluating = false
export let expression = null
$: error = expressionResult === "Error while executing JS"
$: error = expressionError != null
$: empty = expression == null || expression?.trim() === ""
$: success = !error && !empty
$: highlightedResult = highlight(expressionResult)
const formatError = err => {
if (err.code === UserScriptError.code) {
return err.userScriptError.toString()
}
return err.toString()
}
const highlight = json => {
if (json == null) {
return ""
}
// Attempt to parse and then stringify, in case this is valid JSON
// Attempt to parse and then stringify, in case this is valid result
try {
json = JSON.stringify(JSON.parse(json), null, 2)
} catch (err) {
// Ignore
}
return formatHighlight(json, {
keyColor: "#e06c75",
numberColor: "#e5c07b",
@ -34,7 +45,7 @@
}
const copy = () => {
let clipboardVal = expressionResult
let clipboardVal = expressionResult.result
if (typeof clipboardVal === "object") {
clipboardVal = JSON.stringify(clipboardVal, null, 2)
}
@ -73,6 +84,8 @@
<div class="body">
{#if empty}
Your expression will be evaluated here
{:else if error}
{formatError(expressionError)}
{:else}
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
{@html highlightedResult}

4
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
View File

@ -63,7 +63,9 @@
$: rowActions.refreshRowActions(id)
const makeRowActionButtons = actions => {
return (actions || []).map(action => ({
return (actions || [])
.filter(action => action.allowedSources?.includes(id))
.map(action => ({
text: action.name,
onClick: async row => {
await rowActions.trigger(id, action.id, row._id)

3
packages/builder/src/pages/builder/portal/apps/index.svelte
View File

@ -26,6 +26,7 @@
licensing,
environment,
enrichedApps,
sortBy,
} from "stores/portal"
import { goto } from "@roxi/routify"
import AppRow from "components/start/AppRow.svelte"
@ -247,7 +248,7 @@
<div class="app-actions">
<Select
autoWidth
value={$appsStore.sortBy}
value={$sortBy}
on:change={e => {
appsStore.updateSort(e.detail)
}}

12
packages/builder/src/stores/builder/rowActions.js
View File

@ -129,13 +129,15 @@ const derivedStore = derived(store, $store => {
// Generate an entry for every view as well
Object.keys($store || {}).forEach(tableId => {
// We need to have all the actions for the table in order to be displayed in the crud section
map[tableId] = $store[tableId]
for (let action of $store[tableId]) {
for (let viewId of action.allowedViews || []) {
if (!map[viewId]) {
map[viewId] = []
}
map[viewId].push(action)
const otherSources = (action.allowedSources || []).filter(
sourceId => sourceId !== tableId
)
for (let source of otherSources) {
map[source] ??= []
map[source].push(action)
}
}
})

23
packages/builder/src/stores/builder/viewsV2.js
View File

@ -1,6 +1,24 @@
import { writable, derived, get } from "svelte/store"
import { tables } from "./tables"
import { API } from "api"
import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
}
return view
}
function convertToSearchFilterGroup(view) {
if (view.queryUI) {
view.query = view.queryUI
delete view.queryUI
}
return view
}
export function createViewsV2Store() {
const store = writable({
@ -12,7 +30,7 @@ export function createViewsV2Store() {
const views = Object.values(table?.views || {}).filter(view => {
return view.version === 2
})
list = list.concat(views)
list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
})
return {
...$store,
@ -34,6 +52,7 @@ export function createViewsV2Store() {
}
const create = async view => {
view = convertToSearchFilters(view)
const savedViewResponse = await API.viewV2.create(view)
const savedView = savedViewResponse.data
replaceView(savedView.id, savedView)
@ -41,6 +60,7 @@ export function createViewsV2Store() {
}
const save = async view => {
view = convertToSearchFilters(view)
const res = await API.viewV2.update(view)
const savedView = res?.data
replaceView(view.id, savedView)
@ -51,6 +71,7 @@ export function createViewsV2Store() {
if (!viewId) {
return
}
view = convertToSearchFilterGroup(view)
const existingView = get(derivedStore).list.find(view => view.id === viewId)
const tableIndex = get(tables).list.findIndex(table => {
return table._id === view?.tableId || table._id === existingView?.tableId

25
packages/builder/src/stores/portal/apps.js
View File

@ -9,7 +9,6 @@ const DEV_PROPS = ["updatedBy", "updatedAt"]
export const INITIAL_APPS_STATE = {
apps: [],
sortBy: "name",
}
export class AppsStore extends BudiStore {
@ -53,6 +52,15 @@ export class AppsStore extends BudiStore {
...state,
sortBy,
}))
this.updateUserSort(sortBy)
}
async updateUserSort(sortBy) {
try {
await auth.updateSelf({ appSort: sortBy })
} catch (err) {
console.error("couldn't save user sort: ", err)
}
}
async load() {
@ -140,8 +148,14 @@ export class AppsStore extends BudiStore {
export const appsStore = new AppsStore()
export const sortBy = derived([appsStore, auth], ([$store, $auth]) => {
return $store.sortBy || $auth.user?.appSort || "name"
})
// Centralise any logic that enriches the apps list
export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
export const enrichedApps = derived(
[appsStore, auth, sortBy],
([$store, $auth, $sortBy]) => {
const enrichedApps = $store.apps
? $store.apps.map(app => ({
...app,
@ -152,7 +166,7 @@ export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
}))
: []
if ($store.sortBy === "status") {
if ($sortBy === "status") {
return enrichedApps.sort((a, b) => {
if (a.favourite === b.favourite) {
if (a.status === b.status) {
@ -162,7 +176,7 @@ export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
}
return a.favourite ? -1 : 1
})
} else if ($store.sortBy === "updated") {
} else if ($sortBy === "updated") {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
const aUpdated = a.updatedAt || "9999"
@ -179,4 +193,5 @@ export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
return a.favourite ? -1 : 1
})
}
})
}
)

2
packages/builder/src/stores/portal/index.js
View File

@ -3,7 +3,7 @@ import { writable } from "svelte/store"
export { organisation } from "./organisation"
export { users } from "./users"
export { admin } from "./admin"
export { appsStore, enrichedApps } from "./apps"
export { appsStore, enrichedApps, sortBy } from "./apps"
export { email } from "./email"
export { auth } from "./auth"
export { oidc } from "./oidc"

3
packages/frontend-core/src/components/grid/cells/AICell.svelte
View File

@ -70,9 +70,6 @@
align-items: flex-start;
overflow: hidden;
}
.long-form-cell.editable:hover {
cursor: text;
}
.value {
display: -webkit-box;
-webkit-line-clamp: var(--content-lines);

17
packages/frontend-core/src/components/grid/stores/datasource.js
View File

@ -1,6 +1,7 @@
import { derived, get } from "svelte/store"
import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
import { enrichSchemaWithRelColumns, memo } from "../../../utils"
import { cloneDeep } from "lodash"
export const createStores = () => {
const definition = memo(null)
@ -164,10 +165,18 @@ export const createActions = context => {
// Updates the datasources primary display column
const changePrimaryDisplay = async column => {
return await saveDefinition({
...get(definition),
primaryDisplay: column,
})
let newDefinition = cloneDeep(get(definition))
// Update primary display
newDefinition.primaryDisplay = column
// Sanitise schema to ensure field is required and has no default value
if (!newDefinition.schema[column].constraints) {
newDefinition.schema[column].constraints = {}
}
newDefinition.schema[column].constraints.presence = { allowEmpty: false }
delete newDefinition.schema[column].default
return await saveDefinition(newDefinition)
}
// Adds a schema mutation for a single field

14
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -1,4 +1,14 @@
import { get } from "svelte/store"
import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
}
return view
}
const SuppressErrors = true
@ -6,7 +16,7 @@ export const createActions = context => {
const { API, datasource, columns } = context
const saveDefinition = async newDefinition => {
await API.viewV2.update(newDefinition)
await API.viewV2.update(convertToSearchFilters(newDefinition))
}
const saveRow = async row => {
@ -125,7 +135,7 @@ export const initialise = context => {
}
// Only override filter state if we don't have an initial filter
if (!get(initialFilter)) {
filter.set($definition.query)
filter.set($definition.queryUI || $definition.query)
}
})
)

2
packages/pro

@ -1 +1 @@
Subproject commit 0c8f834932670ce1d3b3f639e6e8d0d0086e508d
Subproject commit 1bce30827813e135ee414141fdf51e45236b8f3c

3
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -697,9 +697,8 @@ export class ExternalRequest<T extends Operation> {
const calculationFields = helpers.views.calculationFields(this.source)
for (const [key, field] of Object.entries(calculationFields)) {
aggregations.push({
...field,
name: key,
field: field.field,
calculationType: field.calculationType,
})
}
}

4
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -144,7 +144,9 @@ export async function finaliseRow(
dynamic: false,
contextRows: [enrichedRow],
})
const aiEnabled = await pro.features.isBudibaseAIEnabled() || await pro.features.isAICustomConfigsEnabled()
const aiEnabled =
(await pro.features.isBudibaseAIEnabled()) ||
(await pro.features.isAICustomConfigsEnabled())
if (aiEnabled) {
row = await processAIColumns(table, row, {
contextRows: [enrichedRow],

6
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -125,9 +125,11 @@ export async function buildSqlFieldList(
column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI &&
!existing.find((field: string) => field === columnName)
!existing.find(
(field: string) => field === `${table.name}.${columnName}`
)
.map(column => `${table.name}.${column[0]}`)
)
.map(([columnName]) => `${table.name}.${columnName}`)
}
let fields: string[] = []

22
packages/server/src/api/controllers/row/views.ts
View File

@ -3,6 +3,8 @@ import {
ViewV2,
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
RowSearchParams,
} from "@budibase/types"
import sdk from "../../../sdk"
import { context } from "@budibase/backend-core"
@ -20,30 +22,34 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`)
}
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request
await context.ensureSnippetContext(true)
const result = await sdk.rows.search(
{
viewId: view.id,
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<
Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
> = {
tableId: view.tableId,
viewId: view.id,
query: body.query,
fields: viewFields,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
countRows: body.countRows,
},
{
user: sdk.users.getUserContextBindings(ctx.user),
}
)
const result = await sdk.rows.search(searchOptions, {
user: sdk.users.getUserContextBindings(ctx.user),
})
result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result
}
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) {
return {

87
packages/server/src/api/controllers/rowAction/crud.ts
View File

@ -1,6 +1,7 @@
import {
CreateRowActionRequest,
Ctx,
RowActionPermissions,
RowActionResponse,
RowActionsResponse,
UpdateRowActionRequest,
@ -18,25 +19,26 @@ async function getTable(ctx: Ctx) {
export async function find(ctx: Ctx<void, RowActionsResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
if (!(await sdk.rowActions.docExists(table._id!))) {
if (!(await sdk.rowActions.docExists(tableId))) {
ctx.body = {
actions: {},
}
return
}
const { actions } = await sdk.rowActions.getAll(table._id!)
const { actions } = await sdk.rowActions.getAll(tableId)
const result: RowActionsResponse = {
actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
(acc, [key, action]) => ({
...acc,
[key]: {
id: key,
tableId: table._id!,
tableId,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
},
}),
{}
@ -49,17 +51,18 @@ export async function create(
ctx: Ctx<CreateRowActionRequest, RowActionResponse>
) {
const table = await getTable(ctx)
const tableId = table._id!
const createdAction = await sdk.rowActions.create(table._id!, {
const createdAction = await sdk.rowActions.create(tableId, {
name: ctx.request.body.name,
})
ctx.body = {
tableId: table._id!,
tableId,
id: createdAction.id,
name: createdAction.name,
automationId: createdAction.automationId,
allowedViews: undefined,
allowedSources: flattenAllowedSources(tableId, createdAction.permissions),
}
ctx.status = 201
}
@ -68,18 +71,19 @@ export async function update(
ctx: Ctx<UpdateRowActionRequest, RowActionResponse>
) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.update(table._id!, actionId, {
const action = await sdk.rowActions.update(tableId, actionId, {
name: ctx.request.body.name,
})
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: undefined,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
@ -91,52 +95,89 @@ export async function remove(ctx: Ctx<void, void>) {
ctx.status = 204
}
export async function setTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.setTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.unsetTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.setViewPermission(
table._id!,
tableId,
actionId,
viewId
)
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.unsetViewPermission(
table._id!,
tableId,
actionId,
viewId
)
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
function flattenAllowedViews(
permissions: Record<string, { runAllowed: boolean }>
function flattenAllowedSources(
tableId: string,
permissions: RowActionPermissions
) {
const allowedPermissions = Object.entries(permissions || {})
.filter(([_, p]) => p.runAllowed)
.map(([viewId]) => viewId)
if (!allowedPermissions.length) {
return undefined
const allowedPermissions = []
if (permissions.table.runAllowed) {
allowedPermissions.push(tableId)
}
allowedPermissions.push(
...Object.keys(permissions.views || {}).filter(
viewId => permissions.views[viewId].runAllowed
)
)
return allowedPermissions
}

9
packages/server/src/api/controllers/script.ts
View File

@ -1,11 +1,18 @@
import { Ctx } from "@budibase/types"
import { IsolatedVM } from "../../jsRunner/vm"
import { iifeWrapper } from "@budibase/string-templates"
import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body
const vm = new IsolatedVM()
try {
ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
} catch (err: any) {
if (err.code === UserScriptError.code) {
throw err.userScriptError
}
throw err
}
}
export async function save(ctx: Ctx) {

2
packages/server/src/api/controllers/table/utils.ts
View File

@ -33,7 +33,7 @@ import {
} from "@budibase/types"
import sdk from "../../../sdk"
import env from "../../../environment"
import { runAIColumnChecks, runStaticFormulaChecks } from "./bulkFormula"
import { runStaticFormulaChecks } from "./bulkFormula"
export async function clearColumns(table: Table, columnNames: string[]) {
const db = context.getAppDB()

34
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -11,14 +11,40 @@ import {
ViewCalculationFieldMetadata,
RelationSchemaField,
ViewFieldMetadata,
CalculationType,
} from "@budibase/types"
import { builderSocket, gridSocket } from "../../../websockets"
import { helpers } from "@budibase/shared-core"
function stripUnknownFields(
field: BasicViewFieldMetadata
): RequiredKeys<BasicViewFieldMetadata> {
field: ViewFieldMetadata
): RequiredKeys<ViewFieldMetadata> {
if (helpers.views.isCalculationField(field)) {
if (field.calculationType === CalculationType.COUNT) {
if ("distinct" in field && field.distinct) {
return {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
distinct: field.distinct,
calculationType: field.calculationType,
field: field.field,
columns: field.columns,
}
} else {
return {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
calculationType: field.calculationType,
columns: field.columns,
}
}
}
const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
order: field.order,
width: field.width,
@ -103,9 +129,11 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
name: view.name,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.create(tableId, parsedView)
ctx.status = 201
@ -138,9 +166,11 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
version: view.version,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.update(tableId, parsedView)

10
packages/server/src/api/routes/rowAction.ts
View File

@ -51,6 +51,16 @@ router
authorized(BUILDER),
rowActionController.remove
)
.post(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.setTablePermission
)
.delete(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.unsetTablePermission
)
.post(
"/api/tables/:tableId/actions/:actionId/permissions/:viewId",
authorized(BUILDER),

5
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -49,6 +49,7 @@ import * as uuid from "uuid"
import { Knex } from "knex"
import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
@ -3013,7 +3014,7 @@ describe.each([
let i = 0
for (; i < 10; i++) {
const row = rows[i]
if (row.formula !== "Timed out while executing JS") {
if (row.formula !== JsTimeoutError.message) {
break
}
}
@ -3027,7 +3028,7 @@ describe.each([
for (; i < 10; i++) {
const row = rows[i]
expect(row.text).toBe("foo")
expect(row.formula).toBe("Request JS execution limit hit")
expect(row.formula).toStartWith("CPU time limit exceeded ")
}
}
}

145
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -133,6 +133,7 @@ describe("/rowsActions", () => {
id: expect.stringMatching(/^row_action_\w+/),
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual({
@ -142,6 +143,7 @@ describe("/rowsActions", () => {
id: res.id,
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -180,18 +182,21 @@ describe("/rowsActions", () => {
id: responses[0].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
[responses[1].id]: {
name: rowActions[1].name,
id: responses[1].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
[responses[2].id]: {
name: rowActions[2].name,
id: responses[2].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -224,6 +229,7 @@ describe("/rowsActions", () => {
id: expect.any(String),
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual({
@ -233,6 +239,7 @@ describe("/rowsActions", () => {
id: res.id,
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -354,6 +361,7 @@ describe("/rowsActions", () => {
tableId,
name: updatedName,
automationId: actionData.automationId,
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual(
@ -364,6 +372,7 @@ describe("/rowsActions", () => {
id: actionData.id,
tableId: actionData.tableId,
automationId: actionData.automationId,
allowedSources: [tableId],
},
}),
})
@ -515,6 +524,81 @@ describe("/rowsActions", () => {
})
})
describe("set/unsetTablePermission", () => {
describe.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])("unauthorisedTests for %s", (__, delegateTest) => {
unauthorisedTests((expectations, testConfig) =>
delegateTest(tableId, generator.guid(), expectations, testConfig)
)
})
let actionId1: string, actionId2: string
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
const persisted = await config.api.rowAction.find(tableId)
const actions = _.sampleSize(Object.keys(persisted.actions), 2)
actionId1 = actions[0]
actionId2 = actions[1]
})
it("can set table permission", async () => {
await config.api.rowAction.unsetTablePermission(tableId, actionId1)
await config.api.rowAction.unsetTablePermission(tableId, actionId2)
const actionResult = await config.api.rowAction.setTablePermission(
tableId,
actionId1
)
const expectedAction1 = expect.objectContaining({
allowedSources: [tableId],
})
const expectedActions = expect.objectContaining({
[actionId1]: expectedAction1,
[actionId2]: expect.objectContaining({
allowedSources: [],
}),
})
expect(actionResult).toEqual(expectedAction1)
expect((await config.api.rowAction.find(tableId)).actions).toEqual(
expectedActions
)
})
it("can unset table permission", async () => {
const actionResult = await config.api.rowAction.unsetTablePermission(
tableId,
actionId1
)
const expectedAction = expect.objectContaining({
allowedSources: [],
})
expect(actionResult).toEqual(expectedAction)
expect(
(await config.api.rowAction.find(tableId)).actions[actionId1]
).toEqual(expectedAction)
})
it.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])(
"cannot update permission for unexisting tables (%s)",
async (__, delegateTest) => {
const tableId = generator.guid()
await delegateTest(tableId, actionId1, {
status: 404,
})
}
)
})
describe("set/unsetViewPermission", () => {
describe.each([
["setViewPermission", config.api.rowAction.setViewPermission],
@ -531,11 +615,9 @@ describe("/rowsActions", () => {
)
})
let tableIdForDescribe: string
let actionId1: string, actionId2: string
let viewId1: string, viewId2: string
beforeAll(async () => {
tableIdForDescribe = tableId
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
@ -557,11 +639,6 @@ describe("/rowsActions", () => {
).id
})
beforeEach(() => {
// Hack to reuse tables for these given tests
tableId = tableIdForDescribe
})
it("can set permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId1)
const action1Result = await config.api.rowAction.setViewPermission(
@ -576,10 +653,10 @@ describe("/rowsActions", () => {
)
const expectedAction1 = expect.objectContaining({
allowedViews: [viewId1, viewId2],
allowedSources: [tableId, viewId1, viewId2],
})
const expectedAction2 = expect.objectContaining({
allowedViews: [viewId1],
allowedSources: [tableId, viewId1],
})
const expectedActions = expect.objectContaining({
@ -594,6 +671,8 @@ describe("/rowsActions", () => {
})
it("can unset permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId2, actionId1)
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId2)
const actionResult = await config.api.rowAction.unsetViewPermission(
tableId,
viewId1,
@ -601,7 +680,7 @@ describe("/rowsActions", () => {
)
const expectedAction = expect.objectContaining({
allowedViews: [viewId2],
allowedSources: [tableId, viewId2],
})
expect(actionResult).toEqual(expectedAction)
expect(
@ -672,6 +751,7 @@ describe("/rowsActions", () => {
)
await config.publish()
// Travel time in order to "trim" the selected `getAutomationLogs`
tk.travel(Date.now() + 100)
})
@ -698,6 +778,8 @@ describe("/rowsActions", () => {
inputs: null,
outputs: {
fields: {},
id: rowId,
revision: (await config.api.row.get(tableId, rowId))._rev,
row: await config.api.row.get(tableId, rowId),
table: {
...(await config.api.table.get(tableId)),
@ -712,6 +794,38 @@ describe("/rowsActions", () => {
])
})
it("triggers from an allowed table", async () => {
expect(await getAutomationLogs()).toBeEmpty()
await config.api.rowAction.trigger(tableId, rowAction.id, { rowId })
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([
expect.objectContaining({
automationId: rowAction.automationId,
}),
])
})
it("rejects triggering from a non-allowed table", async () => {
await config.api.rowAction.unsetTablePermission(tableId, rowAction.id)
await config.publish()
await config.api.rowAction.trigger(
tableId,
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([])
})
it("rejects triggering from a non-allowed view", async () => {
const viewId = (
await config.api.viewV2.create(
@ -901,7 +1015,7 @@ describe("/rowsActions", () => {
})
it.each(allowedRoleConfig)(
"does not allow running row actions for tables by default even",
"allow running row actions for tables by default",
async (userRole, resourcePermission) => {
await config.api.permission.add({
level: PermissionLevel.READ,
@ -918,15 +1032,12 @@ describe("/rowsActions", () => {
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
status: 200,
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toBeEmpty()
expect(automationLogs).toHaveLength(1)
})
}
)

371
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -28,11 +28,13 @@ import {
RowSearchParams,
SearchFilters,
SearchResponse,
SearchRowRequest,
SortOrder,
SortType,
Table,
TableSchema,
User,
ViewV2Schema,
} from "@budibase/types"
import _ from "lodash"
import tk from "timekeeper"
@ -64,18 +66,14 @@ describe.each([
let envCleanup: (() => void) | undefined
let datasource: Datasource | undefined
let client: Knex | undefined
let table: Table
let tableOrViewId: string
let rows: Row[]
async function basicRelationshipTables(type: RelationshipType) {
const relatedTable = await createTable(
{
const relatedTable = await createTable({
name: { name: "name", type: FieldType.STRING },
},
generator.guid().substring(0, 10)
)
table = await createTable(
{
})
const tableId = await createTable({
name: { name: "name", type: FieldType.STRING },
//@ts-ignore - API accepts this structure, will build out rest of definition
productCat: {
@ -83,17 +81,15 @@ describe.each([
relationshipType: type,
name: "productCat",
fieldName: "product",
tableId: relatedTable._id!,
tableId: relatedTable,
constraints: {
type: "array",
},
},
},
generator.guid().substring(0, 10)
)
})
return {
relatedTable: await config.api.table.get(relatedTable._id!),
table,
relatedTable: await config.api.table.get(relatedTable),
tableId,
}
}
@ -136,32 +132,69 @@ describe.each([
}
})
async function createTable(schema: TableSchema, name?: string) {
return await config.api.table.save(
tableForDatasource(datasource, { schema, name })
async function createTable(schema: TableSchema) {
const table = await config.api.table.save(
tableForDatasource(datasource, { schema })
)
return table._id!
}
async function createView(tableId: string, schema: ViewV2Schema) {
const view = await config.api.viewV2.create({
tableId: tableId,
name: generator.guid(),
schema,
})
return view.id
}
async function createRows(arr: Record<string, any>[]) {
// Shuffling to avoid false positives given a fixed order
await config.api.row.bulkImport(table._id!, {
rows: _.shuffle(arr),
})
rows = await config.api.row.fetch(table._id!)
for (const row of _.shuffle(arr)) {
await config.api.row.save(tableOrViewId, row)
}
rows = await config.api.row.fetch(tableOrViewId)
}
describe.each([
["table", createTable],
[
"view",
async (schema: TableSchema) => {
const tableId = await createTable(schema)
const viewId = await createView(
tableId,
Object.keys(schema).reduce<ViewV2Schema>((viewSchema, fieldName) => {
const field = schema[fieldName]
viewSchema[fieldName] = {
visible: field.visible ?? true,
readonly: false,
}
return viewSchema
}, {})
)
return viewId
},
],
])("from %s", (sourceType, createTableOrView) => {
const isView = sourceType === "view"
if (isView && isLucene) {
// Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests
return
}
class SearchAssertion {
constructor(private readonly query: RowSearchParams) {}
constructor(private readonly query: SearchRowRequest) {}
private async performSearch(): Promise<SearchResponse<Row>> {
if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), this.query)
return dataFilters.search(_.cloneDeep(rows), {
...this.query,
tableId: tableOrViewId,
})
} else {
const sourceId = this.query.viewId || this.query.tableId
if (!sourceId) {
throw new Error("No source ID provided")
}
return config.api.row.search(sourceId, this.query)
return config.api.row.search(tableOrViewId, this.query)
}
}
@ -331,8 +364,8 @@ describe.each([
}
}
function expectSearch(query: Omit<RowSearchParams, "tableId">) {
return new SearchAssertion({ ...query, tableId: table._id! })
function expectSearch(query: SearchRowRequest) {
return new SearchAssertion(query)
}
function expectQuery(query: SearchFilters) {
@ -341,7 +374,7 @@ describe.each([
describe("boolean", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
})
await createRows([{ isTrue: true }, { isTrue: false }])
@ -429,38 +462,35 @@ describe.each([
{ name: "serverDate", appointment: serverTime.toISOString() },
{
name: "single user, session user",
single_user: JSON.stringify(currentUser),
single_user: currentUser,
},
{
name: "single user",
single_user: JSON.stringify(globalUsers[0]),
single_user: globalUsers[0],
},
{
name: "deprecated single user, session user",
deprecated_single_user: JSON.stringify([currentUser]),
deprecated_single_user: [currentUser],
},
{
name: "deprecated single user",
deprecated_single_user: JSON.stringify([globalUsers[0]]),
deprecated_single_user: [globalUsers[0]],
},
{
name: "multi user",
multi_user: JSON.stringify(globalUsers),
multi_user: globalUsers,
},
{
name: "multi user with session user",
multi_user: JSON.stringify([...globalUsers, currentUser]),
multi_user: [...globalUsers, currentUser],
},
{
name: "deprecated multi user",
deprecated_multi_user: JSON.stringify(globalUsers),
deprecated_multi_user: globalUsers,
},
{
name: "deprecated multi user with session user",
deprecated_multi_user: JSON.stringify([
...globalUsers,
currentUser,
]),
deprecated_multi_user: [...globalUsers, currentUser],
},
]
}
@ -482,7 +512,7 @@ describe.each([
})
)
table = await createTable({
tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING },
appointment: { name: "appointment", type: FieldType.DATETIME },
single_user: {
@ -632,9 +662,11 @@ describe.each([
})
it("should match the session user id in a multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
const allUsers = [...globalUsers, config.getUser()].map(
(user: any) => {
return { _id: user._id }
})
}
)
await expectQuery({
contains: { multi_user: ["{{ [user]._id }}"] },
@ -647,9 +679,11 @@ describe.each([
})
it("should match the session user id in a deprecated multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
const allUsers = [...globalUsers, config.getUser()].map(
(user: any) => {
return { _id: user._id }
})
}
)
await expectQuery({
contains: { deprecated_multi_user: ["{{ [user]._id }}"] },
@ -764,7 +798,7 @@ describe.each([
describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING },
})
await createRows([{ name: "foo" }, { name: "bar" }])
@ -791,6 +825,8 @@ describe.each([
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
// onEmptyFilter cannot be sent to view searches
!isView &&
it("should return nothing if onEmptyFilter is RETURN_NONE", async () => {
await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@ -891,6 +927,8 @@ describe.each([
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
// onEmptyFilter cannot be sent to view searches
!isView &&
it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@ -1055,7 +1093,7 @@ describe.each([
datasourceId: datasource!._id!,
})
table = resp.datasource.entities![tableName]
tableOrViewId = resp.datasource.entities![tableName]._id!
await createRows([{ name: "foo" }, { name: "bar" }])
})
@ -1079,7 +1117,7 @@ describe.each([
describe("numbers", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER },
})
await createRows([{ age: 1 }, { age: 10 }])
@ -1087,7 +1125,9 @@ describe.each([
describe("equal", () => {
it("successfully finds a row", async () => {
await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }])
await expectQuery({ equal: { age: 1 } }).toContainExactly([
{ age: 1 },
])
})
it("fails to find nonexistent row", async () => {
@ -1252,7 +1292,7 @@ describe.each([
const JAN_10TH = "2020-01-10T00:00:00.000Z"
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
dob: { name: "dob", type: FieldType.DATETIME },
})
@ -1324,25 +1364,33 @@ describe.each([
it("greater than equal to", async () => {
await expectQuery({
range: { dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() } },
range: {
dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_10TH }])
})
it("greater than", async () => {
await expectQuery({
range: { dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() } },
range: {
dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_10TH }])
})
it("less than equal to", async () => {
await expectQuery({
range: { dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() } },
range: {
dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_1ST }])
})
it("less than", async () => {
await expectQuery({
range: { dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() } },
range: {
dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_1ST }])
})
})
@ -1399,7 +1447,7 @@ describe.each([
const NULL_TIME__ID = `null_time__id`
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
timeid: { name: "timeid", type: FieldType.STRING },
time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
})
@ -1560,7 +1608,7 @@ describe.each([
describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
numbers: {
name: "numbers",
type: FieldType.ARRAY,
@ -1575,9 +1623,9 @@ describe.each([
describe("contains", () => {
it("successfully finds a row", async () => {
await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([
{ numbers: ["one", "two"] },
])
await expectQuery({
contains: { numbers: ["one"] },
}).toContainExactly([{ numbers: ["one", "two"] }])
})
it("fails to find nonexistent row", async () => {
@ -1657,7 +1705,7 @@ describe.each([
let BIG = "9223372036854775807"
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
num: { name: "num", type: FieldType.BIGINT },
})
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@ -1762,7 +1810,7 @@ describe.each([
isInternal &&
describe("auto", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
auto: {
name: "auto",
type: FieldType.AUTO,
@ -1913,9 +1961,9 @@ describe.each([
// to specify an order for pagination to work.
it("is stable without a sort specified", async () => {
let { rows: fullRowList } = await config.api.row.search(
table._id!,
tableOrViewId,
{
tableId: table._id!,
tableId: tableOrViewId,
query: {},
}
)
@ -1925,8 +1973,8 @@ describe.each([
hasNextPage: boolean | undefined = true,
rowCount: number = 0
do {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
const response = await config.api.row.search(tableOrViewId, {
tableId: tableOrViewId,
limit: 1,
paginate: true,
query: {},
@ -1949,8 +1997,8 @@ describe.each([
// eslint-disable-next-line no-constant-condition
while (true) {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
const response = await config.api.row.search(tableOrViewId, {
tableId: tableOrViewId,
limit: 3,
query: {},
bookmark,
@ -1973,7 +2021,7 @@ describe.each([
describe("field name 1:name", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
"1:name": { name: "1:name", type: FieldType.STRING },
})
await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
@ -1993,8 +2041,7 @@ describe.each([
isSql &&
describe("related formulas", () => {
beforeAll(async () => {
const arrayTable = await createTable(
{
const arrayTable = await createTable({
name: { name: "name", type: FieldType.STRING },
array: {
name: "array",
@ -2004,17 +2051,14 @@ describe.each([
inclusion: ["option 1", "option 2"],
},
},
},
"array"
)
table = await createTable(
{
})
tableOrViewId = await createTableOrView({
relationship: {
type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE,
name: "relationship",
fieldName: "relate",
tableId: arrayTable._id!,
tableId: arrayTable,
constraints: {
type: "array",
},
@ -2026,21 +2070,19 @@ describe.each([
`let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
),
},
},
"main"
)
})
const arrayRows = await Promise.all([
config.api.row.save(arrayTable._id!, {
config.api.row.save(arrayTable, {
name: "foo",
array: ["option 1"],
}),
config.api.row.save(arrayTable._id!, {
config.api.row.save(arrayTable, {
name: "bar",
array: ["option 2"],
}),
])
await Promise.all([
config.api.row.save(table._id!, {
config.api.row.save(tableOrViewId, {
relationship: [arrayRows[0]._id, arrayRows[1]._id],
}),
])
@ -2059,7 +2101,7 @@ describe.each([
user1 = await config.createUser({ _id: `us_${utils.newid()}` })
user2 = await config.createUser({ _id: `us_${utils.newid()}` })
table = await createTable({
tableOrViewId = await createTableOrView({
user: {
name: "user",
type: FieldType.BB_REFERENCE_SINGLE,
@ -2067,11 +2109,7 @@ describe.each([
},
})
await createRows([
{ user: JSON.stringify(user1) },
{ user: JSON.stringify(user2) },
{ user: null },
])
await createRows([{ user: user1 }, { user: user2 }, { user: null }])
})
describe("equal", () => {
@ -2088,18 +2126,15 @@ describe.each([
describe("notEqual", () => {
it("successfully finds a row", async () => {
await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([
{ user: { _id: user2._id } },
{},
])
await expectQuery({ notEqual: { user: user1._id } }).toContainExactly(
[{ user: { _id: user2._id } }, {}]
)
})
it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
{ user: { _id: user1._id } },
{ user: { _id: user2._id } },
{},
])
await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly(
[{ user: { _id: user1._id } }, { user: { _id: user2._id } }, {}]
)
})
})
@ -2139,7 +2174,7 @@ describe.each([
user1 = await config.createUser({ _id: `us_${utils.newid()}` })
user2 = await config.createUser({ _id: `us_${utils.newid()}` })
table = await createTable({
tableOrViewId = await createTableOrView({
users: {
name: "users",
type: FieldType.BB_REFERENCE,
@ -2153,10 +2188,10 @@ describe.each([
})
await createRows([
{ number: 1, users: JSON.stringify([user1]) },
{ number: 2, users: JSON.stringify([user2]) },
{ number: 3, users: JSON.stringify([user1, user2]) },
{ number: 4, users: JSON.stringify([]) },
{ number: 1, users: [user1] },
{ number: 2, users: [user2] },
{ number: 3, users: [user1, user2] },
{ number: 4, users: [] },
])
})
@ -2182,7 +2217,9 @@ describe.each([
})
it("fails to find nonexistent row", async () => {
await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()
await expectQuery({
contains: { users: ["us_none"] },
}).toFindNothing()
})
})
@ -2249,9 +2286,10 @@ describe.each([
let productCategoryTable: Table, productCatRows: Row[]
beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables(
const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.ONE_TO_MANY
)
tableOrViewId = tableId
productCategoryTable = relatedTable
productCatRows = await Promise.all([
@ -2260,15 +2298,15 @@ describe.each([
])
await Promise.all([
config.api.row.save(table._id!, {
config.api.row.save(tableOrViewId, {
name: "foo",
productCat: [productCatRows[0]._id],
}),
config.api.row.save(table._id!, {
config.api.row.save(tableOrViewId, {
name: "bar",
productCat: [productCatRows[1]._id],
}),
config.api.row.save(table._id!, {
config.api.row.save(tableOrViewId, {
name: "baz",
productCat: [],
}),
@ -2301,10 +2339,11 @@ describe.each([
isSql &&
describe("big relations", () => {
beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables(
const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.MANY_TO_ONE
)
const mainRow = await config.api.row.save(table._id!, {
tableOrViewId = tableId
const mainRow = await config.api.row.save(tableOrViewId, {
name: "foo",
})
for (let i = 0; i < 11; i++) {
@ -2329,45 +2368,42 @@ describe.each([
})
;(isSqs || isLucene) &&
describe("relations to same table", () => {
let relatedTable: Table, relatedRows: Row[]
let relatedTable: string, relatedRows: Row[]
beforeAll(async () => {
relatedTable = await createTable(
{
relatedTable = await createTable({
name: { name: "name", type: FieldType.STRING },
},
"productCategory"
)
table = await createTable({
})
tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTable._id!,
tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTable._id!,
tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, { name: "foo" }),
config.api.row.save(relatedTable._id!, { name: "bar" }),
config.api.row.save(relatedTable._id!, { name: "baz" }),
config.api.row.save(relatedTable._id!, { name: "boo" }),
config.api.row.save(relatedTable, { name: "foo" }),
config.api.row.save(relatedTable, { name: "bar" }),
config.api.row.save(relatedTable, { name: "baz" }),
config.api.row.save(relatedTable, { name: "boo" }),
])
await Promise.all([
config.api.row.save(table._id!, {
config.api.row.save(tableOrViewId, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
}),
config.api.row.save(table._id!, {
config.api.row.save(tableOrViewId, {
name: "test2",
related1: [relatedRows[2]._id!],
related2: [relatedRows[3]._id!],
@ -2430,7 +2466,7 @@ describe.each([
isInternal &&
describe("no column error backwards compat", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
name: {
name: "name",
type: FieldType.STRING,
@ -2453,7 +2489,7 @@ describe.each([
!isLucene &&
describe("row counting", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
name: {
name: "name",
type: FieldType.STRING,
@ -2488,7 +2524,7 @@ describe.each([
describe("Invalid column definitions", () => {
beforeAll(async () => {
// need to create an invalid table - means ignoring typescript
table = await createTable({
tableOrViewId = await createTableOrView({
// @ts-ignore
invalid: {
type: FieldType.STRING,
@ -2518,7 +2554,7 @@ describe.each([
"special (%s) case",
column => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
[column]: {
name: column,
type: FieldType.STRING,
@ -2543,8 +2579,8 @@ describe.each([
describe("sample data", () => {
beforeAll(async () => {
await config.api.application.addSampleData(config.appId!)
table = DEFAULT_EMPLOYEE_TABLE_SCHEMA
rows = await config.api.row.fetch(table._id!)
tableOrViewId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id!
rows = await config.api.row.fetch(tableOrViewId)
})
it("should be able to search sample data", async () => {
@ -2567,7 +2603,7 @@ describe.each([
const earlyDate = "2024-07-03T10:00:00.000Z",
laterDate = "2024-07-03T11:00:00.000Z"
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
date: {
name: "date",
type: FieldType.DATETIME,
@ -2610,7 +2646,7 @@ describe.each([
"ชื่อผู้ใช้", // Thai for "username"
])("non-ascii column name: %s", name => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
[name]: {
name,
type: FieldType.STRING,
@ -2637,7 +2673,7 @@ describe.each([
isInternal &&
describe("space at end of column name", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
"name ": {
name: "name ",
type: FieldType.STRING,
@ -2672,7 +2708,7 @@ describe.each([
;(isSqs || isInMemory) &&
describe("space at start of column name", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
" name": {
name: " name",
type: FieldType.STRING,
@ -2703,9 +2739,10 @@ describe.each([
})
isSqs &&
!isView &&
describe("duplicate columns", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
name: {
name: "name",
type: FieldType.STRING,
@ -2713,7 +2750,7 @@ describe.each([
})
await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB()
const tableDoc = await db.get<Table>(table._id!)
const tableDoc = await db.get<Table>(tableOrViewId)
tableDoc.schema.Name = {
name: "Name",
type: FieldType.STRING,
@ -2747,7 +2784,7 @@ describe.each([
type: FieldType.STRING,
},
})
table = await createTable({
tableOrViewId = await createTableOrView({
name: {
name: "name",
type: FieldType.STRING,
@ -2756,15 +2793,15 @@ describe.each([
name: "rel",
type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_MANY,
tableId: toRelateTable._id!,
tableId: toRelateTable,
fieldName: "rel",
},
})
const [row1, row2] = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
config.api.row.save(toRelateTable, { name: "tag 1" }),
config.api.row.save(toRelateTable, { name: "tag 2" }),
])
row = await config.api.row.save(table._id!, {
row = await config.api.row.save(tableOrViewId, {
name: "product 1",
rel: [row1._id, row2._id],
})
@ -2783,7 +2820,7 @@ describe.each([
!isInternal &&
describe("search by composite key", () => {
beforeAll(async () => {
table = await config.api.table.save(
const table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
idColumn1: {
@ -2798,6 +2835,7 @@ describe.each([
primary: ["idColumn1", "idColumn2"],
})
)
tableOrViewId = table._id!
await createRows([{ idColumn1: 1, idColumn2: 2 }])
})
@ -2819,15 +2857,13 @@ describe.each([
isSql &&
describe("primaryDisplay", () => {
beforeAll(async () => {
let toRelateTable = await createTable({
let toRelateTableId = await createTable({
name: {
name: "name",
type: FieldType.STRING,
},
})
table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
tableOrViewId = await createTableOrView({
name: {
name: "name",
type: FieldType.STRING,
@ -2836,33 +2872,30 @@ describe.each([
name: "link",
type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE,
tableId: toRelateTable._id!,
tableId: toRelateTableId,
fieldName: "link",
},
},
})
)
toRelateTable = await config.api.table.get(toRelateTable._id!)
const toRelateTable = await config.api.table.get(toRelateTableId)
await config.api.table.save({
...toRelateTable,
primaryDisplay: "link",
})
const relatedRows = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "test" }),
config.api.row.save(toRelateTable._id!, { name: "related" }),
])
await Promise.all([
config.api.row.save(table._id!, {
await config.api.row.save(tableOrViewId, {
name: "test",
link: relatedRows.map(row => row._id),
}),
])
})
})
it("should be able to query, primary display on related table shouldn't be used", async () => {
// this test makes sure that if a relationship has been specified as the primary display on a table
// it is ignored and another column is used instead
await expectQuery({}).toContain([
{ name: "test", link: [{ primaryDisplay: "test" }] },
{ name: "test", link: [{ primaryDisplay: "related" }] },
])
})
})
@ -2870,7 +2903,7 @@ describe.each([
!isLucene &&
describe("$and", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING },
})
@ -2945,7 +2978,10 @@ describe.each([
await expect(
expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
conditions: [
{ equal: { age: 10 } },
"invalidCondition" as any,
],
},
}).toFindNothing()
).rejects.toThrow(
@ -2973,6 +3009,8 @@ describe.each([
)
})
// onEmptyFilter cannot be sent to view searches
!isView &&
it("returns no rows when onEmptyFilter set to none", async () => {
await expectSearch({
query: {
@ -2999,7 +3037,7 @@ describe.each([
!isLucene &&
describe("$or", () => {
beforeAll(async () => {
table = await createTable({
tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING },
})
@ -3123,6 +3161,8 @@ describe.each([
}).toContainExactly([{ age: 1, name: "Jane" }])
})
// onEmptyFilter cannot be sent to view searches
!isView &&
it("returns no rows when onEmptyFilter set to none", async () => {
await expectSearch({
query: {
@ -3159,20 +3199,20 @@ describe.each([
row[name] = i
}
const relatedTable = await createTable(relatedSchema)
table = await createTable({
tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTable._id!,
tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, row),
config.api.row.save(relatedTable, row),
])
await config.api.row.save(table._id!, {
await config.api.row.save(tableOrViewId, {
name: "foo",
related1: [relatedRows[0]._id],
})
@ -3188,3 +3228,4 @@ describe.each([
})
})
})
})

337
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,9 +22,10 @@ import {
RelationshipType,
TableSchema,
RenameColumn,
ViewFieldMetadata,
FeatureFlag,
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -154,7 +155,7 @@ describe.each([
})
it("can persist views with all fields", async () => {
const newView: Required<CreateViewRequest> = {
const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
name: generator.name(),
tableId: table._id!,
primaryDisplay: "id",
@ -176,6 +177,9 @@ describe.each([
visible: true,
},
},
uiMetadata: {
foo: "bar",
},
}
const res = await config.api.viewV2.create(newView)
@ -540,6 +544,33 @@ describe.each([
status: 201,
})
})
it("can create a view with calculation fields", async () => {
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
})
expect(Object.keys(view.schema!)).toHaveLength(1)
let sum = view.schema!.sum as NumericCalculationFieldMetadata
expect(sum).toBeDefined()
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
view = await config.api.viewV2.get(view.id)
sum = view.schema!.sum as NumericCalculationFieldMetadata
expect(sum).toBeDefined()
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
})
})
describe("update", () => {
@ -584,7 +615,7 @@ describe.each([
it("can update all fields", async () => {
const tableId = table._id!
const updatedData: Required<UpdateViewRequest> = {
const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
version: view.version,
id: view.id,
tableId,
@ -612,6 +643,9 @@ describe.each([
readonly: true,
},
},
uiMetadata: {
foo: "bar",
},
}
await config.api.viewV2.update(updatedData)
@ -836,6 +870,185 @@ describe.each([
}
)
})
!isLucene &&
describe("calculation views", () => {
let table: Table
let view: ViewV2
beforeEach(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
country: {
name: "country",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
country: {
visible: true,
},
age: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Steve",
age: 30,
country: "UK",
},
{
name: "Jane",
age: 31,
country: "UK",
},
{
name: "Ruari",
age: 32,
country: "USA",
},
{
name: "Alice",
age: 33,
country: "USA",
},
],
})
})
it("returns the expected rows prior to modification", async () => {
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
},
{
country: "UK",
age: 61,
},
])
)
})
it("can remove a group by field", async () => {
delete view.schema!.country
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows).toEqual(
expect.arrayContaining([
{
age: 126,
},
])
)
})
it("can remove a calculation field", async () => {
delete view.schema!.age
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(4)
// Because the removal of the calculation field actually makes this
// no longer a calculation view, these rows will now have _id and
// _rev fields.
expect(rows).toEqual(
expect.arrayContaining([
expect.objectContaining({ country: "UK" }),
expect.objectContaining({ country: "UK" }),
expect.objectContaining({ country: "USA" }),
expect.objectContaining({ country: "USA" }),
])
)
})
it("can add a new group by field", async () => {
view.schema!.name = { visible: true }
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(4)
expect(rows).toEqual(
expect.arrayContaining([
{
name: "Steve",
age: 30,
country: "UK",
},
{
name: "Jane",
age: 31,
country: "UK",
},
{
name: "Ruari",
age: 32,
country: "USA",
},
{
name: "Alice",
age: 33,
country: "USA",
},
])
)
})
it("can add a new calculation field", async () => {
view.schema!.count = {
visible: true,
calculationType: CalculationType.COUNT,
}
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
count: 2,
},
{
country: "UK",
age: 61,
count: 2,
},
])
)
})
})
})
describe("delete", () => {
@ -1152,10 +1365,7 @@ describe.each([
return table
}
const createView = async (
tableId: string,
schema: Record<string, ViewFieldMetadata>
) =>
const createView = async (tableId: string, schema: ViewV2Schema) =>
await config.api.viewV2.create({
name: generator.guid(),
tableId,
@ -2545,6 +2755,119 @@ describe.each([
expect(actual).toEqual(expected)
}
})
it("should be able to do a COUNT(DISTINCT)", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "name",
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "John",
},
{
name: "John",
},
{
name: "Sue",
},
],
})
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows[0].count).toEqual(2)
})
it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "does not exist oh no",
},
},
},
{
status: 400,
body: {
message:
'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema',
},
}
)
})
})
!isLucene &&
it("should not need required fields to be present", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
await Promise.all([
config.api.row.save(table._id!, { name: "Steve", age: 30 }),
config.api.row.save(table._id!, { name: "Jane", age: 31 }),
])
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].sum).toEqual(61)
})
})

4
packages/server/src/db/linkedRows/index.ts
View File

@ -23,8 +23,8 @@ import {
Row,
Table,
TableSchema,
ViewFieldMetadata,
ViewV2,
ViewV2Schema,
} from "@budibase/types"
import sdk from "../../sdk"
import { helpers } from "@budibase/shared-core"
@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
FeatureFlag.ENRICHED_RELATIONSHIPS
)
let viewSchema: Record<string, ViewFieldMetadata> = {}
let viewSchema: ViewV2Schema = {}
if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) {
return enriched

4
packages/server/src/jsRunner/index.ts
View File

@ -1,7 +1,7 @@
import { serializeError } from "serialize-error"
import env from "../environment"
import {
JsErrorTimeout,
JsTimeoutError,
setJSRunner,
setOnErrorLog,
} from "@budibase/string-templates"
@ -40,7 +40,7 @@ export function init() {
return vm.withContext(rest, () => vm.execute(js))
} catch (error: any) {
if (error.message === "Script execution timed out.") {
throw new JsErrorTimeout()
throw new JsTimeoutError()
}
throw error
}

5
packages/server/src/jsRunner/tests/jsRunner.spec.ts
View File

@ -41,10 +41,11 @@ describe("jsRunner (using isolated-vm)", () => {
})
it("should prevent sandbox escape", async () => {
const output = await processJS(
expect(
await processJS(
`return this.constructor.constructor("return process.env")()`
)
expect(output).toBe("Error while executing JS")
).toEqual("ReferenceError: process is not defined")
})
describe("helpers", () => {

25
packages/server/src/jsRunner/vm/isolated-vm.ts
View File

@ -7,14 +7,12 @@ import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles"
import { Snippet, VM } from "@budibase/types"
import { iifeWrapper } from "@budibase/string-templates"
import { iifeWrapper, UserScriptError } from "@budibase/string-templates"
import environment from "../../environment"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
export class JsRequestTimeoutError extends Error {
static code = "JS_REQUEST_TIMEOUT_ERROR"
code = JsRequestTimeoutError.code
}
export class IsolatedVM implements VM {
@ -29,6 +27,7 @@ export class IsolatedVM implements VM {
private readonly resultKey = "results"
private runResultKey: string
private runErrorKey: string
constructor({
memoryLimit,
@ -47,6 +46,7 @@ export class IsolatedVM implements VM {
this.jail.setSync("global", this.jail.derefInto())
this.runResultKey = crypto.randomUUID()
this.runErrorKey = crypto.randomUUID()
this.addToContext({
[this.resultKey]: { [this.runResultKey]: "" },
})
@ -210,13 +210,19 @@ export class IsolatedVM implements VM {
if (this.isolateAccumulatedTimeout) {
const cpuMs = Number(this.isolate.cpuTime) / 1e6
if (cpuMs > this.isolateAccumulatedTimeout) {
throw new ExecutionTimeoutError(
throw new JsRequestTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
)
}
}
code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
code = `
try {
results['${this.runResultKey}']=${this.codeWrapper(code)}
} catch (e) {
results['${this.runErrorKey}']=e
}
`
const script = this.isolate.compileScriptSync(code)
@ -227,6 +233,9 @@ export class IsolatedVM implements VM {
// We can't rely on the script run result as it will not work for non-transferable values
const result = this.getFromContext(this.resultKey)
if (result[this.runErrorKey]) {
throw new UserScriptError(result[this.runErrorKey])
}
return result[this.runResultKey]
}

21
packages/server/src/sdk/app/rowActions.ts
View File

@ -75,7 +75,7 @@ export async function create(tableId: string, rowAction: { name: string }) {
name: action.name,
automationId: automation._id!,
permissions: {
table: { runAllowed: false },
table: { runAllowed: true },
views: {},
},
}
@ -163,6 +163,23 @@ async function guardView(tableId: string, viewId: string) {
}
}
export async function setTablePermission(tableId: string, rowActionId: string) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = true
return actionsDoc
})
}
export async function unsetTablePermission(
tableId: string,
rowActionId: string
) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = false
return actionsDoc
})
}
export async function setViewPermission(
tableId: string,
rowActionId: string,
@ -220,6 +237,8 @@ export async function run(tableId: any, rowActionId: any, rowId: string) {
automation,
{
fields: {
id: row._id,
revision: row._rev,
row,
table,
},

50
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,12 +1,10 @@
import {
EmptyFilterOption,
LegacyFilter,
LogicalOperator,
Row,
RowSearchParams,
SearchFilter,
SearchFilterGroup,
SearchFilterKey,
SearchFilters,
SearchResponse,
SortOrder,
Table,
@ -63,7 +61,6 @@ export async function search(
if (options.viewId) {
source = await sdk.views.get(options.viewId)
table = await sdk.views.getTable(source)
options = searchInputMapping(table, options)
} else if (options.tableId) {
source = await sdk.tables.getTable(options.tableId)
table = source
@ -76,7 +73,7 @@ export async function search(
if (options.query) {
const visibleFields = (
options.fields || Object.keys(table.schema)
).filter(field => table.schema[field].visible !== false)
).filter(field => table.schema[field]?.visible !== false)
const queryableFields = await getQueryableFields(table, visibleFields)
options.query = removeInvalidFilters(options.query, queryableFields)
@ -84,40 +81,59 @@ export async function search(
options.query = {}
}
// need to make sure filters in correct shape before checking for view
options = searchInputMapping(table, options)
if (options.viewId) {
const view = await sdk.views.get(options.viewId)
// Delete extraneous search params that cannot be overridden
delete options.query.onEmptyFilter
const view = source as ViewV2
// Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access.
let viewQuery = dataFilters.buildQuery(view.query || [])
let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
delete viewQuery?.onEmptyFilter
if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
// Lucene does not accept conditional filters, so we need to keep the old logic
const query: SearchFilters = viewQuery || {}
const viewFilters = view.query as SearchFilter[]
const sqsEnabled = await features.flags.isEnabled("SQS")
const supportsLogicalOperators =
isExternalTableID(view.tableId) || sqsEnabled
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
delete options.query.onEmptyFilter
// Extract existing fields
const existingFields =
viewFilters
queryFilters
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
viewQuery ??= {}
// Carry over filters for unused fields
Object.keys(options.query || {}).forEach(key => {
Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = options.query[operator]![field]
viewQuery![operator]![field] = options.query[operator]![field]
}
})
})
options.query = query
options.query = viewQuery
} else {
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [viewQuery as SearchFilterGroup, options.query],
conditions: [...conditions, options.query],
},
}
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
}
}
@ -146,8 +162,6 @@ export async function search(
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
options = searchInputMapping(table, options)
let result: SearchResponse<Row>
if (isExternalTable) {
span?.addTags({ searchType: "external" })

17
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -1,5 +1,6 @@
import {
Aggregation,
CalculationType,
Datasource,
DocumentType,
FieldType,
@ -369,6 +370,21 @@ export async function search(
continue
}
if (field.calculationType === CalculationType.COUNT) {
if ("distinct" in field && field.distinct) {
aggregations.push({
name: key,
distinct: true,
field: mapToUserColumn(field.field),
calculationType: field.calculationType,
})
} else {
aggregations.push({
name: key,
calculationType: field.calculationType,
})
}
} else {
aggregations.push({
name: key,
field: mapToUserColumn(field.field),
@ -376,6 +392,7 @@ export async function search(
})
}
}
}
const request: QueryJson = {
endpoint: {

2
packages/server/src/sdk/app/rows/search/utils.ts
View File

@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
}
return dataFilters.recurseLogicalOperators(filters, checkFilters)
}
if (options.query) {
options.query = checkFilters(options.query)
}
return options
}

43
packages/server/src/sdk/app/views/index.ts
View File

@ -1,4 +1,5 @@
import {
CalculationType,
FieldType,
PermissionLevel,
RelationSchemaField,
@ -63,6 +64,15 @@ async function guardCalculationViewSchema(
const calculationFields = helpers.views.calculationFields(view)
for (const calculationFieldName of Object.keys(calculationFields)) {
const schema = calculationFields[calculationFieldName]
const isCount = schema.calculationType === CalculationType.COUNT
const isDistinct = isCount && "distinct" in schema && schema.distinct
// Count fields that aren't distinct don't need to reference another field,
// so we don't validate it.
if (isCount && !isDistinct) {
continue
}
const targetSchema = table.schema[schema.field]
if (!targetSchema) {
throw new HTTPError(
@ -71,7 +81,7 @@ async function guardCalculationViewSchema(
)
}
if (!helpers.schema.isNumeric(targetSchema)) {
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
400
@ -255,19 +265,12 @@ export async function enrichSchema(
view: ViewV2,
tableSchema: TableSchema
): Promise<ViewV2Enriched> {
const tableCache: Record<string, Table> = {}
async function populateRelTableSchema(
tableId: string,
viewFields: Record<string, RelationSchemaField>
) {
if (!tableCache[tableId]) {
tableCache[tableId] = await sdk.tables.getTable(tableId)
}
const relTable = tableCache[tableId]
const relTable = await sdk.tables.getTable(tableId)
const result: Record<string, ViewV2ColumnEnriched> = {}
for (const relTableFieldName of Object.keys(relTable.schema)) {
const relTableField = relTable.schema[relTableFieldName]
if (
@ -300,15 +303,24 @@ export async function enrichSchema(
const viewSchema = view.schema || {}
const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null)
for (const key of Object.keys(tableSchema).filter(
k => tableSchema[k].visible !== false
)) {
const visibleSchemaFields = Object.keys(viewSchema).filter(key => {
if (helpers.views.isCalculationField(viewSchema[key])) {
return viewSchema[key].visible !== false
}
return key in tableSchema && tableSchema[key].visible !== false
})
const visibleTableFields = Object.keys(tableSchema).filter(
key => tableSchema[key].visible !== false
)
const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields])
for (const key of visibleFields) {
// if nothing specified in view, then it is not visible
const ui = viewSchema[key] || { visible: false }
schema[key] = {
...tableSchema[key],
...ui,
order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order,
order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
columns: undefined,
}
@ -320,10 +332,7 @@ export async function enrichSchema(
}
}
return {
...view,
schema: schema,
}
return { ...view, schema }
}
export function syncSchema(

4
packages/server/src/tests/utilities/api/row.ts
View File

@ -7,11 +7,11 @@ import {
BulkImportRequest,
BulkImportResponse,
SearchRowResponse,
RowSearchParams,
DeleteRows,
DeleteRow,
PaginatedSearchRowResponse,
RowExportFormat,
SearchRowRequest,
} from "@budibase/types"
import { Expectations, TestAPI } from "./base"
@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
)
}
search = async <T extends RowSearchParams>(
search = async <T extends SearchRowRequest>(
sourceId: string,
params?: T,
expectations?: Expectations

36
packages/server/src/tests/utilities/api/rowAction.ts
View File

@ -72,6 +72,42 @@ export class RowActionAPI extends TestAPI {
)
}
setTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._post<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
unsetTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._delete<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
setViewPermission = async (
tableId: string,
viewId: string,

21
packages/shared-core/src/filters.ts
View File

@ -3,7 +3,7 @@ import {
BBReferenceFieldSubType,
FieldType,
FormulaType,
SearchFilter,
LegacyFilter,
SearchFilters,
SearchQueryFields,
ArrayOperator,
@ -129,7 +129,7 @@ export function recurseLogicalOperators(
fn: (f: SearchFilters) => SearchFilters
) {
for (const logical of LOGICAL_OPERATORS) {
if (filters?.[logical]) {
if (filters[logical]) {
filters[logical]!.conditions = filters[logical]!.conditions.map(
condition => fn(condition)
)
@ -165,9 +165,6 @@ export function recurseSearchFilters(
* https://github.com/Budibase/budibase/issues/10118
*/
export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) {
if (!query[filterField]) {
continue
@ -313,7 +310,7 @@ export class ColumnSplitter {
* @param filter the builder filter structure
*/
const buildCondition = (expression: SearchFilter) => {
const buildCondition = (expression: LegacyFilter) => {
// Filter body
let query: SearchFilters = {
string: {},
@ -439,8 +436,13 @@ const buildCondition = (expression: SearchFilter) => {
}
export const buildQueryLegacy = (
filter?: SearchFilterGroup | SearchFilter[]
filter?: LegacyFilter[] | SearchFilters
): SearchFilters | undefined => {
// this is of type SearchFilters or is undefined
if (!Array.isArray(filter)) {
return filter
}
let query: SearchFilters = {
string: {},
fuzzy: {},
@ -574,7 +576,7 @@ export const buildQueryLegacy = (
*/
export const buildQuery = (
filter?: SearchFilterGroup | SearchFilter[]
filter?: SearchFilterGroup | LegacyFilter[]
): SearchFilters | undefined => {
const parsedFilter: SearchFilterGroup | undefined =
processSearchFilters(filter)
@ -596,7 +598,7 @@ export const buildQuery = (
const globalOperator: LogicalOperator =
operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
const coreRequest: SearchFilters = {
return {
...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
[globalOperator]: {
conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@ -610,7 +612,6 @@ export const buildQuery = (
}),
},
}
return coreRequest
}
// The frontend can send single values for array fields sometimes, so to handle

201
packages/shared-core/src/utils.ts
View File

@ -1,5 +1,5 @@
import {
SearchFilter,
LegacyFilter,
SearchFilterGroup,
FilterGroupLogicalOperator,
SearchFilters,
@ -9,6 +9,10 @@ import {
import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
// an array of keys from filter type to properties that are in the type
// this can then be converted using .fromEntries to an object
type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
export function unreachable(
value: never,
message = `No such case in exhaustive switch: ${value}`
@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
return result
}
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {SearchFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: SearchFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterWhitelistKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
const migratedSetting: SearchFilterGroup = filters.reduce(
(acc: SearchFilterGroup, filter: SearchFilter) => {
// Sort the properties for easier debugging
const filterEntries = Object.entries(filter)
.sort((a, b) => {
return a[0].localeCompare(b[0])
})
.filter(x => x[1] ?? false)
if (filterEntries.length == 1) {
const [key, value] = filterEntries[0]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
(acc: [string, any][], entry: [string, any]) => {
const [key, value] = entry
if (filterWhitelistKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: SearchFilter = Object.fromEntries(
whiteListedFilterSettings
) as SearchFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
},
defaultCfg
)
return migratedSetting
} else if (!filters?.groups) {
return
}
return filters
}
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
{ op: BasicOperator.STRING, key: "email" },
@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
}
return true
}
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {LegacyFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: LegacyFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterAllowedKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
// Sort the properties for easier debugging
const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
.sort((a, b) => {
return a.localeCompare(b)
})
.filter(key => key in filter)
if (filterPropertyKeys.length == 1) {
const key = filterPropertyKeys[0],
value = filter[key]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
(acc: AllowedFilters, key) => {
const value = filter[key]
if (filterAllowedKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: LegacyFilter = Object.fromEntries(
allowedFilterSettings
) as LegacyFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
}, defaultCfg)
} else if (!filters?.groups) {
return
}
return filters
}

21
packages/string-templates/src/errors.ts
View File

@ -1,3 +1,20 @@
export class JsErrorTimeout extends Error {
code = "ERR_SCRIPT_EXECUTION_TIMEOUT"
export class JsTimeoutError extends Error {
static message = "Timed out while executing JS"
static code = "JS_TIMEOUT_ERROR"
code: string = JsTimeoutError.code
constructor() {
super(JsTimeoutError.message)
}
}
export class UserScriptError extends Error {
static code = "USER_SCRIPT_ERROR"
code: string = UserScriptError.code
constructor(readonly userScriptError: Error) {
super(
`error while running user-supplied JavaScript: ${userScriptError.toString()}`
)
}
}

42
packages/string-templates/src/helpers/javascript.ts
View File

@ -3,6 +3,7 @@ import cloneDeep from "lodash/fp/cloneDeep"
import { LITERAL_MARKER } from "../helpers/constants"
import { getJsHelperList } from "./list"
import { iifeWrapper } from "../iife"
import { JsTimeoutError, UserScriptError } from "../errors"
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.js or index.mjs).
@ -94,12 +95,49 @@ export function processJS(handlebars: string, context: any) {
} catch (error: any) {
onErrorLog && onErrorLog(error)
const { noThrow = true } = context.__opts || {}
// The error handling below is quite messy, because it has fallen to
// string-templates to handle a variety of types of error specific to usages
// above it in the stack. It would be nice some day to refactor this to
// allow each user of processStringSync to handle errors in the way they see
// fit.
// This is to catch the error vm.runInNewContext() throws when the timeout
// is exceeded.
if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
return "Timed out while executing JS"
}
if (error.name === "ExecutionTimeoutError") {
return "Request JS execution limit hit"
// This is to catch the JsRequestTimeoutError we throw when we detect a
// timeout across an entire request in the backend. We use a magic string
// because we can't import from the backend into string-templates.
if (error.code === "JS_REQUEST_TIMEOUT_ERROR") {
return error.message
}
// This is to catch the JsTimeoutError we throw when we detect a timeout in
// a single JS execution.
if (error.code === JsTimeoutError.code) {
return JsTimeoutError.message
}
// This is to catch the error that happens if a user-supplied JS script
// throws for reasons introduced by the user.
if (error.code === UserScriptError.code) {
if (noThrow) {
return error.userScriptError.toString()
}
throw error
}
if (error.name === "SyntaxError") {
if (noThrow) {
return error.toString()
}
throw error
}
return "Error while executing JS"
}
}

35
packages/string-templates/src/index.ts
View File

@ -16,6 +16,7 @@ import { removeJSRunner, setJSRunner } from "./helpers/javascript"
import manifest from "./manifest.json"
import { ProcessOptions } from "./types"
import { UserScriptError } from "./errors"
export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list"
export { FIND_ANY_HBS_REGEX } from "./utilities"
@ -122,7 +123,7 @@ function createTemplate(
export async function processObject<T extends Record<string, any>>(
object: T,
context: object,
opts?: { noHelpers?: boolean; escapeNewlines?: boolean; onlyFound?: boolean }
opts?: ProcessOptions
): Promise<T> {
testObject(object)
@ -172,7 +173,7 @@ export async function processString(
export function processObjectSync(
object: { [x: string]: any },
context: any,
opts: any
opts?: ProcessOptions
): object | Array<any> {
testObject(object)
for (let key of Object.keys(object || {})) {
@ -229,9 +230,13 @@ export function processStringSync(
} else {
return process(string)
}
} catch (err) {
} catch (err: any) {
const { noThrow = true } = opts || {}
if (noThrow) {
return input
}
throw err
}
}
/**
@ -448,7 +453,7 @@ export function convertToJS(hbs: string) {
return `${varBlock}${js}`
}
export { JsErrorTimeout } from "./errors"
export { JsTimeoutError, UserScriptError } from "./errors"
export function defaultJSSetup() {
if (!isBackendService()) {
@ -463,7 +468,27 @@ export function defaultJSSetup() {
setTimeout: undefined,
}
createContext(context)
return runInNewContext(js, context, { timeout: 1000 })
const wrappedJs = `
result = {
result: null,
error: null,
};
try {
result.result = ${js};
} catch (e) {
result.error = e;
}
result;
`
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
} else {
removeJSRunner()

1
packages/string-templates/src/types.ts
View File

@ -3,6 +3,7 @@ export interface ProcessOptions {
noEscaping?: boolean
noHelpers?: boolean
noFinalise?: boolean
noThrow?: boolean
escapeNewlines?: boolean
onlyFound?: boolean
disabledHelpers?: string[]

2
packages/types/src/api/web/app/rowAction.ts
View File

@ -8,7 +8,7 @@ export interface RowActionResponse extends RowActionData {
id: string
tableId: string
automationId: string
allowedViews: string[] | undefined
allowedSources: string[] | undefined
}
export interface RowActionsResponse {

1
packages/types/src/api/web/auth.ts
View File

@ -21,6 +21,7 @@ export interface UpdateSelfRequest {
freeTrialConfirmedAt?: string
appFavourites?: string[]
tours?: Record<string, Date>
appSort?: string
}
export interface UpdateSelfResponse {

5
packages/types/src/api/web/searchFilter.ts
View File

@ -5,7 +5,7 @@ import {
SearchFilters,
} from "../../sdk"
export type SearchFilter = {
export type LegacyFilter = {
operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
onEmptyFilter?: EmptyFilterOption
field: string
@ -14,9 +14,10 @@ export type SearchFilter = {
externalType?: string
}
// this is a type purely used by the UI
export type SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator
onEmptyFilter?: EmptyFilterOption
groups?: SearchFilterGroup[]
filters?: SearchFilter[]
filters?: LegacyFilter[]
}

6
packages/types/src/documents/app/rowAction.ts
View File

@ -8,8 +8,10 @@ export interface TableRowActions extends Document {
export interface RowActionData {
name: string
automationId: string
permissions: {
permissions: RowActionPermissions
}
export interface RowActionPermissions {
table: { runAllowed: boolean }
views: Record<string, { runAllowed: boolean }>
}
}

37
packages/types/src/documents/app/view.ts
View File

@ -1,7 +1,7 @@
import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table"
import { Document } from "../document"
import { DBView } from "../../sdk"
import { DBView, SearchFilters } from "../../sdk"
export type ViewTemplateOpts = {
field: string
@ -42,11 +42,31 @@ export interface RelationSchemaField extends UIFieldMetadata {
readonly?: boolean
}
export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata {
calculationType: CalculationType
export interface NumericCalculationFieldMetadata
extends BasicViewFieldMetadata {
calculationType:
| CalculationType.MIN
| CalculationType.MAX
| CalculationType.SUM
| CalculationType.AVG
field: string
}
export interface CountCalculationFieldMetadata extends BasicViewFieldMetadata {
calculationType: CalculationType.COUNT
}
export interface CountDistinctCalculationFieldMetadata
extends CountCalculationFieldMetadata {
distinct: true
field: string
}
export type ViewCalculationFieldMetadata =
| NumericCalculationFieldMetadata
| CountCalculationFieldMetadata
| CountDistinctCalculationFieldMetadata
export type ViewFieldMetadata =
| BasicViewFieldMetadata
| ViewCalculationFieldMetadata
@ -65,15 +85,20 @@ export interface ViewV2 {
name: string
primaryDisplay?: string
tableId: string
query?: SearchFilter[] | SearchFilterGroup
query?: LegacyFilter[] | SearchFilters
// duplicate to store UI information about filters
queryUI?: SearchFilterGroup
sort?: {
field: string
order?: SortOrder
type?: SortType
}
schema?: Record<string, ViewFieldMetadata>
schema?: ViewV2Schema
uiMetadata?: Record<string, any>
}
export type ViewV2Schema = Record<string, ViewFieldMetadata>
export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
export interface ViewCountOrSumSchema {

1
packages/types/src/documents/global/user.ts
View File

@ -67,6 +67,7 @@ export interface User extends Document {
scimInfo?: { isSync: true } & Record<string, any>
appFavourites?: string[]
ssoId?: string
appSort?: string
}
export enum UserStatus {

2
packages/types/src/sdk/ai.ts
View File

@ -63,7 +63,7 @@ type TranslateSchema = BaseSchema & {
type CategoriseTextSchema = BaseSchema & {
operation: AIOperationEnum.CATEGORISE_TEXT
columns: string[]
categories: string
categories: string[]
}
type SentimentAnalysisSchema = BaseSchema & {

25
packages/types/src/sdk/row.ts
View File

@ -3,12 +3,33 @@ import { SearchFilters } from "./search"
import { CalculationType, Row } from "../documents"
import { WithRequired } from "../shared"
export interface Aggregation {
export interface BaseAggregation {
name: string
calculationType: CalculationType
}
export interface NumericAggregation extends BaseAggregation {
calculationType:
| CalculationType.AVG
| CalculationType.MAX
| CalculationType.MIN
| CalculationType.SUM
field: string
}
export interface CountAggregation extends BaseAggregation {
calculationType: CalculationType.COUNT
}
export interface CountDistinctAggregation extends CountAggregation {
distinct: true
field: string
}
export type Aggregation =
| NumericAggregation
| CountAggregation
| CountDistinctAggregation
export interface SearchParams {
tableId?: string
viewId?: string

1
packages/worker/src/api/routes/validation/users.ts
View File

@ -29,6 +29,7 @@ export const buildSelfSaveValidation = () => {
freeTrialConfirmedAt: Joi.string().optional(),
appFavourites: Joi.array().optional(),
tours: Joi.object().optional(),
appSort: Joi.string().optional(),
}
return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
}

22
yarn.lock
View File

@ -2051,7 +2051,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.10":
"@budibase/backend-core@2.32.11":
version "0.0.0"
dependencies:
"@budibase/nano" "10.1.5"
@ -2132,15 +2132,15 @@
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "2.32.10"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.10.tgz#19fcbb3ced74791a7e96dfdc5a1270165792eea5"
integrity sha512-TbVp2bjmA0rHK+TKi9NVW06+O23fhDm7IJ/FlpWPHIBIZW7xDkCYu6LUOhSwSWMbOTcWzaJFuMbpN1HoTc/YjQ==
version "2.32.11"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.11.tgz#c94d534f829ca0ef252677757e157a7e58b87b4d"
integrity sha512-mOkqJpqHKWsfTWZwWcvBCYFUIluSUHltQNinc1ZRsg9rC3OKoHSDop6gzm744++H/GzGRN8V86kLhCgtNIlkpA==
dependencies:
"@anthropic-ai/sdk" "^0.27.3"
"@budibase/backend-core" "2.32.10"
"@budibase/shared-core" "2.32.10"
"@budibase/string-templates" "2.32.10"
"@budibase/types" "2.32.10"
"@budibase/backend-core" "2.32.11"
"@budibase/shared-core" "2.32.11"
"@budibase/string-templates" "2.32.11"
"@budibase/types" "2.32.11"
"@koa/router" "8.0.8"
bull "4.10.1"
dd-trace "5.2.0"
@ -2153,13 +2153,13 @@
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.10":
"@budibase/shared-core@2.32.11":
version "0.0.0"
dependencies:
"@budibase/types" "0.0.0"
cron-validate "1.4.5"
"@budibase/string-templates@2.32.10":
"@budibase/string-templates@2.32.11":
version "0.0.0"
dependencies:
"@budibase/handlebars-helpers" "^0.13.2"
@ -2167,7 +2167,7 @@
handlebars "^4.7.8"
lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.10":
"@budibase/types@2.32.11":
version "0.0.0"
dependencies:
scim-patch "^0.8.1"