diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index d16bca3203..17069aa8c6 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -371,6 +371,7 @@
delete editableColumn.relationshipType
delete editableColumn.formulaType
delete editableColumn.constraints
+ delete editableColumn.responseType
// Add in defaults and initial definition
const definition = fieldDefinitions[type?.toUpperCase()]
@@ -386,6 +387,7 @@
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
} else if (editableColumn.type === FieldType.FORMULA) {
editableColumn.formulaType = "dynamic"
+ editableColumn.responseType = field.responseType || FIELDS.STRING.type
}
}
@@ -767,6 +769,25 @@
{/if}
+
diff --git a/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte b/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte
index b4db795e44..3dabbb94c0 100644
--- a/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte
@@ -1,5 +1,21 @@
-
+{#if responseType === FieldType.NUMBER}
+
+{:else if responseType === FieldType.BOOLEAN}
+
+{:else if responseType === FieldType.DATETIME}
+
+{:else}
+
+{/if}
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 7d65345584..5bdd341beb 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -32,6 +32,7 @@ import {
JsonFieldSubType,
RowExportFormat,
RelationSchemaField,
+ FormulaResponseType,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import _, { merge } from "lodash"
@@ -40,6 +41,7 @@ import { Knex } from "knex"
import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
+import { isDate } from "../../../utilities"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
@@ -79,6 +81,10 @@ async function waitForEvent(
return await p
}
+function encodeJS(binding: string) {
+ return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
+}
+
datasourceDescribe(
{ name: "/rows (%s)", exclude: [DatabaseName.MONGODB] },
({ config, dsProvider, isInternal, isMSSQL, isOracle }) => {
@@ -3199,7 +3205,7 @@ datasourceDescribe(
describe("Formula fields", () => {
let table: Table
let otherTable: Table
- let relatedRow: Row
+ let relatedRow: Row, mainRow: Row
beforeAll(async () => {
otherTable = await config.api.table.save(defaultTable())
@@ -3227,7 +3233,7 @@ datasourceDescribe(
name: generator.word(),
description: generator.paragraph(),
})
- await config.api.row.save(table._id!, {
+ mainRow = await config.api.row.save(table._id!, {
name: generator.word(),
description: generator.paragraph(),
tableId: table._id!,
@@ -3235,6 +3241,25 @@ datasourceDescribe(
})
})
+ async function updateFormulaColumn(
+ formula: string,
+ opts?: { responseType?: FormulaResponseType; formulaType?: FormulaType }
+ ) {
+ table = await config.api.table.save({
+ ...table,
+ schema: {
+ ...table.schema,
+ formula: {
+ name: "formula",
+ type: FieldType.FORMULA,
+ formula: formula,
+ responseType: opts?.responseType,
+ formulaType: opts?.formulaType || FormulaType.DYNAMIC,
+ },
+ },
+ })
+ }
+
it("should be able to search for rows containing formulas", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
@@ -3242,12 +3267,72 @@ datasourceDescribe(
const row = rows[0]
expect(row.formula).toBe(relatedRow.name)
})
+
+ it("should coerce - number response type", async () => {
+ await updateFormulaColumn(encodeJS("return 1"), {
+ responseType: FieldType.NUMBER,
+ })
+ const { rows } = await config.api.row.search(table._id!)
+ expect(rows[0].formula).toBe(1)
+ })
+
+ it("should coerce - boolean response type", async () => {
+ await updateFormulaColumn(encodeJS("return true"), {
+ responseType: FieldType.BOOLEAN,
+ })
+ const { rows } = await config.api.row.search(table._id!)
+ expect(rows[0].formula).toBe(true)
+ })
+
+ it("should coerce - datetime response type", async () => {
+ await updateFormulaColumn(encodeJS("return new Date()"), {
+ responseType: FieldType.DATETIME,
+ })
+ const { rows } = await config.api.row.search(table._id!)
+ expect(isDate(rows[0].formula)).toBe(true)
+ })
+
+ it("should coerce - datetime with invalid value", async () => {
+ await updateFormulaColumn(encodeJS("return 'a'"), {
+ responseType: FieldType.DATETIME,
+ })
+ const { rows } = await config.api.row.search(table._id!)
+ expect(rows[0].formula).toBeUndefined()
+ })
+
+ it("should coerce handlebars", async () => {
+ await updateFormulaColumn("{{ add 1 1 }}", {
+ responseType: FieldType.NUMBER,
+ })
+ const { rows } = await config.api.row.search(table._id!)
+ expect(rows[0].formula).toBe(2)
+ })
+
+ it("should coerce handlebars to string (default)", async () => {
+ await updateFormulaColumn("{{ add 1 1 }}", {
+ responseType: FieldType.STRING,
+ })
+ const { rows } = await config.api.row.search(table._id!)
+ expect(rows[0].formula).toBe("2")
+ })
+
+ isInternal &&
+ it("should coerce a static handlebars formula", async () => {
+ await updateFormulaColumn(encodeJS("return 1"), {
+ responseType: FieldType.NUMBER,
+ formulaType: FormulaType.STATIC,
+ })
+ // save the row to store the static value
+ await config.api.row.save(table._id!, mainRow)
+ const { rows } = await config.api.row.search(table._id!)
+ expect(rows[0].formula).toBe(1)
+ })
})
describe("Formula JS protection", () => {
it("should time out JS execution if a single cell takes too long", async () => {
await withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => {
- const js = Buffer.from(
+ const js = encodeJS(
`
let i = 0;
while (true) {
@@ -3255,7 +3340,7 @@ datasourceDescribe(
}
return i;
`
- ).toString("base64")
+ )
const table = await config.api.table.save(
saveTableRequest({
@@ -3267,7 +3352,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
- formula: `{{ js "${js}"}}`,
+ formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
@@ -3290,7 +3375,7 @@ datasourceDescribe(
JS_PER_REQUEST_TIMEOUT_MS: 80,
},
async () => {
- const js = Buffer.from(
+ const js = encodeJS(
`
let i = 0;
while (true) {
@@ -3298,7 +3383,7 @@ datasourceDescribe(
}
return i;
`
- ).toString("base64")
+ )
const table = await config.api.table.save(
saveTableRequest({
@@ -3310,7 +3395,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
- formula: `{{ js "${js}"}}`,
+ formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
@@ -3352,7 +3437,7 @@ datasourceDescribe(
})
it("should not carry over context between formulas", async () => {
- const js = Buffer.from(`return $("[text]");`).toString("base64")
+ const js = encodeJS(`return $("[text]");`)
const table = await config.api.table.save(
saveTableRequest({
schema: {
@@ -3363,7 +3448,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
- formula: `{{ js "${js}"}}`,
+ formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 907dcb1de4..7d6d537302 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -161,33 +161,33 @@ async function processDefaultValues(table: Table, row: Row) {
/**
* This will coerce a value to the correct types based on the type transform map
- * @param row The value to coerce
+ * @param value The value to coerce
* @param type The type fo coerce to
* @returns The coerced value
*/
-export function coerce(row: any, type: string) {
+export function coerce(value: unknown, type: string) {
// no coercion specified for type, skip it
if (!TYPE_TRANSFORM_MAP[type]) {
- return row
+ return value
}
// eslint-disable-next-line no-prototype-builtins
- if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
+ if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) {
// @ts-ignore
- return TYPE_TRANSFORM_MAP[type][row]
+ return TYPE_TRANSFORM_MAP[type][value]
} else if (TYPE_TRANSFORM_MAP[type].parse) {
// @ts-ignore
- return TYPE_TRANSFORM_MAP[type].parse(row)
+ return TYPE_TRANSFORM_MAP[type].parse(value)
}
- return row
+ return value
}
/**
* Given an input route this function will apply all the necessary pre-processing to it, such as coercion
* of column values or adding auto-column values.
- * @param user the user which is performing the input.
+ * @param userId the ID of the user which is performing the input.
* @param row the row which is being created/updated.
- * @param table the table which the row is being saved to.
+ * @param source the table/view which the row is being saved to.
* @param opts some input processing options (like disabling auto-column relationships).
* @returns the row which has been prepared to be written to the DB.
*/
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 33aba5eb3a..9dbeb8ebb2 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -10,11 +10,13 @@ import {
FieldType,
OperationFieldTypeEnum,
AIOperationEnum,
+ AIFieldMetadata,
} from "@budibase/types"
import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace"
import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
+import { coerce } from "./index"
interface FormulaOpts {
dynamic?: boolean
@@ -67,7 +69,18 @@ export async function processFormulas(
continue
}
+ const responseType = schema.responseType
const isStatic = schema.formulaType === FormulaType.STATIC
+ const formula = schema.formula
+
+ // coerce static values
+ if (isStatic) {
+ rows.forEach(row => {
+ if (row[column] && responseType) {
+ row[column] = coerce(row[column], responseType)
+ }
+ })
+ }
if (
schema.formula == null ||
@@ -80,12 +93,18 @@ export async function processFormulas(
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
- let formula = schema.formula
rows[i] = {
...row,
[column]: tracer.trace("processStringSync", {}, span => {
span?.addTags({ table_id: table._id, column, static: isStatic })
- return processStringSync(formula, context)
+ const result = processStringSync(formula, context)
+ try {
+ return responseType ? coerce(result, responseType) : result
+ } catch (err: any) {
+ // if the coercion fails, we return empty row contents
+ span?.addTags({ coercionError: err.message })
+ return undefined
+ }
}),
}
}
@@ -117,12 +136,13 @@ export async function processAIColumns(
continue
}
+ const operation = schema.operation
+ const aiSchema: AIFieldMetadata = schema
const rowUpdates = rows.map((row, i) => {
const contextRow = contextRows ? contextRows[i] : row
// Check if the type is bindable and pass through HBS if so
- const operationField =
- OperationFields[schema.operation as AIOperationEnum]
+ const operationField = OperationFields[operation as AIOperationEnum]
for (const key in schema) {
const fieldType = operationField[key as keyof typeof operationField]
if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
@@ -131,7 +151,10 @@ export async function processAIColumns(
}
}
- const prompt = llm.buildPromptFromAIOperation({ schema, row })
+ const prompt = llm.buildPromptFromAIOperation({
+ schema: aiSchema,
+ row,
+ })
return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column })
diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index b0c5267b37..6b6b38a5cf 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -134,6 +134,12 @@ export const JsonTypes = [
FieldType.ARRAY,
]
+export type FormulaResponseType =
+ | FieldType.STRING
+ | FieldType.NUMBER
+ | FieldType.BOOLEAN
+ | FieldType.DATETIME
+
export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT]
export function isNumeric(type: FieldType) {
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 7e79902a49..771192e2f5 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -1,6 +1,6 @@
// all added by grid/table when defining the
// column size, position and whether it can be viewed
-import { FieldType } from "../row"
+import { FieldType, FormulaResponseType } from "../row"
import {
AutoFieldSubType,
AutoReason,
@@ -115,6 +115,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
type: FieldType.FORMULA
formula: string
formulaType?: FormulaType
+ responseType?: FormulaResponseType
}
export interface AIFieldMetadata extends BaseFieldSchema {