diff --git a/lerna.json b/lerna.json index 582f95b303..e5b0013535 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.3", + "version": "3.2.5", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 0830f8ab6f..aeb7418526 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -81,6 +81,7 @@ "@spectrum-css/typography": "3.0.1", "@spectrum-css/underlay": "2.0.9", "@spectrum-css/vars": "3.0.1", + "atrament": "^4.3.0", "dayjs": "^1.10.8", "easymde": "^2.16.1", "svelte-dnd-action": "^0.9.8", diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte index f134c787ca..5ec66870a8 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte @@ -1,4 +1,5 @@ - +{#if responseType === FieldType.NUMBER} + +{:else if responseType === FieldType.BOOLEAN} + +{:else if responseType === FieldType.DATETIME} + +{:else} + +{/if} diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 7d65345584..5bdd341beb 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -32,6 +32,7 @@ import { JsonFieldSubType, RowExportFormat, RelationSchemaField, + FormulaResponseType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import _, { merge } from "lodash" @@ -40,6 +41,7 @@ import { Knex } from "knex" import { InternalTables } from "../../../db/utils" import { withEnv } from "../../../environment" import { JsTimeoutError } from "@budibase/string-templates" +import { isDate } from "../../../utilities" jest.mock("@budibase/pro", () => ({ ...jest.requireActual("@budibase/pro"), @@ -79,6 +81,10 @@ async function waitForEvent( return await p } +function encodeJS(binding: string) { + return `{{ js "${Buffer.from(binding).toString("base64")}"}}` +} + datasourceDescribe( { name: "/rows (%s)", exclude: [DatabaseName.MONGODB] }, ({ config, dsProvider, isInternal, isMSSQL, isOracle }) => { @@ -3199,7 +3205,7 @@ datasourceDescribe( describe("Formula fields", () => { let table: Table let otherTable: Table - let relatedRow: Row + let relatedRow: Row, mainRow: Row beforeAll(async () => { otherTable = await config.api.table.save(defaultTable()) @@ -3227,7 +3233,7 @@ datasourceDescribe( name: generator.word(), description: generator.paragraph(), }) - await config.api.row.save(table._id!, { + mainRow = await config.api.row.save(table._id!, { name: generator.word(), description: generator.paragraph(), tableId: table._id!, @@ -3235,6 +3241,25 @@ datasourceDescribe( }) }) + async function updateFormulaColumn( + formula: string, + opts?: { responseType?: FormulaResponseType; formulaType?: FormulaType } + ) { + table = await config.api.table.save({ + ...table, + schema: { + ...table.schema, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: formula, + responseType: opts?.responseType, + formulaType: opts?.formulaType || FormulaType.DYNAMIC, + }, + }, + }) + } + it("should be able to search for rows containing formulas", async () => { const { rows } = await config.api.row.search(table._id!) expect(rows.length).toBe(1) @@ -3242,12 +3267,72 @@ datasourceDescribe( const row = rows[0] expect(row.formula).toBe(relatedRow.name) }) + + it("should coerce - number response type", async () => { + await updateFormulaColumn(encodeJS("return 1"), { + responseType: FieldType.NUMBER, + }) + const { rows } = await config.api.row.search(table._id!) + expect(rows[0].formula).toBe(1) + }) + + it("should coerce - boolean response type", async () => { + await updateFormulaColumn(encodeJS("return true"), { + responseType: FieldType.BOOLEAN, + }) + const { rows } = await config.api.row.search(table._id!) + expect(rows[0].formula).toBe(true) + }) + + it("should coerce - datetime response type", async () => { + await updateFormulaColumn(encodeJS("return new Date()"), { + responseType: FieldType.DATETIME, + }) + const { rows } = await config.api.row.search(table._id!) + expect(isDate(rows[0].formula)).toBe(true) + }) + + it("should coerce - datetime with invalid value", async () => { + await updateFormulaColumn(encodeJS("return 'a'"), { + responseType: FieldType.DATETIME, + }) + const { rows } = await config.api.row.search(table._id!) + expect(rows[0].formula).toBeUndefined() + }) + + it("should coerce handlebars", async () => { + await updateFormulaColumn("{{ add 1 1 }}", { + responseType: FieldType.NUMBER, + }) + const { rows } = await config.api.row.search(table._id!) + expect(rows[0].formula).toBe(2) + }) + + it("should coerce handlebars to string (default)", async () => { + await updateFormulaColumn("{{ add 1 1 }}", { + responseType: FieldType.STRING, + }) + const { rows } = await config.api.row.search(table._id!) + expect(rows[0].formula).toBe("2") + }) + + isInternal && + it("should coerce a static handlebars formula", async () => { + await updateFormulaColumn(encodeJS("return 1"), { + responseType: FieldType.NUMBER, + formulaType: FormulaType.STATIC, + }) + // save the row to store the static value + await config.api.row.save(table._id!, mainRow) + const { rows } = await config.api.row.search(table._id!) + expect(rows[0].formula).toBe(1) + }) }) describe("Formula JS protection", () => { it("should time out JS execution if a single cell takes too long", async () => { await withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => { - const js = Buffer.from( + const js = encodeJS( ` let i = 0; while (true) { @@ -3255,7 +3340,7 @@ datasourceDescribe( } return i; ` - ).toString("base64") + ) const table = await config.api.table.save( saveTableRequest({ @@ -3267,7 +3352,7 @@ datasourceDescribe( formula: { name: "formula", type: FieldType.FORMULA, - formula: `{{ js "${js}"}}`, + formula: js, formulaType: FormulaType.DYNAMIC, }, }, @@ -3290,7 +3375,7 @@ datasourceDescribe( JS_PER_REQUEST_TIMEOUT_MS: 80, }, async () => { - const js = Buffer.from( + const js = encodeJS( ` let i = 0; while (true) { @@ -3298,7 +3383,7 @@ datasourceDescribe( } return i; ` - ).toString("base64") + ) const table = await config.api.table.save( saveTableRequest({ @@ -3310,7 +3395,7 @@ datasourceDescribe( formula: { name: "formula", type: FieldType.FORMULA, - formula: `{{ js "${js}"}}`, + formula: js, formulaType: FormulaType.DYNAMIC, }, }, @@ -3352,7 +3437,7 @@ datasourceDescribe( }) it("should not carry over context between formulas", async () => { - const js = Buffer.from(`return $("[text]");`).toString("base64") + const js = encodeJS(`return $("[text]");`) const table = await config.api.table.save( saveTableRequest({ schema: { @@ -3363,7 +3448,7 @@ datasourceDescribe( formula: { name: "formula", type: FieldType.FORMULA, - formula: `{{ js "${js}"}}`, + formula: js, formulaType: FormulaType.DYNAMIC, }, }, diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 907dcb1de4..7d6d537302 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -161,33 +161,33 @@ async function processDefaultValues(table: Table, row: Row) { /** * This will coerce a value to the correct types based on the type transform map - * @param row The value to coerce + * @param value The value to coerce * @param type The type fo coerce to * @returns The coerced value */ -export function coerce(row: any, type: string) { +export function coerce(value: unknown, type: string) { // no coercion specified for type, skip it if (!TYPE_TRANSFORM_MAP[type]) { - return row + return value } // eslint-disable-next-line no-prototype-builtins - if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) { + if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) { // @ts-ignore - return TYPE_TRANSFORM_MAP[type][row] + return TYPE_TRANSFORM_MAP[type][value] } else if (TYPE_TRANSFORM_MAP[type].parse) { // @ts-ignore - return TYPE_TRANSFORM_MAP[type].parse(row) + return TYPE_TRANSFORM_MAP[type].parse(value) } - return row + return value } /** * Given an input route this function will apply all the necessary pre-processing to it, such as coercion * of column values or adding auto-column values. - * @param user the user which is performing the input. + * @param userId the ID of the user which is performing the input. * @param row the row which is being created/updated. - * @param table the table which the row is being saved to. + * @param source the table/view which the row is being saved to. * @param opts some input processing options (like disabling auto-column relationships). * @returns the row which has been prepared to be written to the DB. */ diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 33aba5eb3a..9dbeb8ebb2 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -10,11 +10,13 @@ import { FieldType, OperationFieldTypeEnum, AIOperationEnum, + AIFieldMetadata, } from "@budibase/types" import { OperationFields } from "@budibase/shared-core" import tracer from "dd-trace" import { context } from "@budibase/backend-core" import * as pro from "@budibase/pro" +import { coerce } from "./index" interface FormulaOpts { dynamic?: boolean @@ -67,7 +69,18 @@ export async function processFormulas( continue } + const responseType = schema.responseType const isStatic = schema.formulaType === FormulaType.STATIC + const formula = schema.formula + + // coerce static values + if (isStatic) { + rows.forEach(row => { + if (row[column] && responseType) { + row[column] = coerce(row[column], responseType) + } + }) + } if ( schema.formula == null || @@ -80,12 +93,18 @@ export async function processFormulas( for (let i = 0; i < rows.length; i++) { let row = rows[i] let context = contextRows ? contextRows[i] : row - let formula = schema.formula rows[i] = { ...row, [column]: tracer.trace("processStringSync", {}, span => { span?.addTags({ table_id: table._id, column, static: isStatic }) - return processStringSync(formula, context) + const result = processStringSync(formula, context) + try { + return responseType ? coerce(result, responseType) : result + } catch (err: any) { + // if the coercion fails, we return empty row contents + span?.addTags({ coercionError: err.message }) + return undefined + } }), } } @@ -117,12 +136,13 @@ export async function processAIColumns( continue } + const operation = schema.operation + const aiSchema: AIFieldMetadata = schema const rowUpdates = rows.map((row, i) => { const contextRow = contextRows ? contextRows[i] : row // Check if the type is bindable and pass through HBS if so - const operationField = - OperationFields[schema.operation as AIOperationEnum] + const operationField = OperationFields[operation as AIOperationEnum] for (const key in schema) { const fieldType = operationField[key as keyof typeof operationField] if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) { @@ -131,7 +151,10 @@ export async function processAIColumns( } } - const prompt = llm.buildPromptFromAIOperation({ schema, row }) + const prompt = llm.buildPromptFromAIOperation({ + schema: aiSchema, + row, + }) return tracer.trace("processAIColumn", {}, async span => { span?.addTags({ table_id: table._id, column }) diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index b0c5267b37..6b6b38a5cf 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -134,6 +134,12 @@ export const JsonTypes = [ FieldType.ARRAY, ] +export type FormulaResponseType = + | FieldType.STRING + | FieldType.NUMBER + | FieldType.BOOLEAN + | FieldType.DATETIME + export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT] export function isNumeric(type: FieldType) { diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 7e79902a49..771192e2f5 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -1,6 +1,6 @@ // all added by grid/table when defining the // column size, position and whether it can be viewed -import { FieldType } from "../row" +import { FieldType, FormulaResponseType } from "../row" import { AutoFieldSubType, AutoReason, @@ -115,6 +115,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema { type: FieldType.FORMULA formula: string formulaType?: FormulaType + responseType?: FormulaResponseType } export interface AIFieldMetadata extends BaseFieldSchema {