Merge branch 'master' into fix/block-native-searching

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.3",
+  "version": "3.2.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/bbui/package.json

@@ -81,6 +81,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
+    "atrament": "^4.3.0",
     "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-dnd-action": "^0.9.8",

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { tick } from "svelte"
   import {
     ModalContent,
     TextArea,
@@ -8,7 +9,6 @@
   import { automationStore, selectedAutomation } from "stores/builder"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import { cloneDeep } from "lodash/fp"
-  import { memo } from "@budibase/frontend-core"
   import { AutomationEventType } from "@budibase/types"
 
   let failedParse = null
@@ -63,8 +63,7 @@
     return true
   }
 
-  const memoTestData = memo(parseTestData($selectedAutomation.data.testData))
+  $: testData = testData || parseTestData($selectedAutomation.data.testData)
-  $: memoTestData.set(parseTestData($selectedAutomation.data.testData))
 
   $: {
     // clone the trigger so we're not mutating the reference
@@ -83,7 +82,7 @@
   $: isError =
     !isTriggerValid(trigger) ||
     !(trigger.schema.outputs.required || []).every(
-      required => $memoTestData?.[required] || required !== "row"
+      required => testData?.[required] || required !== "row"
     )
 
   function parseTestJSON(e) {
@@ -110,11 +109,10 @@
   }
 
   const testAutomation = async () => {
+    // Ensure testData reactiveness is processed
+    await tick()
     try {
-      await automationStore.actions.test(
+      await automationStore.actions.test($selectedAutomation.data, testData)
-        $selectedAutomation.data,
-        $memoTestData
-      )
       $automationStore.showTestPanel = true
     } catch (error) {
       notifications.error(error)
@@ -152,7 +150,7 @@
   {#if selectedValues}
     <div class="tab-content-padding">
       <AutomationBlockSetup
-        testData={$memoTestData}
+        bind:testData
         {schemaProperties}
         isTestModal
         block={trigger}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -503,7 +503,15 @@
       row: { "Active": true, "Order Id" : 14, ... }
     })
    */
-  const onChange = Utils.sequential(async update => {
+  const onChange = async update => {
+    if (isTestModal) {
+      testData = update
+    }
+
+    updateAutomation(update)
+  }
+
+  const updateAutomation = Utils.sequential(async update => {
     const request = cloneDeep(update)
     // Process app trigger updates
     if (isTrigger && !isTestModal) {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -371,6 +371,7 @@
     delete editableColumn.relationshipType
     delete editableColumn.formulaType
     delete editableColumn.constraints
+    delete editableColumn.responseType
 
     // Add in defaults and initial definition
     const definition = fieldDefinitions[type?.toUpperCase()]
@@ -386,6 +387,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
+      editableColumn.responseType = field.responseType || FIELDS.STRING.type
     }
   }
 
@@ -767,6 +769,25 @@
         </div>
       </div>
     {/if}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Response Type</Label>
+      </div>
+      <div class="input-length">
+        <Select
+          bind:value={editableColumn.responseType}
+          options={[
+            FIELDS.STRING,
+            FIELDS.NUMBER,
+            FIELDS.BOOLEAN,
+            FIELDS.DATETIME,
+          ]}
+          getOptionLabel={option => option.name}
+          getOptionValue={option => option.type}
+          tooltip="Formulas by default will return a string - however if you need another type the response can be coerced."
+        />
+      </div>
+    </div>
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Formula</Label>

packages/client/package.json

@@ -33,8 +33,7 @@
     "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
-    "svelte-spa-router": "^4.0.1",
+    "svelte-spa-router": "^4.0.1"
-    "atrament": "^4.3.0"
   },
   "devDependencies": {
     "@rollup/plugin-alias": "^5.1.0",

packages/frontend-core/src/components/grid/cells/FormulaCell.svelte

@@ -1,5 +1,21 @@
 <script>
   import TextCell from "./TextCell.svelte"
+  import DateCell from "./DateCell.svelte"
+  import NumberCell from "./NumberCell.svelte"
+  import BooleanCell from "./BooleanCell.svelte"
+  import { FieldType } from "@budibase/types"
+
+  export let schema
+
+  $: responseType = schema.responseType
 </script>
 
-<TextCell {...$$props} readonly />
+{#if responseType === FieldType.NUMBER}
+  <NumberCell {...$$props} readonly />
+{:else if responseType === FieldType.BOOLEAN}
+  <BooleanCell {...$$props} readonly />
+{:else if responseType === FieldType.DATETIME}
+  <DateCell {...$$props} readonly />
+{:else}
+  <TextCell {...$$props} readonly />
+{/if}

packages/server/src/api/routes/tests/row.spec.ts

@@ -32,6 +32,7 @@ import {
   JsonFieldSubType,
   RowExportFormat,
   RelationSchemaField,
+  FormulaResponseType,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import _, { merge } from "lodash"
@@ -40,6 +41,7 @@ import { Knex } from "knex"
 import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
+import { isDate } from "../../../utilities"
 
 jest.mock("@budibase/pro", () => ({
   ...jest.requireActual("@budibase/pro"),
@@ -79,6 +81,10 @@ async function waitForEvent(
   return await p
 }
 
+function encodeJS(binding: string) {
+  return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
+}
+
 datasourceDescribe(
   { name: "/rows (%s)", exclude: [DatabaseName.MONGODB] },
   ({ config, dsProvider, isInternal, isMSSQL, isOracle }) => {
@@ -3199,7 +3205,7 @@ datasourceDescribe(
     describe("Formula fields", () => {
       let table: Table
       let otherTable: Table
-      let relatedRow: Row
+      let relatedRow: Row, mainRow: Row
 
       beforeAll(async () => {
         otherTable = await config.api.table.save(defaultTable())
@@ -3227,7 +3233,7 @@ datasourceDescribe(
           name: generator.word(),
           description: generator.paragraph(),
         })
-        await config.api.row.save(table._id!, {
+        mainRow = await config.api.row.save(table._id!, {
           name: generator.word(),
           description: generator.paragraph(),
           tableId: table._id!,
@@ -3235,6 +3241,25 @@ datasourceDescribe(
         })
       })
 
+      async function updateFormulaColumn(
+        formula: string,
+        opts?: { responseType?: FormulaResponseType; formulaType?: FormulaType }
+      ) {
+        table = await config.api.table.save({
+          ...table,
+          schema: {
+            ...table.schema,
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: formula,
+              responseType: opts?.responseType,
+              formulaType: opts?.formulaType || FormulaType.DYNAMIC,
+            },
+          },
+        })
+      }
+
       it("should be able to search for rows containing formulas", async () => {
         const { rows } = await config.api.row.search(table._id!)
         expect(rows.length).toBe(1)
@@ -3242,12 +3267,72 @@ datasourceDescribe(
         const row = rows[0]
         expect(row.formula).toBe(relatedRow.name)
       })
+
+      it("should coerce - number response type", async () => {
+        await updateFormulaColumn(encodeJS("return 1"), {
+          responseType: FieldType.NUMBER,
+        })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows[0].formula).toBe(1)
+      })
+
+      it("should coerce - boolean response type", async () => {
+        await updateFormulaColumn(encodeJS("return true"), {
+          responseType: FieldType.BOOLEAN,
+        })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows[0].formula).toBe(true)
+      })
+
+      it("should coerce - datetime response type", async () => {
+        await updateFormulaColumn(encodeJS("return new Date()"), {
+          responseType: FieldType.DATETIME,
+        })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(isDate(rows[0].formula)).toBe(true)
+      })
+
+      it("should coerce - datetime with invalid value", async () => {
+        await updateFormulaColumn(encodeJS("return 'a'"), {
+          responseType: FieldType.DATETIME,
+        })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows[0].formula).toBeUndefined()
+      })
+
+      it("should coerce handlebars", async () => {
+        await updateFormulaColumn("{{ add 1 1 }}", {
+          responseType: FieldType.NUMBER,
+        })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows[0].formula).toBe(2)
+      })
+
+      it("should coerce handlebars to string (default)", async () => {
+        await updateFormulaColumn("{{ add 1 1 }}", {
+          responseType: FieldType.STRING,
+        })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows[0].formula).toBe("2")
+      })
+
+      isInternal &&
+        it("should coerce a static handlebars formula", async () => {
+          await updateFormulaColumn(encodeJS("return 1"), {
+            responseType: FieldType.NUMBER,
+            formulaType: FormulaType.STATIC,
+          })
+          // save the row to store the static value
+          await config.api.row.save(table._id!, mainRow)
+          const { rows } = await config.api.row.search(table._id!)
+          expect(rows[0].formula).toBe(1)
+        })
     })
 
     describe("Formula JS protection", () => {
       it("should time out JS execution if a single cell takes too long", async () => {
         await withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => {
-          const js = Buffer.from(
+          const js = encodeJS(
             `
               let i = 0;
               while (true) {
@@ -3255,7 +3340,7 @@ datasourceDescribe(
               }
               return i;
             `
-          ).toString("base64")
+          )
 
           const table = await config.api.table.save(
             saveTableRequest({
@@ -3267,7 +3352,7 @@ datasourceDescribe(
                 formula: {
                   name: "formula",
                   type: FieldType.FORMULA,
-                  formula: `{{ js "${js}"}}`,
+                  formula: js,
                   formulaType: FormulaType.DYNAMIC,
                 },
               },
@@ -3290,7 +3375,7 @@ datasourceDescribe(
             JS_PER_REQUEST_TIMEOUT_MS: 80,
           },
           async () => {
-            const js = Buffer.from(
+            const js = encodeJS(
               `
               let i = 0;
               while (true) {
@@ -3298,7 +3383,7 @@ datasourceDescribe(
               }
               return i;
             `
-            ).toString("base64")
+            )
 
             const table = await config.api.table.save(
               saveTableRequest({
@@ -3310,7 +3395,7 @@ datasourceDescribe(
                   formula: {
                     name: "formula",
                     type: FieldType.FORMULA,
-                    formula: `{{ js "${js}"}}`,
+                    formula: js,
                     formulaType: FormulaType.DYNAMIC,
                   },
                 },
@@ -3352,7 +3437,7 @@ datasourceDescribe(
       })
 
       it("should not carry over context between formulas", async () => {
-        const js = Buffer.from(`return $("[text]");`).toString("base64")
+        const js = encodeJS(`return $("[text]");`)
         const table = await config.api.table.save(
           saveTableRequest({
             schema: {
@@ -3363,7 +3448,7 @@ datasourceDescribe(
               formula: {
                 name: "formula",
                 type: FieldType.FORMULA,
-                formula: `{{ js "${js}"}}`,
+                formula: js,
                 formulaType: FormulaType.DYNAMIC,
               },
             },

packages/server/src/utilities/rowProcessor/index.ts

@@ -161,33 +161,33 @@ async function processDefaultValues(table: Table, row: Row) {
 
 /**
  * This will coerce a value to the correct types based on the type transform map
- * @param row The value to coerce
+ * @param value The value to coerce
  * @param type The type fo coerce to
  * @returns The coerced value
  */
-export function coerce(row: any, type: string) {
+export function coerce(value: unknown, type: string) {
   // no coercion specified for type, skip it
   if (!TYPE_TRANSFORM_MAP[type]) {
-    return row
+    return value
   }
   // eslint-disable-next-line no-prototype-builtins
-  if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
+  if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) {
     // @ts-ignore
-    return TYPE_TRANSFORM_MAP[type][row]
+    return TYPE_TRANSFORM_MAP[type][value]
   } else if (TYPE_TRANSFORM_MAP[type].parse) {
     // @ts-ignore
-    return TYPE_TRANSFORM_MAP[type].parse(row)
+    return TYPE_TRANSFORM_MAP[type].parse(value)
   }
 
-  return row
+  return value
 }
 
 /**
  * Given an input route this function will apply all the necessary pre-processing to it, such as coercion
  * of column values or adding auto-column values.
- * @param user the user which is performing the input.
+ * @param userId the ID of the user which is performing the input.
  * @param row the row which is being created/updated.
- * @param table the table which the row is being saved to.
+ * @param source the table/view which the row is being saved to.
  * @param opts some input processing options (like disabling auto-column relationships).
  * @returns the row which has been prepared to be written to the DB.
  */

packages/server/src/utilities/rowProcessor/utils.ts

@@ -10,11 +10,13 @@ import {
   FieldType,
   OperationFieldTypeEnum,
   AIOperationEnum,
+  AIFieldMetadata,
 } from "@budibase/types"
 import { OperationFields } from "@budibase/shared-core"
 import tracer from "dd-trace"
 import { context } from "@budibase/backend-core"
 import * as pro from "@budibase/pro"
+import { coerce } from "./index"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -67,7 +69,18 @@ export async function processFormulas<T extends Row | Row[]>(
           continue
         }
 
+        const responseType = schema.responseType
         const isStatic = schema.formulaType === FormulaType.STATIC
+        const formula = schema.formula
+
+        // coerce static values
+        if (isStatic) {
+          rows.forEach(row => {
+            if (row[column] && responseType) {
+              row[column] = coerce(row[column], responseType)
+            }
+          })
+        }
 
         if (
           schema.formula == null ||
@@ -80,12 +93,18 @@ export async function processFormulas<T extends Row | Row[]>(
         for (let i = 0; i < rows.length; i++) {
           let row = rows[i]
           let context = contextRows ? contextRows[i] : row
-          let formula = schema.formula
           rows[i] = {
             ...row,
             [column]: tracer.trace("processStringSync", {}, span => {
               span?.addTags({ table_id: table._id, column, static: isStatic })
-              return processStringSync(formula, context)
+              const result = processStringSync(formula, context)
+              try {
+                return responseType ? coerce(result, responseType) : result
+              } catch (err: any) {
+                // if the coercion fails, we return empty row contents
+                span?.addTags({ coercionError: err.message })
+                return undefined
+              }
             }),
           }
         }
@@ -117,12 +136,13 @@ export async function processAIColumns<T extends Row | Row[]>(
           continue
         }
 
+        const operation = schema.operation
+        const aiSchema: AIFieldMetadata = schema
         const rowUpdates = rows.map((row, i) => {
           const contextRow = contextRows ? contextRows[i] : row
 
           // Check if the type is bindable and pass through HBS if so
-          const operationField =
+          const operationField = OperationFields[operation as AIOperationEnum]
-            OperationFields[schema.operation as AIOperationEnum]
           for (const key in schema) {
             const fieldType = operationField[key as keyof typeof operationField]
             if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
@@ -131,7 +151,10 @@ export async function processAIColumns<T extends Row | Row[]>(
             }
           }
 
-          const prompt = llm.buildPromptFromAIOperation({ schema, row })
+          const prompt = llm.buildPromptFromAIOperation({
+            schema: aiSchema,
+            row,
+          })
 
           return tracer.trace("processAIColumn", {}, async span => {
             span?.addTags({ table_id: table._id, column })

packages/types/src/documents/app/row.ts

@@ -134,6 +134,12 @@ export const JsonTypes = [
   FieldType.ARRAY,
 ]
 
+export type FormulaResponseType =
+  | FieldType.STRING
+  | FieldType.NUMBER
+  | FieldType.BOOLEAN
+  | FieldType.DATETIME
+
 export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT]
 
 export function isNumeric(type: FieldType) {

packages/types/src/documents/app/table/schema.ts

@@ -1,6 +1,6 @@
 // all added by grid/table when defining the
 // column size, position and whether it can be viewed
-import { FieldType } from "../row"
+import { FieldType, FormulaResponseType } from "../row"
 import {
   AutoFieldSubType,
   AutoReason,
@@ -115,6 +115,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   type: FieldType.FORMULA
   formula: string
   formulaType?: FormulaType
+  responseType?: FormulaResponseType
 }
 
 export interface AIFieldMetadata extends BaseFieldSchema {