MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/block-native-searching

This commit is contained in:
Andrew Kingston 2024-11-18 16:39:35 +00:00 committed by GitHub
parent e972f4b95b 668df22176
commit bac448c29a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 197 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.3",
"version": "3.2.5",
"npmClient": "yarn",
"packages": [
"packages/*",

1
packages/bbui/package.json
View File

@ -81,6 +81,7 @@
"@spectrum-css/typography": "3.0.1",
"@spectrum-css/underlay": "2.0.9",
"@spectrum-css/vars": "3.0.1",
"atrament": "^4.3.0",
"dayjs": "^1.10.8",
"easymde": "^2.16.1",
"svelte-dnd-action": "^0.9.8",

16
packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte
View File

@ -1,4 +1,5 @@
<script>
import { tick } from "svelte"
import {
ModalContent,
TextArea,
@ -8,7 +9,6 @@
import { automationStore, selectedAutomation } from "stores/builder"
import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
import { cloneDeep } from "lodash/fp"
import { memo } from "@budibase/frontend-core"
import { AutomationEventType } from "@budibase/types"
let failedParse = null
@ -63,8 +63,7 @@
return true
}
const memoTestData = memo(parseTestData($selectedAutomation.data.testData))
$: memoTestData.set(parseTestData($selectedAutomation.data.testData))
$: testData = testData || parseTestData($selectedAutomation.data.testData)
$: {
// clone the trigger so we're not mutating the reference
@ -83,7 +82,7 @@
$: isError =
!isTriggerValid(trigger) ||
!(trigger.schema.outputs.required || []).every(
required => $memoTestData?.[required] || required !== "row"
required => testData?.[required] || required !== "row"
)
function parseTestJSON(e) {
@ -110,11 +109,10 @@
}
const testAutomation = async () => {
// Ensure testData reactiveness is processed
await tick()
try {
await automationStore.actions.test(
$selectedAutomation.data,
$memoTestData
)
await automationStore.actions.test($selectedAutomation.data, testData)
$automationStore.showTestPanel = true
} catch (error) {
notifications.error(error)
@ -152,7 +150,7 @@
{#if selectedValues}
<div class="tab-content-padding">
<AutomationBlockSetup
testData={$memoTestData}
bind:testData
{schemaProperties}
isTestModal
block={trigger}

10
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -503,7 +503,15 @@
row: { "Active": true, "Order Id" : 14, ... }
})
*/
const onChange = Utils.sequential(async update => {
const onChange = async update => {
if (isTestModal) {
testData = update
}
updateAutomation(update)
}
const updateAutomation = Utils.sequential(async update => {
const request = cloneDeep(update)
// Process app trigger updates
if (isTrigger && !isTestModal) {

21
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -371,6 +371,7 @@
delete editableColumn.relationshipType
delete editableColumn.formulaType
delete editableColumn.constraints
delete editableColumn.responseType
// Add in defaults and initial definition
const definition = fieldDefinitions[type?.toUpperCase()]
@ -386,6 +387,7 @@
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
} else if (editableColumn.type === FieldType.FORMULA) {
editableColumn.formulaType = "dynamic"
editableColumn.responseType = field.responseType || FIELDS.STRING.type
}
}
@ -767,6 +769,25 @@
</div>
</div>
{/if}
<div class="split-label">
<div class="label-length">
<Label size="M">Response Type</Label>
</div>
<div class="input-length">
<Select
bind:value={editableColumn.responseType}
options={[
FIELDS.STRING,
FIELDS.NUMBER,
FIELDS.BOOLEAN,
FIELDS.DATETIME,
]}
getOptionLabel={option => option.name}
getOptionValue={option => option.type}
tooltip="Formulas by default will return a string - however if you need another type the response can be coerced."
/>
</div>
</div>
<div class="split-label">
<div class="label-length">
<Label size="M">Formula</Label>

3
packages/client/package.json
View File

@ -33,8 +33,7 @@
"sanitize-html": "^2.13.0",
"screenfull": "^6.0.1",
"shortid": "^2.2.15",
"svelte-spa-router": "^4.0.1",
"atrament": "^4.3.0"
"svelte-spa-router": "^4.0.1"
},
"devDependencies": {
"@rollup/plugin-alias": "^5.1.0",

18
packages/frontend-core/src/components/grid/cells/FormulaCell.svelte
View File

@ -1,5 +1,21 @@
<script>
import TextCell from "./TextCell.svelte"
import DateCell from "./DateCell.svelte"
import NumberCell from "./NumberCell.svelte"
import BooleanCell from "./BooleanCell.svelte"
import { FieldType } from "@budibase/types"
export let schema
$: responseType = schema.responseType
</script>
<TextCell {...$$props} readonly />
{#if responseType === FieldType.NUMBER}
<NumberCell {...$$props} readonly />
{:else if responseType === FieldType.BOOLEAN}
<BooleanCell {...$$props} readonly />
{:else if responseType === FieldType.DATETIME}
<DateCell {...$$props} readonly />
{:else}
<TextCell {...$$props} readonly />
{/if}

105
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -32,6 +32,7 @@ import {
JsonFieldSubType,
RowExportFormat,
RelationSchemaField,
FormulaResponseType,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import _, { merge } from "lodash"
@ -40,6 +41,7 @@ import { Knex } from "knex"
import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
import { isDate } from "../../../utilities"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
@ -79,6 +81,10 @@ async function waitForEvent(
return await p
}
function encodeJS(binding: string) {
return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
}
datasourceDescribe(
{ name: "/rows (%s)", exclude: [DatabaseName.MONGODB] },
({ config, dsProvider, isInternal, isMSSQL, isOracle }) => {
@ -3199,7 +3205,7 @@ datasourceDescribe(
describe("Formula fields", () => {
let table: Table
let otherTable: Table
let relatedRow: Row
let relatedRow: Row, mainRow: Row
beforeAll(async () => {
otherTable = await config.api.table.save(defaultTable())
@ -3227,7 +3233,7 @@ datasourceDescribe(
name: generator.word(),
description: generator.paragraph(),
})
await config.api.row.save(table._id!, {
mainRow = await config.api.row.save(table._id!, {
name: generator.word(),
description: generator.paragraph(),
tableId: table._id!,
@ -3235,6 +3241,25 @@ datasourceDescribe(
})
})
async function updateFormulaColumn(
formula: string,
opts?: { responseType?: FormulaResponseType; formulaType?: FormulaType }
) {
table = await config.api.table.save({
...table,
schema: {
...table.schema,
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: formula,
responseType: opts?.responseType,
formulaType: opts?.formulaType || FormulaType.DYNAMIC,
},
},
})
}
it("should be able to search for rows containing formulas", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
@ -3242,12 +3267,72 @@ datasourceDescribe(
const row = rows[0]
expect(row.formula).toBe(relatedRow.name)
})
it("should coerce - number response type", async () => {
await updateFormulaColumn(encodeJS("return 1"), {
responseType: FieldType.NUMBER,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(1)
})
it("should coerce - boolean response type", async () => {
await updateFormulaColumn(encodeJS("return true"), {
responseType: FieldType.BOOLEAN,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(true)
})
it("should coerce - datetime response type", async () => {
await updateFormulaColumn(encodeJS("return new Date()"), {
responseType: FieldType.DATETIME,
})
const { rows } = await config.api.row.search(table._id!)
expect(isDate(rows[0].formula)).toBe(true)
})
it("should coerce - datetime with invalid value", async () => {
await updateFormulaColumn(encodeJS("return 'a'"), {
responseType: FieldType.DATETIME,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBeUndefined()
})
it("should coerce handlebars", async () => {
await updateFormulaColumn("{{ add 1 1 }}", {
responseType: FieldType.NUMBER,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(2)
})
it("should coerce handlebars to string (default)", async () => {
await updateFormulaColumn("{{ add 1 1 }}", {
responseType: FieldType.STRING,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe("2")
})
isInternal &&
it("should coerce a static handlebars formula", async () => {
await updateFormulaColumn(encodeJS("return 1"), {
responseType: FieldType.NUMBER,
formulaType: FormulaType.STATIC,
})
// save the row to store the static value
await config.api.row.save(table._id!, mainRow)
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(1)
})
})
describe("Formula JS protection", () => {
it("should time out JS execution if a single cell takes too long", async () => {
await withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => {
const js = Buffer.from(
const js = encodeJS(
`
let i = 0;
while (true) {
@ -3255,7 +3340,7 @@ datasourceDescribe(
}
return i;
`
).toString("base64")
)
const table = await config.api.table.save(
saveTableRequest({
@ -3267,7 +3352,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
@ -3290,7 +3375,7 @@ datasourceDescribe(
JS_PER_REQUEST_TIMEOUT_MS: 80,
},
async () => {
const js = Buffer.from(
const js = encodeJS(
`
let i = 0;
while (true) {
@ -3298,7 +3383,7 @@ datasourceDescribe(
}
return i;
`
).toString("base64")
)
const table = await config.api.table.save(
saveTableRequest({
@ -3310,7 +3395,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
@ -3352,7 +3437,7 @@ datasourceDescribe(
})
it("should not carry over context between formulas", async () => {
const js = Buffer.from(`return $("[text]");`).toString("base64")
const js = encodeJS(`return $("[text]");`)
const table = await config.api.table.save(
saveTableRequest({
schema: {
@ -3363,7 +3448,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formula: js,
formulaType: FormulaType.DYNAMIC,
},
},

18
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -161,33 +161,33 @@ async function processDefaultValues(table: Table, row: Row) {
/**
* This will coerce a value to the correct types based on the type transform map
* @param row The value to coerce
* @param value The value to coerce
* @param type The type fo coerce to
* @returns The coerced value
*/
export function coerce(row: any, type: string) {
export function coerce(value: unknown, type: string) {
// no coercion specified for type, skip it
if (!TYPE_TRANSFORM_MAP[type]) {
return row
return value
}
// eslint-disable-next-line no-prototype-builtins
if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) {
// @ts-ignore
return TYPE_TRANSFORM_MAP[type][row]
return TYPE_TRANSFORM_MAP[type][value]
} else if (TYPE_TRANSFORM_MAP[type].parse) {
// @ts-ignore
return TYPE_TRANSFORM_MAP[type].parse(row)
return TYPE_TRANSFORM_MAP[type].parse(value)
}
return row
return value
}
/**
* Given an input route this function will apply all the necessary pre-processing to it, such as coercion
* of column values or adding auto-column values.
* @param user the user which is performing the input.
* @param userId the ID of the user which is performing the input.
* @param row the row which is being created/updated.
* @param table the table which the row is being saved to.
* @param source the table/view which the row is being saved to.
* @param opts some input processing options (like disabling auto-column relationships).
* @returns the row which has been prepared to be written to the DB.
*/

33
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -10,11 +10,13 @@ import {
FieldType,
OperationFieldTypeEnum,
AIOperationEnum,
AIFieldMetadata,
} from "@budibase/types"
import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace"
import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
import { coerce } from "./index"
interface FormulaOpts {
dynamic?: boolean
@ -67,7 +69,18 @@ export async function processFormulas<T extends Row | Row[]>(
continue
}
const responseType = schema.responseType
const isStatic = schema.formulaType === FormulaType.STATIC
const formula = schema.formula
// coerce static values
if (isStatic) {
rows.forEach(row => {
if (row[column] && responseType) {
row[column] = coerce(row[column], responseType)
}
})
}
if (
schema.formula == null ||
@ -80,12 +93,18 @@ export async function processFormulas<T extends Row | Row[]>(
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
let formula = schema.formula
rows[i] = {
...row,
[column]: tracer.trace("processStringSync", {}, span => {
span?.addTags({ table_id: table._id, column, static: isStatic })
return processStringSync(formula, context)
const result = processStringSync(formula, context)
try {
return responseType ? coerce(result, responseType) : result
} catch (err: any) {
// if the coercion fails, we return empty row contents
span?.addTags({ coercionError: err.message })
return undefined
}
}),
}
}
@ -117,12 +136,13 @@ export async function processAIColumns<T extends Row | Row[]>(
continue
}
const operation = schema.operation
const aiSchema: AIFieldMetadata = schema
const rowUpdates = rows.map((row, i) => {
const contextRow = contextRows ? contextRows[i] : row
// Check if the type is bindable and pass through HBS if so
const operationField =
OperationFields[schema.operation as AIOperationEnum]
const operationField = OperationFields[operation as AIOperationEnum]
for (const key in schema) {
const fieldType = operationField[key as keyof typeof operationField]
if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
@ -131,7 +151,10 @@ export async function processAIColumns<T extends Row | Row[]>(
}
}
const prompt = llm.buildPromptFromAIOperation({ schema, row })
const prompt = llm.buildPromptFromAIOperation({
schema: aiSchema,
row,
})
return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column })

6
packages/types/src/documents/app/row.ts
View File

@ -134,6 +134,12 @@ export const JsonTypes = [
FieldType.ARRAY,
]
export type FormulaResponseType =
| FieldType.STRING
| FieldType.NUMBER
| FieldType.BOOLEAN
| FieldType.DATETIME
export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT]
export function isNumeric(type: FieldType) {

3
packages/types/src/documents/app/table/schema.ts
View File

@ -1,6 +1,6 @@
// all added by grid/table when defining the
// column size, position and whether it can be viewed
import { FieldType } from "../row"
import { FieldType, FormulaResponseType } from "../row"
import {
AutoFieldSubType,
AutoReason,
@ -115,6 +115,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
type: FieldType.FORMULA
formula: string
formulaType?: FormulaType
responseType?: FormulaResponseType
}
export interface AIFieldMetadata extends BaseFieldSchema {