MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into chore/detach-account-portal

This commit is contained in:
Adria Navarro 2024-11-19 13:30:44 +01:00 committed by GitHub
parent cfd092638e cc8525f53c
commit bdce481f96
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 254 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.4",
"version": "3.2.6",
"npmClient": "yarn",
"concurrency": 20,
"command": {

4
packages/backend-core/.npmignore
View File

@ -1,6 +1,4 @@
*
!dist/**/*
dist/tsconfig.build.tsbuildinfo
!package.json
!src/**
!tests/**
!package.json

7
packages/backend-core/package.json
View File

@ -9,6 +9,13 @@
"./tests": "./dist/tests/index.js",
"./*": "./dist/*.js"
},
"typesVersions": {
"*": {
"tests": [
"dist/tests/index.d.ts"
]
}
},
"author": "Budibase",
"license": "GPL-3.0",
"scripts": {

7
packages/backend-core/src/environment.ts
View File

@ -19,6 +19,12 @@ function isDev() {
return process.env.NODE_ENV !== "production"
}
function parseIntSafe(number?: string) {
if (number) {
return parseInt(number)
}
}
let LOADED = false
if (!LOADED && isDev() && !isTest()) {
require("dotenv").config()
@ -231,6 +237,7 @@ const environment = {
MIN_VERSION_WITHOUT_POWER_ROLE:
process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
}
export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

21
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -371,6 +371,7 @@
delete editableColumn.relationshipType
delete editableColumn.formulaType
delete editableColumn.constraints
delete editableColumn.responseType
// Add in defaults and initial definition
const definition = fieldDefinitions[type?.toUpperCase()]
@ -386,6 +387,7 @@
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
} else if (editableColumn.type === FieldType.FORMULA) {
editableColumn.formulaType = "dynamic"
editableColumn.responseType = field.responseType || FIELDS.STRING.type
}
}
@ -767,6 +769,25 @@
</div>
</div>
{/if}
<div class="split-label">
<div class="label-length">
<Label size="M">Response Type</Label>
</div>
<div class="input-length">
<Select
bind:value={editableColumn.responseType}
options={[
FIELDS.STRING,
FIELDS.NUMBER,
FIELDS.BOOLEAN,
FIELDS.DATETIME,
]}
getOptionLabel={option => option.name}
getOptionValue={option => option.type}
tooltip="Formulas by default will return a string - however if you need another type the response can be coerced."
/>
</div>
</div>
<div class="split-label">
<div class="label-length">
<Label size="M">Formula</Label>

27
packages/client/src/utils/blocks.js
View File

@ -1,5 +1,7 @@
import { makePropSafe as safe } from "@budibase/string-templates"
import { API } from "../api/index.js"
import { UILogicalOperator } from "@budibase/types"
import { OnEmptyFilter } from "@budibase/frontend-core/src/constants.js"
// Map of data types to component types for search fields inside blocks
const schemaComponentMap = {
@ -60,7 +62,11 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
* @param formId the ID of the form containing the search fields
*/
export const enrichFilter = (filter, columns, formId) => {
let enrichedFilter = [...(filter || [])]
if (!columns?.length) {
return filter
}
let newFilters = []
columns?.forEach(column => {
const safePath = column.name.split(".").map(safe).join(".")
const stringType = column.type === "string" || column.type === "formula"
@ -69,7 +75,7 @@ export const enrichFilter = (filter, columns, formId) => {
// For dates, use a range of the entire day selected
if (dateType) {
enrichedFilter.push({
newFilters.push({
field: column.name,
type: column.type,
operator: "rangeLow",
@ -79,7 +85,7 @@ export const enrichFilter = (filter, columns, formId) => {
const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
hbs = `{{#if ${binding} }}${hbs}{{/if}}`
enrichedFilter.push({
newFilters.push({
field: column.name,
type: column.type,
operator: "rangeHigh",
@ -90,7 +96,7 @@ export const enrichFilter = (filter, columns, formId) => {
// For other fields, do an exact match
else {
enrichedFilter.push({
newFilters.push({
field: column.name,
type: column.type,
operator: stringType ? "string" : "equal",
@ -99,5 +105,16 @@ export const enrichFilter = (filter, columns, formId) => {
})
}
})
return enrichedFilter
return {
logicalOperator: UILogicalOperator.ALL,
onEmptyFilter: OnEmptyFilter.RETURN_ALL,
groups: [
...(filter?.groups || []),
{
logicalOperator: UILogicalOperator.ALL,
filters: newFilters,
},
],
}
}

18
packages/frontend-core/src/components/grid/cells/FormulaCell.svelte
View File

@ -1,5 +1,21 @@
<script>
import TextCell from "./TextCell.svelte"
import DateCell from "./DateCell.svelte"
import NumberCell from "./NumberCell.svelte"
import BooleanCell from "./BooleanCell.svelte"
import { FieldType } from "@budibase/types"
export let schema
$: responseType = schema.responseType
</script>
<TextCell {...$$props} readonly />
{#if responseType === FieldType.NUMBER}
<NumberCell {...$$props} readonly />
{:else if responseType === FieldType.BOOLEAN}
<BooleanCell {...$$props} readonly />
{:else if responseType === FieldType.DATETIME}
<DateCell {...$$props} readonly />
{:else}
<TextCell {...$$props} readonly />
{/if}

3
packages/server/src/api/controllers/query/index.ts
View File

@ -23,6 +23,7 @@ import {
} from "@budibase/types"
import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
import { findHBSBlocks } from "@budibase/string-templates"
import { ObjectId } from "mongodb"
const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT,
@ -223,6 +224,8 @@ export async function preview(
} else {
fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
}
} else if (field instanceof ObjectId) {
fieldMetadata = makeQuerySchema(FieldType.STRING, key)
} else {
fieldMetadata = makeQuerySchema(FieldType.JSON, key)
}

105
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -32,6 +32,7 @@ import {
JsonFieldSubType,
RowExportFormat,
RelationSchemaField,
FormulaResponseType,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import _, { merge } from "lodash"
@ -40,6 +41,7 @@ import { Knex } from "knex"
import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
import { isDate } from "../../../utilities"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
@ -79,6 +81,10 @@ async function waitForEvent(
return await p
}
function encodeJS(binding: string) {
return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
}
datasourceDescribe(
{ name: "/rows (%s)", exclude: [DatabaseName.MONGODB] },
({ config, dsProvider, isInternal, isMSSQL, isOracle }) => {
@ -3199,7 +3205,7 @@ datasourceDescribe(
describe("Formula fields", () => {
let table: Table
let otherTable: Table
let relatedRow: Row
let relatedRow: Row, mainRow: Row
beforeAll(async () => {
otherTable = await config.api.table.save(defaultTable())
@ -3227,7 +3233,7 @@ datasourceDescribe(
name: generator.word(),
description: generator.paragraph(),
})
await config.api.row.save(table._id!, {
mainRow = await config.api.row.save(table._id!, {
name: generator.word(),
description: generator.paragraph(),
tableId: table._id!,
@ -3235,6 +3241,25 @@ datasourceDescribe(
})
})
async function updateFormulaColumn(
formula: string,
opts?: { responseType?: FormulaResponseType; formulaType?: FormulaType }
) {
table = await config.api.table.save({
...table,
schema: {
...table.schema,
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: formula,
responseType: opts?.responseType,
formulaType: opts?.formulaType || FormulaType.DYNAMIC,
},
},
})
}
it("should be able to search for rows containing formulas", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
@ -3242,12 +3267,72 @@ datasourceDescribe(
const row = rows[0]
expect(row.formula).toBe(relatedRow.name)
})
it("should coerce - number response type", async () => {
await updateFormulaColumn(encodeJS("return 1"), {
responseType: FieldType.NUMBER,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(1)
})
it("should coerce - boolean response type", async () => {
await updateFormulaColumn(encodeJS("return true"), {
responseType: FieldType.BOOLEAN,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(true)
})
it("should coerce - datetime response type", async () => {
await updateFormulaColumn(encodeJS("return new Date()"), {
responseType: FieldType.DATETIME,
})
const { rows } = await config.api.row.search(table._id!)
expect(isDate(rows[0].formula)).toBe(true)
})
it("should coerce - datetime with invalid value", async () => {
await updateFormulaColumn(encodeJS("return 'a'"), {
responseType: FieldType.DATETIME,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBeUndefined()
})
it("should coerce handlebars", async () => {
await updateFormulaColumn("{{ add 1 1 }}", {
responseType: FieldType.NUMBER,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(2)
})
it("should coerce handlebars to string (default)", async () => {
await updateFormulaColumn("{{ add 1 1 }}", {
responseType: FieldType.STRING,
})
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe("2")
})
isInternal &&
it("should coerce a static handlebars formula", async () => {
await updateFormulaColumn(encodeJS("return 1"), {
responseType: FieldType.NUMBER,
formulaType: FormulaType.STATIC,
})
// save the row to store the static value
await config.api.row.save(table._id!, mainRow)
const { rows } = await config.api.row.search(table._id!)
expect(rows[0].formula).toBe(1)
})
})
describe("Formula JS protection", () => {
it("should time out JS execution if a single cell takes too long", async () => {
await withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => {
const js = Buffer.from(
const js = encodeJS(
`
let i = 0;
while (true) {
@ -3255,7 +3340,7 @@ datasourceDescribe(
}
return i;
`
).toString("base64")
)
const table = await config.api.table.save(
saveTableRequest({
@ -3267,7 +3352,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
@ -3290,7 +3375,7 @@ datasourceDescribe(
JS_PER_REQUEST_TIMEOUT_MS: 80,
},
async () => {
const js = Buffer.from(
const js = encodeJS(
`
let i = 0;
while (true) {
@ -3298,7 +3383,7 @@ datasourceDescribe(
}
return i;
`
).toString("base64")
)
const table = await config.api.table.save(
saveTableRequest({
@ -3310,7 +3395,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formula: js,
formulaType: FormulaType.DYNAMIC,
},
},
@ -3352,7 +3437,7 @@ datasourceDescribe(
})
it("should not carry over context between formulas", async () => {
const js = Buffer.from(`return $("[text]");`).toString("base64")
const js = encodeJS(`return $("[text]");`)
const table = await config.api.table.save(
saveTableRequest({
schema: {
@ -3363,7 +3448,7 @@ datasourceDescribe(
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formula: js,
formulaType: FormulaType.DYNAMIC,
},
},

5
packages/server/src/startup/index.ts
View File

@ -28,6 +28,7 @@ import Koa from "koa"
import { Server } from "http"
import { AddressInfo } from "net"
import fs from "fs"
import bson from "bson"
let STARTUP_RAN = false
@ -193,6 +194,10 @@ export async function startup(
})
}
if (coreEnv.BSON_BUFFER_SIZE) {
bson.setInternalBufferSize(coreEnv.BSON_BUFFER_SIZE)
}
console.log("Initialising JS runner")
jsRunner.init()
}

22
packages/server/src/threads/query.ts
View File

@ -136,21 +136,23 @@ class QueryRunner {
pagination = output.pagination
}
// transform as required
if (transformer) {
// We avoid invoking the transformer if it's trivial because there is a cost
// to passing data in and out of the isolate, especially for MongoDB where
// we have to bson serialise/deserialise the data.
const hasTransformer =
transformer != null &&
transformer.length > 0 &&
transformer.trim() !== "return data" &&
transformer.trim() !== "return data;"
if (transformer && hasTransformer) {
transformer = iifeWrapper(transformer)
let vm = new IsolatedVM()
if (datasource.source === SourceName.MONGODB) {
vm = vm.withParsingBson(rows)
}
const ctx = {
data: rows,
params: enrichedParameters,
}
if (transformer != null) {
rows = vm.withContext(ctx, () => vm.execute(transformer!))
}
const ctx = { data: rows, params: enrichedParameters }
rows = vm.withContext(ctx, () => vm.execute(transformer!))
}
// if the request fails we retry once, invalidating the cached value

18
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -161,33 +161,33 @@ async function processDefaultValues(table: Table, row: Row) {
/**
* This will coerce a value to the correct types based on the type transform map
* @param row The value to coerce
* @param value The value to coerce
* @param type The type fo coerce to
* @returns The coerced value
*/
export function coerce(row: any, type: string) {
export function coerce(value: unknown, type: string) {
// no coercion specified for type, skip it
if (!TYPE_TRANSFORM_MAP[type]) {
return row
return value
}
// eslint-disable-next-line no-prototype-builtins
if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(value)) {
// @ts-ignore
return TYPE_TRANSFORM_MAP[type][row]
return TYPE_TRANSFORM_MAP[type][value]
} else if (TYPE_TRANSFORM_MAP[type].parse) {
// @ts-ignore
return TYPE_TRANSFORM_MAP[type].parse(row)
return TYPE_TRANSFORM_MAP[type].parse(value)
}
return row
return value
}
/**
* Given an input route this function will apply all the necessary pre-processing to it, such as coercion
* of column values or adding auto-column values.
* @param user the user which is performing the input.
* @param userId the ID of the user which is performing the input.
* @param row the row which is being created/updated.
* @param table the table which the row is being saved to.
* @param source the table/view which the row is being saved to.
* @param opts some input processing options (like disabling auto-column relationships).
* @returns the row which has been prepared to be written to the DB.
*/

33
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -10,11 +10,13 @@ import {
FieldType,
OperationFieldTypeEnum,
AIOperationEnum,
AIFieldMetadata,
} from "@budibase/types"
import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace"
import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
import { coerce } from "./index"
interface FormulaOpts {
dynamic?: boolean
@ -67,7 +69,18 @@ export async function processFormulas<T extends Row | Row[]>(
continue
}
const responseType = schema.responseType
const isStatic = schema.formulaType === FormulaType.STATIC
const formula = schema.formula
// coerce static values
if (isStatic) {
rows.forEach(row => {
if (row[column] && responseType) {
row[column] = coerce(row[column], responseType)
}
})
}
if (
schema.formula == null ||
@ -80,12 +93,18 @@ export async function processFormulas<T extends Row | Row[]>(
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
let formula = schema.formula
rows[i] = {
...row,
[column]: tracer.trace("processStringSync", {}, span => {
span?.addTags({ table_id: table._id, column, static: isStatic })
return processStringSync(formula, context)
const result = processStringSync(formula, context)
try {
return responseType ? coerce(result, responseType) : result
} catch (err: any) {
// if the coercion fails, we return empty row contents
span?.addTags({ coercionError: err.message })
return undefined
}
}),
}
}
@ -117,12 +136,13 @@ export async function processAIColumns<T extends Row | Row[]>(
continue
}
const operation = schema.operation
const aiSchema: AIFieldMetadata = schema
const rowUpdates = rows.map((row, i) => {
const contextRow = contextRows ? contextRows[i] : row
// Check if the type is bindable and pass through HBS if so
const operationField =
OperationFields[schema.operation as AIOperationEnum]
const operationField = OperationFields[operation as AIOperationEnum]
for (const key in schema) {
const fieldType = operationField[key as keyof typeof operationField]
if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
@ -131,7 +151,10 @@ export async function processAIColumns<T extends Row | Row[]>(
}
}
const prompt = llm.buildPromptFromAIOperation({ schema, row })
const prompt = llm.buildPromptFromAIOperation({
schema: aiSchema,
row,
})
return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column })

1
packages/server/tsconfig.json
View File

@ -1,6 +1,7 @@
{
"extends": "./tsconfig.build.json",
"compilerOptions": {
"lib": ["es2020", "dom"],
"composite": true,
"baseUrl": "."
},

4
packages/string-templates/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!dist/**/*
dist/tsconfig.build.tsbuildinfo
!package.json

8
packages/string-templates/package.json
View File

@ -4,7 +4,7 @@
"description": "Handlebars wrapper for Budibase templating.",
"main": "dist/bundle.cjs",
"module": "dist/bundle.mjs",
"types": "src/index.ts",
"types": "dist/index.d.ts",
"license": "MPL-2.0",
"exports": {
".": {
@ -12,12 +12,8 @@
"import": "./dist/bundle.mjs"
},
"./package.json": "./package.json",
"./iife": "./src/iife.js"
"./iife": "./dist/iife.mjs"
},
"files": [
"dist",
"src"
],
"scripts": {
"build": "tsc --emitDeclarationOnly && rollup -c",
"dev": "rollup -cw",

9
packages/string-templates/rollup.config.js
View File

@ -10,8 +10,8 @@ import inject from "@rollup/plugin-inject"
const production = !process.env.ROLLUP_WATCH
const config = (format, outputFile) => ({
input: "src/index.ts",
const config = (input, outputFile, format) => ({
input,
output: {
sourcemap: !production,
format,
@ -42,6 +42,7 @@ const config = (format, outputFile) => ({
})
export default [
config("cjs", "./dist/bundle.cjs"),
config("esm", "./dist/bundle.mjs"),
config("src/index.ts", "./dist/bundle.cjs", "cjs"),
config("src/index.ts", "./dist/bundle.mjs", "esm"),
config("src/iife.ts", "./dist/iife.mjs", "esm"),
]

6
packages/types/src/documents/app/row.ts
View File

@ -134,6 +134,12 @@ export const JsonTypes = [
FieldType.ARRAY,
]
export type FormulaResponseType =
| FieldType.STRING
| FieldType.NUMBER
| FieldType.BOOLEAN
| FieldType.DATETIME
export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT]
export function isNumeric(type: FieldType) {

3
packages/types/src/documents/app/table/schema.ts
View File

@ -1,6 +1,6 @@
// all added by grid/table when defining the
// column size, position and whether it can be viewed
import { FieldType } from "../row"
import { FieldType, FormulaResponseType } from "../row"
import {
AutoFieldSubType,
AutoReason,
@ -115,6 +115,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
type: FieldType.FORMULA
formula: string
formulaType?: FormulaType
responseType?: FormulaResponseType
}
export interface AIFieldMetadata extends BaseFieldSchema {

6
packages/worker/src/index.ts
View File

@ -24,6 +24,7 @@ import {
db.init()
import koaBody from "koa-body"
import http from "http"
import bson from "bson"
import api from "./api"
const koaSession = require("koa-session")
@ -99,6 +100,11 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"`
}
console.log(startupLog)
if (coreEnv.BSON_BUFFER_SIZE) {
bson.setInternalBufferSize(coreEnv.BSON_BUFFER_SIZE)
}
await initPro()
await redis.clients.init()
features.init()