diff --git a/.prettierignore b/.prettierignore index b1ee287391..b0f9f8cdbf 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,4 +9,5 @@ packages/backend-core/coverage packages/builder/.routify packages/sdk/sdk packages/pro/coverage -**/*.ivm.bundle.js \ No newline at end of file +**/*.ivm.bundle.js +!**/bson-polyfills.ivm.bundle.js \ No newline at end of file diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index 9348706399..d9d894c33e 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -41,12 +41,11 @@ module.exports = { if ( /^@budibase\/[^/]+\/.*$/.test(importPath) && importPath !== "@budibase/backend-core/tests" && - importPath !== "@budibase/string-templates/test/utils" && - importPath !== "@budibase/client/manifest.json" + importPath !== "@budibase/string-templates/test/utils" ) { context.report({ node, - message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`, }) } }, diff --git a/lerna.json b/lerna.json index 8d7460f053..15f405c847 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.3.3", + "version": "3.3.6", "npmClient": "yarn", "concurrency": 20, "command": { diff --git a/packages/builder/src/helpers/duplicate.ts b/packages/builder/src/helpers/duplicate.ts index b4740b3e52..eb705c6525 100644 --- a/packages/builder/src/helpers/duplicate.ts +++ b/packages/builder/src/helpers/duplicate.ts @@ -76,13 +76,15 @@ export const getSequentialName = ( { getName, numberFirstItem, + separator = "", }: { getName?: (item: T) => string numberFirstItem?: boolean + separator?: string } = {} ) => { if (!prefix?.length) { - return null + return "" } const trimmedPrefix = prefix.trim() const firstName = numberFirstItem ? `${prefix}1` : trimmedPrefix @@ -107,5 +109,5 @@ export const getSequentialName = ( max = num } }) - return max === 0 ? firstName : `${prefix}${max + 1}` + return max === 0 ? firstName : `${prefix}${separator}${max + 1}` } diff --git a/packages/builder/src/helpers/screen.ts b/packages/builder/src/helpers/screen.ts deleted file mode 100644 index 296a597adb..0000000000 --- a/packages/builder/src/helpers/screen.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Component, Screen, ScreenProps } from "@budibase/types" -import clientManifest from "@budibase/client/manifest.json" - -export function findComponentsBySettingsType( - screen: Screen, - type: string | string[] -) { - const typesArray = Array.isArray(type) ? type : [type] - - const result: { - component: Component - setting: { - type: string - key: string - } - }[] = [] - function recurseFieldComponentsInChildren(component: ScreenProps) { - if (!component) { - return - } - - const definition = getManifestDefinition(component) - const setting = - "settings" in definition && - definition.settings.find((s: any) => typesArray.includes(s.type)) - if (setting && "type" in setting) { - result.push({ - component, - setting: { type: setting.type!, key: setting.key! }, - }) - } - component._children?.forEach(child => { - recurseFieldComponentsInChildren(child) - }) - } - - recurseFieldComponentsInChildren(screen?.props) - return result -} - -function getManifestDefinition(component: Component) { - const componentType = component._component.split("/").slice(-1)[0] - const definition = - clientManifest[componentType as keyof typeof clientManifest] - return definition -} diff --git a/packages/builder/src/helpers/tests/duplicate.test.ts b/packages/builder/src/helpers/tests/duplicate.test.ts index 131e76a6c2..5e956d7b1c 100644 --- a/packages/builder/src/helpers/tests/duplicate.test.ts +++ b/packages/builder/src/helpers/tests/duplicate.test.ts @@ -49,7 +49,7 @@ describe("getSequentialName", () => { it("handles nullish prefix", async () => { const name = getSequentialName([], null) - expect(name).toBe(null) + expect(name).toBe("") }) it("handles just the prefix", async () => { diff --git a/packages/builder/src/stores/builder/automations.ts b/packages/builder/src/stores/builder/automations.ts index 98213581a6..ddb8706482 100644 --- a/packages/builder/src/stores/builder/automations.ts +++ b/packages/builder/src/stores/builder/automations.ts @@ -2,7 +2,7 @@ import { derived, get } from "svelte/store" import { API } from "@/api" import { cloneDeep } from "lodash/fp" import { generate } from "shortid" -import { createHistoryStore } from "@/stores/builder/history" +import { createHistoryStore, HistoryStore } from "@/stores/builder/history" import { licensing } from "@/stores/portal" import { tables, appStore } from "@/stores/builder" import { notifications } from "@budibase/bbui" @@ -1428,7 +1428,7 @@ const automationActions = (store: AutomationStore) => ({ }) class AutomationStore extends BudiStore { - history: any + history: HistoryStore actions: ReturnType constructor() { diff --git a/packages/builder/src/stores/builder/componentTreeNodes.ts b/packages/builder/src/stores/builder/componentTreeNodes.ts index cec4e3a4a0..b88563de6a 100644 --- a/packages/builder/src/stores/builder/componentTreeNodes.ts +++ b/packages/builder/src/stores/builder/componentTreeNodes.ts @@ -1,7 +1,7 @@ import { get } from "svelte/store" import { selectedScreen as selectedScreenStore } from "./screens" import { findComponentPath } from "@/helpers/components" -import { Screen, Component } from "@budibase/types" +import { Component, Screen } from "@budibase/types" import { BudiStore, PersistenceType } from "@/stores/BudiStore" interface OpenNodesState { diff --git a/packages/builder/src/stores/builder/components.ts b/packages/builder/src/stores/builder/components.ts index 90e1abfecf..6693cbf4dc 100644 --- a/packages/builder/src/stores/builder/components.ts +++ b/packages/builder/src/stores/builder/components.ts @@ -20,6 +20,7 @@ import { previewStore, tables, componentTreeNodesStore, + screenComponents, } from "@/stores/builder" import { buildFormSchema, getSchemaForDatasource } from "@/dataBinding" import { @@ -37,6 +38,7 @@ import { Table, } from "@budibase/types" import { utils } from "@budibase/shared-core" +import { getSequentialName } from "@/helpers/duplicate" interface Component extends ComponentType { _id: string @@ -452,7 +454,7 @@ export class ComponentStore extends BudiStore { * @returns */ createInstance( - componentName: string, + componentType: string, presetProps: any, parent: any ): Component | null { @@ -461,11 +463,20 @@ export class ComponentStore extends BudiStore { throw "A valid screen must be selected" } - const definition = this.getDefinition(componentName) + const definition = this.getDefinition(componentType) if (!definition) { return null } + const componentName = getSequentialName( + get(screenComponents), + `New ${definition.friendlyName || definition.name}`, + { + getName: c => c._instanceName, + separator: " ", + } + ) + // Generate basic component structure let instance: Component = { _id: Helpers.uuid(), @@ -475,7 +486,7 @@ export class ComponentStore extends BudiStore { hover: {}, active: {}, }, - _instanceName: `New ${definition.friendlyName || definition.name}`, + _instanceName: componentName, ...presetProps, } @@ -500,7 +511,7 @@ export class ComponentStore extends BudiStore { } // Add step name to form steps - if (componentName.endsWith("/formstep")) { + if (componentType.endsWith("/formstep")) { const parentForm = findClosestMatchingComponent( screen.props, get(selectedComponent)?._id, @@ -529,14 +540,14 @@ export class ComponentStore extends BudiStore { * @returns */ async create( - componentName: string, + componentType: string, presetProps: any, parent: Component, index: number ) { const state = get(this.store) const componentInstance = this.createInstance( - componentName, + componentType, presetProps, parent ) diff --git a/packages/builder/src/stores/builder/history.js b/packages/builder/src/stores/builder/history.ts similarity index 79% rename from packages/builder/src/stores/builder/history.js rename to packages/builder/src/stores/builder/history.ts index 50d68bd543..6568c5abca 100644 --- a/packages/builder/src/stores/builder/history.js +++ b/packages/builder/src/stores/builder/history.ts @@ -1,10 +1,25 @@ -import * as jsonpatch from "fast-json-patch/index.mjs" -import { writable, derived, get } from "svelte/store" +import { Document } from "@budibase/types" +import * as jsonpatch from "fast-json-patch" +import { writable, derived, get, Readable } from "svelte/store" -export const Operations = { - Add: "Add", - Delete: "Delete", - Change: "Change", +export const enum Operations { + Add = "Add", + Delete = "Delete", + Change = "Change", +} + +interface Operator { + id?: number + type: Operations + doc: T + forwardPatch?: jsonpatch.Operation[] + backwardsPatch?: jsonpatch.Operation[] +} + +interface HistoryState { + history: Operator[] + position: number + loading?: boolean } export const initialState = { @@ -13,14 +28,38 @@ export const initialState = { loading: false, } -export const createHistoryStore = ({ +export interface HistoryStore + extends Readable< + HistoryState & { + canUndo: boolean + canRedo: boolean + } + > { + wrapSaveDoc: ( + fn: (doc: T) => Promise + ) => (doc: T, operationId?: number) => Promise + wrapDeleteDoc: ( + fn: (doc: T) => Promise + ) => (doc: T, operationId?: number) => Promise + + reset: () => void + undo: () => Promise + redo: () => Promise +} + +export const createHistoryStore = ({ getDoc, selectDoc, - beforeAction = () => {}, - afterAction = () => {}, -}) => { + beforeAction, + afterAction, +}: { + getDoc: (id: string) => T | undefined + selectDoc: (id: string) => void + beforeAction?: (operation?: Operator) => void + afterAction?: (operation?: Operator) => void +}): HistoryStore => { // Use a derived store to check if we are able to undo or redo any operations - const store = writable(initialState) + const store = writable>(initialState) const derivedStore = derived(store, $store => { return { ...$store, @@ -31,8 +70,8 @@ export const createHistoryStore = ({ // Wrapped versions of essential functions which we call ourselves when using // undo and redo - let saveFn - let deleteFn + let saveFn: (doc: T, operationId?: number) => Promise + let deleteFn: (doc: T, operationId?: number) => Promise /** * Internal util to set the loading flag @@ -66,7 +105,7 @@ export const createHistoryStore = ({ * For internal use only. * @param operation the operation to save */ - const saveOperation = operation => { + const saveOperation = (operation: Operator) => { store.update(state => { // Update history let history = state.history @@ -93,15 +132,15 @@ export const createHistoryStore = ({ * @param fn the save function * @returns {function} a wrapped version of the save function */ - const wrapSaveDoc = fn => { - saveFn = async (doc, operationId) => { + const wrapSaveDoc = (fn: (doc: T) => Promise) => { + saveFn = async (doc: T, operationId?: number) => { // Only works on a single doc at a time if (!doc || Array.isArray(doc)) { return } startLoading() try { - const oldDoc = getDoc(doc._id) + const oldDoc = getDoc(doc._id!) const newDoc = jsonpatch.deepClone(await fn(doc)) // Store the change @@ -141,8 +180,8 @@ export const createHistoryStore = ({ * @param fn the delete function * @returns {function} a wrapped version of the delete function */ - const wrapDeleteDoc = fn => { - deleteFn = async (doc, operationId) => { + const wrapDeleteDoc = (fn: (doc: T) => Promise) => { + deleteFn = async (doc: T, operationId?: number) => { // Only works on a single doc at a time if (!doc || Array.isArray(doc)) { return @@ -201,7 +240,7 @@ export const createHistoryStore = ({ // Undo ADD if (operation.type === Operations.Add) { // Try to get the latest doc version to delete - const latestDoc = getDoc(operation.doc._id) + const latestDoc = getDoc(operation.doc._id!) const doc = latestDoc || operation.doc await deleteFn(doc, operation.id) } @@ -219,7 +258,7 @@ export const createHistoryStore = ({ // Undo CHANGE else { // Get the current doc and apply the backwards patch on top of it - let doc = jsonpatch.deepClone(getDoc(operation.doc._id)) + let doc = jsonpatch.deepClone(getDoc(operation.doc._id!)) if (doc) { jsonpatch.applyPatch( doc, @@ -283,7 +322,7 @@ export const createHistoryStore = ({ // Redo DELETE else if (operation.type === Operations.Delete) { // Try to get the latest doc version to delete - const latestDoc = getDoc(operation.doc._id) + const latestDoc = getDoc(operation.doc._id!) const doc = latestDoc || operation.doc await deleteFn(doc, operation.id) } @@ -291,7 +330,7 @@ export const createHistoryStore = ({ // Redo CHANGE else { // Get the current doc and apply the forwards patch on top of it - let doc = jsonpatch.deepClone(getDoc(operation.doc._id)) + let doc = jsonpatch.deepClone(getDoc(operation.doc._id!)) if (doc) { jsonpatch.applyPatch(doc, jsonpatch.deepClone(operation.forwardPatch)) await saveFn(doc, operation.id) diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 54c1aa39a2..7dd7f67828 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -16,7 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" import { contextMenuStore } from "./contextMenu.js" import { snippets } from "./snippets" -import { screenComponentErrors } from "./screenComponent" +import { screenComponents, screenComponentErrors } from "./screenComponent" // Backend import { tables } from "./tables" @@ -68,6 +68,7 @@ export { snippets, rowActions, appPublished, + screenComponents, screenComponentErrors, } diff --git a/packages/builder/src/stores/builder/screenComponent.ts b/packages/builder/src/stores/builder/screenComponent.ts index 226d233090..f1a9440c02 100644 --- a/packages/builder/src/stores/builder/screenComponent.ts +++ b/packages/builder/src/stores/builder/screenComponent.ts @@ -2,12 +2,18 @@ import { derived } from "svelte/store" import { tables } from "./tables" import { selectedScreen } from "./screens" import { viewsV2 } from "./viewsV2" -import { findComponentsBySettingsType } from "@/helpers/screen" -import { UIDatasourceType, Screen } from "@budibase/types" +import { + UIDatasourceType, + Screen, + Component, + ScreenProps, +} from "@budibase/types" import { queries } from "./queries" import { views } from "./views" import { bindings, featureFlag } from "@/helpers" import { getBindableProperties } from "@/dataBinding" +import { componentStore, ComponentDefinition } from "./components" +import { findAllComponents } from "@/helpers/components" function reduceBy( key: TKey, @@ -38,12 +44,16 @@ const validationKeyByType: Record = { } export const screenComponentErrors = derived( - [selectedScreen, tables, views, viewsV2, queries], - ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record< - string, - string[] - > => { - if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) { + [selectedScreen, tables, views, viewsV2, queries, componentStore], + ([ + $selectedScreen, + $tables, + $views, + $viewsV2, + $queries, + $componentStore, + ]): Record => { + if (!featureFlag.isEnabled("CHECK_COMPONENT_SETTINGS_ERRORS")) { return {} } function getInvalidDatasources( @@ -51,9 +61,11 @@ export const screenComponentErrors = derived( datasources: Record ) { const result: Record = {} + for (const { component, setting } of findComponentsBySettingsType( screen, - ["table", "dataSource"] + ["table", "dataSource"], + $componentStore.components )) { const componentSettings = component[setting.key] if (!componentSettings) { @@ -111,3 +123,53 @@ export const screenComponentErrors = derived( return getInvalidDatasources($selectedScreen, datasources) } ) + +function findComponentsBySettingsType( + screen: Screen, + type: string | string[], + definitions: Record +) { + const typesArray = Array.isArray(type) ? type : [type] + + const result: { + component: Component + setting: { + type: string + key: string + } + }[] = [] + + function recurseFieldComponentsInChildren(component: ScreenProps) { + if (!component) { + return + } + + const definition = definitions[component._component] + + const setting = definition?.settings?.find((s: any) => + typesArray.includes(s.type) + ) + if (setting && "type" in setting) { + result.push({ + component, + setting: { type: setting.type!, key: setting.key! }, + }) + } + component._children?.forEach(child => { + recurseFieldComponentsInChildren(child) + }) + } + + recurseFieldComponentsInChildren(screen?.props) + return result +} + +export const screenComponents = derived( + [selectedScreen], + ([$selectedScreen]) => { + if (!$selectedScreen) { + return [] + } + return findAllComponents($selectedScreen.props) as Component[] + } +) diff --git a/packages/builder/src/stores/builder/screens.ts b/packages/builder/src/stores/builder/screens.ts index a1b51875b3..64fe31752d 100644 --- a/packages/builder/src/stores/builder/screens.ts +++ b/packages/builder/src/stores/builder/screens.ts @@ -10,7 +10,7 @@ import { navigationStore, selectedComponent, } from "@/stores/builder" -import { createHistoryStore } from "@/stores/builder/history" +import { createHistoryStore, HistoryStore } from "@/stores/builder/history" import { API } from "@/api" import { BudiStore } from "../BudiStore" import { @@ -33,9 +33,9 @@ export const initialScreenState: ScreenState = { // Review the nulls export class ScreenStore extends BudiStore { - history: any - delete: any - save: any + history: HistoryStore + delete: (screens: Screen) => Promise + save: (screen: Screen) => Promise constructor() { super(initialScreenState) @@ -280,7 +280,10 @@ export class ScreenStore extends BudiStore { * supports deeply mutating the current doc rather than just appending data. */ sequentialScreenPatch = Utils.sequential( - async (patchFn: (screen: Screen) => any, screenId: string) => { + async ( + patchFn: (screen: Screen) => boolean, + screenId: string + ): Promise => { const state = get(this.store) const screen = state.screens.find(screen => screen._id === screenId) if (!screen) { @@ -361,10 +364,10 @@ export class ScreenStore extends BudiStore { * Any deleted screens will then have their routes/links purged * * Wrapped by {@link delete} - * @param {Screen | Screen[]} screens + * @param {Screen } screens */ - async deleteScreen(screens: Screen | Screen[]) { - const screensToDelete = Array.isArray(screens) ? screens : [screens] + async deleteScreen(screen: Screen) { + const screensToDelete = [screen] // Build array of promises to speed up bulk deletions let promises: Promise[] = [] let deleteUrls: string[] = [] diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.ts similarity index 81% rename from packages/frontend-core/src/utils/utils.js rename to packages/frontend-core/src/utils/utils.ts index 3dbaa23d3c..85cc5de54b 100644 --- a/packages/frontend-core/src/utils/utils.js +++ b/packages/frontend-core/src/utils/utils.ts @@ -1,8 +1,10 @@ import { makePropSafe as safe } from "@budibase/string-templates" import { Helpers } from "@budibase/bbui" import { cloneDeep } from "lodash" +import { SearchFilterGroup, UISearchFilter } from "@budibase/types" -export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) +export const sleep = (ms: number) => + new Promise(resolve => setTimeout(resolve, ms)) /** * Utility to wrap an async function and ensure all invocations happen @@ -10,12 +12,18 @@ export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) * @param fn the async function to run * @return {Function} a sequential version of the function */ -export const sequential = fn => { - let queue = [] - return (...params) => { - return new Promise((resolve, reject) => { +export const sequential = < + TReturn, + TFunction extends (...args: any[]) => Promise +>( + fn: TFunction +): TFunction => { + let queue: (() => Promise)[] = [] + const result = (...params: Parameters) => { + return new Promise((resolve, reject) => { queue.push(async () => { - let data, error + let data: TReturn | undefined + let error: unknown try { data = await fn(...params) } catch (err) { @@ -28,7 +36,7 @@ export const sequential = fn => { if (error) { reject(error) } else { - resolve(data) + resolve(data!) } }) if (queue.length === 1) { @@ -36,6 +44,7 @@ export const sequential = fn => { } }) } + return result as TFunction } /** @@ -45,9 +54,9 @@ export const sequential = fn => { * @param minDelay the minimum delay between invocations * @returns a debounced version of the callback */ -export const debounce = (callback, minDelay = 1000) => { - let timeout - return async (...params) => { +export const debounce = (callback: Function, minDelay = 1000) => { + let timeout: ReturnType + return async (...params: any[]) => { return new Promise(resolve => { if (timeout) { clearTimeout(timeout) @@ -70,11 +79,11 @@ export const debounce = (callback, minDelay = 1000) => { * @param minDelay * @returns {Function} a throttled version function */ -export const throttle = (callback, minDelay = 1000) => { - let lastParams +export const throttle = (callback: Function, minDelay = 1000) => { + let lastParams: any[] let stalled = false let pending = false - const invoke = (...params) => { + const invoke = (...params: any[]) => { lastParams = params if (stalled) { pending = true @@ -98,10 +107,10 @@ export const throttle = (callback, minDelay = 1000) => { * @param callback the function to run * @returns {Function} */ -export const domDebounce = callback => { +export const domDebounce = (callback: Function) => { let active = false - let lastParams - return (...params) => { + let lastParams: any[] + return (...params: any[]) => { lastParams = params if (!active) { active = true @@ -119,7 +128,17 @@ export const domDebounce = callback => { * * @param {any} props * */ -export const buildFormBlockButtonConfig = props => { +export const buildFormBlockButtonConfig = (props?: { + _id?: string + actionType?: string + dataSource?: { resourceId: string } + notificationOverride?: boolean + actionUrl?: string + showDeleteButton?: boolean + deleteButtonLabel?: string + showSaveButton?: boolean + saveButtonLabel?: string +}) => { const { _id, actionType, @@ -227,7 +246,11 @@ export const buildFormBlockButtonConfig = props => { const defaultButtons = [] - if (["Update", "Create"].includes(actionType) && showSaveButton !== false) { + if ( + actionType && + ["Update", "Create"].includes(actionType) && + showSaveButton !== false + ) { defaultButtons.push({ text: saveText || "Save", _id: Helpers.uuid(), @@ -251,7 +274,13 @@ export const buildFormBlockButtonConfig = props => { return defaultButtons } -export const buildMultiStepFormBlockDefaultProps = props => { +export const buildMultiStepFormBlockDefaultProps = (props?: { + _id: string + stepCount: number + currentStep: number + actionType: string + dataSource: { resourceId: string } +}) => { const { _id, stepCount, currentStep, actionType, dataSource } = props || {} // Sanity check @@ -361,7 +390,7 @@ export const buildMultiStepFormBlockDefaultProps = props => { * @param {Object} filter UI filter * @returns {Object} parsed filter */ -export function parseFilter(filter) { +export function parseFilter(filter: UISearchFilter) { if (!filter?.groups) { return filter } @@ -369,13 +398,13 @@ export function parseFilter(filter) { const update = cloneDeep(filter) update.groups = update.groups - .map(group => { - group.filters = group.filters.filter(filter => { + ?.map(group => { + group.filters = group.filters?.filter((filter: any) => { return filter.field && filter.operator }) - return group.filters.length ? group : null + return group.filters?.length ? group : null }) - .filter(group => group) + .filter((group): group is SearchFilterGroup => !!group) return update } diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index a37957fe7e..37af4e74e1 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -634,6 +634,130 @@ if (descriptions.length) { } }) }) + + it("should be able to select a ObjectId in a transformer", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "find", + }, + }, + transformer: "return data.map(x => ({ id: x._id }))", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { id: expectValidId }, + { id: expectValidId }, + { id: expectValidId }, + { id: expectValidId }, + { id: expectValidId }, + ]) + }) + + it("can handle all bson field types with transformers", async () => { + collection = generator.guid() + await withCollection(async collection => { + await collection.insertOne({ + _id: new BSON.ObjectId("65b0123456789abcdef01234"), + stringField: "This is a string", + numberField: 42, + doubleField: new BSON.Double(42.42), + integerField: new BSON.Int32(123), + longField: new BSON.Long("9223372036854775807"), + booleanField: true, + nullField: null, + arrayField: [1, 2, 3, "four", { nested: true }], + objectField: { + nestedString: "nested", + nestedNumber: 99, + }, + dateField: new Date(Date.UTC(2025, 0, 30, 12, 30, 20)), + timestampField: new BSON.Timestamp({ t: 1706616000, i: 1 }), + binaryField: new BSON.Binary( + new TextEncoder().encode("bufferValue") + ), + objectIdField: new BSON.ObjectId("65b0123456789abcdef01235"), + regexField: new BSON.BSONRegExp("^Hello.*", "i"), + minKeyField: new BSON.MinKey(), + maxKeyField: new BSON.MaxKey(), + decimalField: new BSON.Decimal128("12345.6789"), + codeField: new BSON.Code( + "function() { return 'Hello, World!'; }" + ), + codeWithScopeField: new BSON.Code( + "function(x) { return x * 2; }", + { x: 10 } + ), + }) + }) + + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "find", + collection, + }, + }, + transformer: `return data.map(x => ({ + ...x, + binaryField: x.binaryField?.toString('utf8'), + decimalField: x.decimalField.toString(), + longField: x.longField.toString(), + regexField: x.regexField.toString(), + // TODO: currenlty not supported, it looks like there is bug in the library. Getting: Timestamp constructed from { t, i } must provide t as a number + timestampField: null + }))`, + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + _id: "65b0123456789abcdef01234", + arrayField: [ + 1, + 2, + 3, + "four", + { + nested: true, + }, + ], + binaryField: "bufferValue", + booleanField: true, + codeField: { + code: "function() { return 'Hello, World!'; }", + }, + codeWithScopeField: { + code: "function(x) { return x * 2; }", + scope: { + x: 10, + }, + }, + dateField: "2025-01-30T12:30:20.000Z", + decimalField: "12345.6789", + doubleField: 42.42, + integerField: 123, + longField: "9223372036854775807", + maxKeyField: {}, + minKeyField: {}, + nullField: null, + numberField: 42, + objectField: { + nestedNumber: 99, + nestedString: "nested", + }, + objectIdField: "65b0123456789abcdef01235", + regexField: "/^Hello.*/i", + stringField: "This is a string", + timestampField: null, + }, + ]) + }) }) it("should throw an error if the incorrect actionType is specified", async () => { diff --git a/packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js b/packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js new file mode 100644 index 0000000000..94d08b84ed --- /dev/null +++ b/packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js @@ -0,0 +1,122 @@ +if (typeof btoa !== "function") { + var chars = { + ascii: function () { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" + }, + indices: function () { + if (!this.cache) { + this.cache = {} + var ascii = chars.ascii() + + for (var c = 0; c < ascii.length; c++) { + var chr = ascii[c] + this.cache[chr] = c + } + } + return this.cache + }, + } + + function atob(b64) { + var indices = chars.indices(), + pos = b64.indexOf("="), + padded = pos > -1, + len = padded ? pos : b64.length, + i = -1, + data = "" + + while (i < len) { + var code = + (indices[b64[++i]] << 18) | + (indices[b64[++i]] << 12) | + (indices[b64[++i]] << 6) | + indices[b64[++i]] + if (code !== 0) { + data += String.fromCharCode( + (code >>> 16) & 255, + (code >>> 8) & 255, + code & 255 + ) + } + } + + if (padded) { + data = data.slice(0, pos - b64.length) + } + + return data + } + + function btoa(data) { + var ascii = chars.ascii(), + len = data.length - 1, + i = -1, + b64 = "" + + while (i < len) { + var code = + (data.charCodeAt(++i) << 16) | + (data.charCodeAt(++i) << 8) | + data.charCodeAt(++i) + b64 += + ascii[(code >>> 18) & 63] + + ascii[(code >>> 12) & 63] + + ascii[(code >>> 6) & 63] + + ascii[code & 63] + } + + var pads = data.length % 3 + if (pads > 0) { + b64 = b64.slice(0, pads - 3) + + while (b64.length % 4 !== 0) { + b64 += "=" + } + } + + return b64 + } +} + +if (typeof TextDecoder === "undefined") { + globalThis.TextDecoder = class { + constructor(encoding = "utf8") { + if (encoding !== "utf8") { + throw new Error( + `Only UTF-8 is supported in this polyfill. Recieved: ${encoding}` + ) + } + } + decode(buffer) { + return String.fromCharCode(...buffer) + } + } +} + +if (typeof TextEncoder === "undefined") { + globalThis.TextEncoder = class { + encode(str) { + const utf8 = [] + for (const i = 0; i < str.length; i++) { + const codePoint = str.charCodeAt(i) + + if (codePoint < 0x80) { + utf8.push(codePoint) + } else if (codePoint < 0x800) { + utf8.push(0xc0 | (codePoint >> 6)) + utf8.push(0x80 | (codePoint & 0x3f)) + } else if (codePoint < 0x10000) { + utf8.push(0xe0 | (codePoint >> 12)) + utf8.push(0x80 | ((codePoint >> 6) & 0x3f)) + utf8.push(0x80 | (codePoint & 0x3f)) + } else { + utf8.push(0xf0 | (codePoint >> 18)) + utf8.push(0x80 | ((codePoint >> 12) & 0x3f)) + utf8.push(0x80 | ((codePoint >> 6) & 0x3f)) + utf8.push(0x80 | (codePoint & 0x3f)) + } + } + return new Uint8Array(utf8) + } + } +} diff --git a/packages/server/src/jsRunner/bundles/index.ts b/packages/server/src/jsRunner/bundles/index.ts index b62adac1cc..3a00ee96cc 100644 --- a/packages/server/src/jsRunner/bundles/index.ts +++ b/packages/server/src/jsRunner/bundles/index.ts @@ -5,6 +5,7 @@ export const enum BundleType { BSON = "bson", SNIPPETS = "snippets", BUFFER = "buffer", + BSON_POLYFILLS = "bson_polyfills", } const bundleSourceFile: Record = { @@ -12,6 +13,7 @@ const bundleSourceFile: Record = { [BundleType.BSON]: "./bson.ivm.bundle.js", [BundleType.SNIPPETS]: "./snippets.ivm.bundle.js", [BundleType.BUFFER]: "./buffer.ivm.bundle.js", + [BundleType.BSON_POLYFILLS]: "./bson-polyfills.ivm.bundle.js", } const bundleSourceCode: Partial> = {} diff --git a/packages/server/src/jsRunner/bundles/snippets.ts b/packages/server/src/jsRunner/bundles/snippets.ts index 8244b2eef8..343cccd36f 100644 --- a/packages/server/src/jsRunner/bundles/snippets.ts +++ b/packages/server/src/jsRunner/bundles/snippets.ts @@ -1,6 +1,5 @@ // @ts-ignore -// eslint-disable-next-line local-rules/no-budibase-imports -import { iifeWrapper } from "@budibase/string-templates/iife" +import { iifeWrapper } from "@budibase/string-templates" export default new Proxy( {}, diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 3863be742d..37ee048dc2 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -161,42 +161,10 @@ export class IsolatedVM implements VM { const bsonSource = loadBundle(BundleType.BSON) - this.addToContext({ - textDecoderCb: new ivm.Callback( - (args: { - constructorArgs: any - functionArgs: Parameters["decode"]> - }) => { - const result = new TextDecoder(...args.constructorArgs).decode( - ...args.functionArgs - ) - return result - } - ), - }) - - // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library - const textDecoderPolyfill = class TextDecoderMock { - constructorArgs - - constructor(...constructorArgs: any) { - this.constructorArgs = constructorArgs - } - - decode(...input: any) { - // @ts-expect-error - this is going to run in the isolate, where this function will be available - // eslint-disable-next-line no-undef - return textDecoderCb({ - constructorArgs: this.constructorArgs, - functionArgs: input, - }) - } - } - .toString() - .replace(/TextDecoderMock/, "TextDecoder") + const bsonPolyfills = loadBundle(BundleType.BSON_POLYFILLS) const script = this.isolate.compileScriptSync( - `${textDecoderPolyfill};${bsonSource}` + `${bsonPolyfills};${bsonSource}` ) script.runSync(this.vm, { timeout: this.invocationTimeout, release: false }) new Promise(() => { diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index b1b4b9ef55..acd0ea4fa8 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -11,8 +11,7 @@ "require": "./dist/bundle.cjs", "import": "./dist/bundle.mjs" }, - "./package.json": "./package.json", - "./iife": "./dist/iife.mjs" + "./package.json": "./package.json" }, "scripts": { "build": "tsc --emitDeclarationOnly && rollup -c", diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index d9f092c80a..97893a1b5e 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,6 +1,6 @@ export enum FeatureFlag { USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR", - CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS = "CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS", + CHECK_COMPONENT_SETTINGS_ERRORS = "CHECK_COMPONENT_SETTINGS_ERRORS", // Account-portal DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL", @@ -8,7 +8,7 @@ export enum FeatureFlag { export const FeatureFlagDefaults = { [FeatureFlag.USE_ZOD_VALIDATOR]: false, - [FeatureFlag.CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS]: false, + [FeatureFlag.CHECK_COMPONENT_SETTINGS_ERRORS]: false, // Account-portal [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,