Merge branch 'master' of https://github.com/budibase/budibase into csv-delims

.prettierignore

@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/builder/.routify
 packages/sdk/sdk
 packages/pro/coverage
-**/*.ivm.bundle.js
+**/*.ivm.bundle.js
+!**/bson-polyfills.ivm.bundle.js

eslint-local-rules/index.js

@@ -41,12 +41,11 @@ module.exports = {
           if (
             /^@budibase\/[^/]+\/.*$/.test(importPath) &&
             importPath !== "@budibase/backend-core/tests" &&
-            importPath !== "@budibase/string-templates/test/utils" &&
+            importPath !== "@budibase/string-templates/test/utils"
-            importPath !== "@budibase/client/manifest.json"
           ) {
             context.report({
               node,
-              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests, @budibase/string-templates/test/utils and @budibase/client/manifest.json.`,
+              message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`,
             })
           }
         },

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.3.3",
+  "version": "3.3.6",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/builder/src/helpers/duplicate.ts

@@ -76,13 +76,15 @@ export const getSequentialName = <T extends any>(
   {
     getName,
     numberFirstItem,
+    separator = "",
   }: {
     getName?: (item: T) => string
     numberFirstItem?: boolean
+    separator?: string
   } = {}
 ) => {
   if (!prefix?.length) {
-    return null
+    return ""
   }
   const trimmedPrefix = prefix.trim()
   const firstName = numberFirstItem ? `${prefix}1` : trimmedPrefix
@@ -107,5 +109,5 @@ export const getSequentialName = <T extends any>(
       max = num
     }
   })
-  return max === 0 ? firstName : `${prefix}${max + 1}`
+  return max === 0 ? firstName : `${prefix}${separator}${max + 1}`
 }

packages/builder/src/helpers/screen.ts

@@ -1,46 +0,0 @@
-import { Component, Screen, ScreenProps } from "@budibase/types"
-import clientManifest from "@budibase/client/manifest.json"
-
-export function findComponentsBySettingsType(
-  screen: Screen,
-  type: string | string[]
-) {
-  const typesArray = Array.isArray(type) ? type : [type]
-
-  const result: {
-    component: Component
-    setting: {
-      type: string
-      key: string
-    }
-  }[] = []
-  function recurseFieldComponentsInChildren(component: ScreenProps) {
-    if (!component) {
-      return
-    }
-
-    const definition = getManifestDefinition(component)
-    const setting =
-      "settings" in definition &&
-      definition.settings.find((s: any) => typesArray.includes(s.type))
-    if (setting && "type" in setting) {
-      result.push({
-        component,
-        setting: { type: setting.type!, key: setting.key! },
-      })
-    }
-    component._children?.forEach(child => {
-      recurseFieldComponentsInChildren(child)
-    })
-  }
-
-  recurseFieldComponentsInChildren(screen?.props)
-  return result
-}
-
-function getManifestDefinition(component: Component) {
-  const componentType = component._component.split("/").slice(-1)[0]
-  const definition =
-    clientManifest[componentType as keyof typeof clientManifest]
-  return definition
-}

packages/builder/src/helpers/tests/duplicate.test.ts

@@ -49,7 +49,7 @@ describe("getSequentialName", () => {
 
   it("handles nullish prefix", async () => {
     const name = getSequentialName([], null)
-    expect(name).toBe(null)
+    expect(name).toBe("")
   })
 
   it("handles just the prefix", async () => {

packages/builder/src/stores/builder/automations.ts

@@ -2,7 +2,7 @@ import { derived, get } from "svelte/store"
 import { API } from "@/api"
 import { cloneDeep } from "lodash/fp"
 import { generate } from "shortid"
-import { createHistoryStore } from "@/stores/builder/history"
+import { createHistoryStore, HistoryStore } from "@/stores/builder/history"
 import { licensing } from "@/stores/portal"
 import { tables, appStore } from "@/stores/builder"
 import { notifications } from "@budibase/bbui"
@@ -1428,7 +1428,7 @@ const automationActions = (store: AutomationStore) => ({
 })
 
 class AutomationStore extends BudiStore<AutomationState> {
-  history: any
+  history: HistoryStore<Automation>
   actions: ReturnType<typeof automationActions>
 
   constructor() {

packages/builder/src/stores/builder/componentTreeNodes.ts

@@ -1,7 +1,7 @@
 import { get } from "svelte/store"
 import { selectedScreen as selectedScreenStore } from "./screens"
 import { findComponentPath } from "@/helpers/components"
-import { Screen, Component } from "@budibase/types"
+import { Component, Screen } from "@budibase/types"
 import { BudiStore, PersistenceType } from "@/stores/BudiStore"
 
 interface OpenNodesState {

packages/builder/src/stores/builder/components.ts

@@ -20,6 +20,7 @@ import {
   previewStore,
   tables,
   componentTreeNodesStore,
+  screenComponents,
 } from "@/stores/builder"
 import { buildFormSchema, getSchemaForDatasource } from "@/dataBinding"
 import {
@@ -37,6 +38,7 @@ import {
   Table,
 } from "@budibase/types"
 import { utils } from "@budibase/shared-core"
+import { getSequentialName } from "@/helpers/duplicate"
 
 interface Component extends ComponentType {
   _id: string
@@ -452,7 +454,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
    * @returns
    */
   createInstance(
-    componentName: string,
+    componentType: string,
     presetProps: any,
     parent: any
   ): Component | null {
@@ -461,11 +463,20 @@ export class ComponentStore extends BudiStore<ComponentState> {
       throw "A valid screen must be selected"
     }
 
-    const definition = this.getDefinition(componentName)
+    const definition = this.getDefinition(componentType)
     if (!definition) {
       return null
     }
 
+    const componentName = getSequentialName(
+      get(screenComponents),
+      `New ${definition.friendlyName || definition.name}`,
+      {
+        getName: c => c._instanceName,
+        separator: " ",
+      }
+    )
+
     // Generate basic component structure
     let instance: Component = {
       _id: Helpers.uuid(),
@@ -475,7 +486,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         hover: {},
         active: {},
       },
-      _instanceName: `New ${definition.friendlyName || definition.name}`,
+      _instanceName: componentName,
       ...presetProps,
     }
 
@@ -500,7 +511,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
 
     // Add step name to form steps
-    if (componentName.endsWith("/formstep")) {
+    if (componentType.endsWith("/formstep")) {
       const parentForm = findClosestMatchingComponent(
         screen.props,
         get(selectedComponent)?._id,
@@ -529,14 +540,14 @@ export class ComponentStore extends BudiStore<ComponentState> {
    * @returns
    */
   async create(
-    componentName: string,
+    componentType: string,
     presetProps: any,
     parent: Component,
     index: number
   ) {
     const state = get(this.store)
     const componentInstance = this.createInstance(
-      componentName,
+      componentType,
       presetProps,
       parent
     )

packages/builder/src/stores/builder/history.ts

@@ -1,10 +1,25 @@
-import * as jsonpatch from "fast-json-patch/index.mjs"
+import { Document } from "@budibase/types"
-import { writable, derived, get } from "svelte/store"
+import * as jsonpatch from "fast-json-patch"
+import { writable, derived, get, Readable } from "svelte/store"
 
-export const Operations = {
+export const enum Operations {
-  Add: "Add",
+  Add = "Add",
-  Delete: "Delete",
+  Delete = "Delete",
-  Change: "Change",
+  Change = "Change",
+}
+
+interface Operator<T extends Document> {
+  id?: number
+  type: Operations
+  doc: T
+  forwardPatch?: jsonpatch.Operation[]
+  backwardsPatch?: jsonpatch.Operation[]
+}
+
+interface HistoryState<T extends Document> {
+  history: Operator<T>[]
+  position: number
+  loading?: boolean
 }
 
 export const initialState = {
@@ -13,14 +28,38 @@ export const initialState = {
   loading: false,
 }
 
-export const createHistoryStore = ({
+export interface HistoryStore<T extends Document>
+  extends Readable<
+    HistoryState<T> & {
+      canUndo: boolean
+      canRedo: boolean
+    }
+  > {
+  wrapSaveDoc: (
+    fn: (doc: T) => Promise<T>
+  ) => (doc: T, operationId?: number) => Promise<T>
+  wrapDeleteDoc: (
+    fn: (doc: T) => Promise<void>
+  ) => (doc: T, operationId?: number) => Promise<void>
+
+  reset: () => void
+  undo: () => Promise<void>
+  redo: () => Promise<void>
+}
+
+export const createHistoryStore = <T extends Document>({
   getDoc,
   selectDoc,
-  beforeAction = () => {},
+  beforeAction,
-  afterAction = () => {},
+  afterAction,
-}) => {
+}: {
+  getDoc: (id: string) => T | undefined
+  selectDoc: (id: string) => void
+  beforeAction?: (operation?: Operator<T>) => void
+  afterAction?: (operation?: Operator<T>) => void
+}): HistoryStore<T> => {
   // Use a derived store to check if we are able to undo or redo any operations
-  const store = writable(initialState)
+  const store = writable<HistoryState<T>>(initialState)
   const derivedStore = derived(store, $store => {
     return {
       ...$store,
@@ -31,8 +70,8 @@ export const createHistoryStore = ({
 
   // Wrapped versions of essential functions which we call ourselves when using
   // undo and redo
-  let saveFn
+  let saveFn: (doc: T, operationId?: number) => Promise<T>
-  let deleteFn
+  let deleteFn: (doc: T, operationId?: number) => Promise<void>
 
   /**
    * Internal util to set the loading flag
@@ -66,7 +105,7 @@ export const createHistoryStore = ({
    * For internal use only.
    * @param operation the operation to save
    */
-  const saveOperation = operation => {
+  const saveOperation = (operation: Operator<T>) => {
     store.update(state => {
       // Update history
       let history = state.history
@@ -93,15 +132,15 @@ export const createHistoryStore = ({
    * @param fn the save function
    * @returns {function} a wrapped version of the save function
    */
-  const wrapSaveDoc = fn => {
+  const wrapSaveDoc = (fn: (doc: T) => Promise<T>) => {
-    saveFn = async (doc, operationId) => {
+    saveFn = async (doc: T, operationId?: number) => {
       // Only works on a single doc at a time
       if (!doc || Array.isArray(doc)) {
         return
       }
       startLoading()
       try {
-        const oldDoc = getDoc(doc._id)
+        const oldDoc = getDoc(doc._id!)
         const newDoc = jsonpatch.deepClone(await fn(doc))
 
         // Store the change
@@ -141,8 +180,8 @@ export const createHistoryStore = ({
    * @param fn the delete function
    * @returns {function} a wrapped version of the delete function
    */
-  const wrapDeleteDoc = fn => {
+  const wrapDeleteDoc = (fn: (doc: T) => Promise<void>) => {
-    deleteFn = async (doc, operationId) => {
+    deleteFn = async (doc: T, operationId?: number) => {
       // Only works on a single doc at a time
       if (!doc || Array.isArray(doc)) {
         return
@@ -201,7 +240,7 @@ export const createHistoryStore = ({
       // Undo ADD
       if (operation.type === Operations.Add) {
         // Try to get the latest doc version to delete
-        const latestDoc = getDoc(operation.doc._id)
+        const latestDoc = getDoc(operation.doc._id!)
         const doc = latestDoc || operation.doc
         await deleteFn(doc, operation.id)
       }
@@ -219,7 +258,7 @@ export const createHistoryStore = ({
       // Undo CHANGE
       else {
         // Get the current doc and apply the backwards patch on top of it
-        let doc = jsonpatch.deepClone(getDoc(operation.doc._id))
+        let doc = jsonpatch.deepClone(getDoc(operation.doc._id!))
         if (doc) {
           jsonpatch.applyPatch(
             doc,
@@ -283,7 +322,7 @@ export const createHistoryStore = ({
       // Redo DELETE
       else if (operation.type === Operations.Delete) {
         // Try to get the latest doc version to delete
-        const latestDoc = getDoc(operation.doc._id)
+        const latestDoc = getDoc(operation.doc._id!)
         const doc = latestDoc || operation.doc
         await deleteFn(doc, operation.id)
       }
@@ -291,7 +330,7 @@ export const createHistoryStore = ({
       // Redo CHANGE
       else {
         // Get the current doc and apply the forwards patch on top of it
-        let doc = jsonpatch.deepClone(getDoc(operation.doc._id))
+        let doc = jsonpatch.deepClone(getDoc(operation.doc._id!))
         if (doc) {
           jsonpatch.applyPatch(doc, jsonpatch.deepClone(operation.forwardPatch))
           await saveFn(doc, operation.id)

packages/builder/src/stores/builder/index.js

@@ -16,7 +16,7 @@ import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
 import { deploymentStore } from "./deployments.js"
 import { contextMenuStore } from "./contextMenu.js"
 import { snippets } from "./snippets"
-import { screenComponentErrors } from "./screenComponent"
+import { screenComponents, screenComponentErrors } from "./screenComponent"
 
 // Backend
 import { tables } from "./tables"
@@ -68,6 +68,7 @@ export {
   snippets,
   rowActions,
   appPublished,
+  screenComponents,
   screenComponentErrors,
 }
 

packages/builder/src/stores/builder/screenComponent.ts

@@ -2,12 +2,18 @@ import { derived } from "svelte/store"
 import { tables } from "./tables"
 import { selectedScreen } from "./screens"
 import { viewsV2 } from "./viewsV2"
-import { findComponentsBySettingsType } from "@/helpers/screen"
+import {
-import { UIDatasourceType, Screen } from "@budibase/types"
+  UIDatasourceType,
+  Screen,
+  Component,
+  ScreenProps,
+} from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
 import { bindings, featureFlag } from "@/helpers"
 import { getBindableProperties } from "@/dataBinding"
+import { componentStore, ComponentDefinition } from "./components"
+import { findAllComponents } from "@/helpers/components"
 
 function reduceBy<TItem extends {}, TKey extends keyof TItem>(
   key: TKey,
@@ -38,12 +44,16 @@ const validationKeyByType: Record<UIDatasourceType, string | null> = {
 }
 
 export const screenComponentErrors = derived(
-  [selectedScreen, tables, views, viewsV2, queries],
+  [selectedScreen, tables, views, viewsV2, queries, componentStore],
-  ([$selectedScreen, $tables, $views, $viewsV2, $queries]): Record<
+  ([
-    string,
+    $selectedScreen,
-    string[]
+    $tables,
-  > => {
+    $views,
-    if (!featureFlag.isEnabled("CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS")) {
+    $viewsV2,
+    $queries,
+    $componentStore,
+  ]): Record<string, string[]> => {
+    if (!featureFlag.isEnabled("CHECK_COMPONENT_SETTINGS_ERRORS")) {
       return {}
     }
     function getInvalidDatasources(
@@ -51,9 +61,11 @@ export const screenComponentErrors = derived(
       datasources: Record<string, any>
     ) {
       const result: Record<string, string[]> = {}
+
       for (const { component, setting } of findComponentsBySettingsType(
         screen,
-        ["table", "dataSource"]
+        ["table", "dataSource"],
+        $componentStore.components
       )) {
         const componentSettings = component[setting.key]
         if (!componentSettings) {
@@ -111,3 +123,53 @@ export const screenComponentErrors = derived(
     return getInvalidDatasources($selectedScreen, datasources)
   }
 )
+
+function findComponentsBySettingsType(
+  screen: Screen,
+  type: string | string[],
+  definitions: Record<string, ComponentDefinition>
+) {
+  const typesArray = Array.isArray(type) ? type : [type]
+
+  const result: {
+    component: Component
+    setting: {
+      type: string
+      key: string
+    }
+  }[] = []
+
+  function recurseFieldComponentsInChildren(component: ScreenProps) {
+    if (!component) {
+      return
+    }
+
+    const definition = definitions[component._component]
+
+    const setting = definition?.settings?.find((s: any) =>
+      typesArray.includes(s.type)
+    )
+    if (setting && "type" in setting) {
+      result.push({
+        component,
+        setting: { type: setting.type!, key: setting.key! },
+      })
+    }
+    component._children?.forEach(child => {
+      recurseFieldComponentsInChildren(child)
+    })
+  }
+
+  recurseFieldComponentsInChildren(screen?.props)
+  return result
+}
+
+export const screenComponents = derived(
+  [selectedScreen],
+  ([$selectedScreen]) => {
+    if (!$selectedScreen) {
+      return []
+    }
+    return findAllComponents($selectedScreen.props) as Component[]
+  }
+)

packages/builder/src/stores/builder/screens.ts

@@ -10,7 +10,7 @@ import {
   navigationStore,
   selectedComponent,
 } from "@/stores/builder"
-import { createHistoryStore } from "@/stores/builder/history"
+import { createHistoryStore, HistoryStore } from "@/stores/builder/history"
 import { API } from "@/api"
 import { BudiStore } from "../BudiStore"
 import {
@@ -33,9 +33,9 @@ export const initialScreenState: ScreenState = {
 
 // Review the nulls
 export class ScreenStore extends BudiStore<ScreenState> {
-  history: any
+  history: HistoryStore<Screen>
-  delete: any
+  delete: (screens: Screen) => Promise<void>
-  save: any
+  save: (screen: Screen) => Promise<Screen>
 
   constructor() {
     super(initialScreenState)
@@ -280,7 +280,10 @@ export class ScreenStore extends BudiStore<ScreenState> {
    * supports deeply mutating the current doc rather than just appending data.
    */
   sequentialScreenPatch = Utils.sequential(
-    async (patchFn: (screen: Screen) => any, screenId: string) => {
+    async (
+      patchFn: (screen: Screen) => boolean,
+      screenId: string
+    ): Promise<Screen | void> => {
       const state = get(this.store)
       const screen = state.screens.find(screen => screen._id === screenId)
       if (!screen) {
@@ -361,10 +364,10 @@ export class ScreenStore extends BudiStore<ScreenState> {
    * Any deleted screens will then have their routes/links purged
    *
    * Wrapped by {@link delete}
-   * @param {Screen | Screen[]} screens
+   * @param {Screen } screens
    */
-  async deleteScreen(screens: Screen | Screen[]) {
+  async deleteScreen(screen: Screen) {
-    const screensToDelete = Array.isArray(screens) ? screens : [screens]
+    const screensToDelete = [screen]
     // Build array of promises to speed up bulk deletions
     let promises: Promise<DeleteScreenResponse>[] = []
     let deleteUrls: string[] = []

packages/frontend-core/src/utils/utils.ts

@@ -1,8 +1,10 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { Helpers } from "@budibase/bbui"
 import { cloneDeep } from "lodash"
+import { SearchFilterGroup, UISearchFilter } from "@budibase/types"
 
-export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+export const sleep = (ms: number) =>
+  new Promise(resolve => setTimeout(resolve, ms))
 
 /**
  * Utility to wrap an async function and ensure all invocations happen
@@ -10,12 +12,18 @@ export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
  * @param fn the async function to run
  * @return {Function} a sequential version of the function
  */
-export const sequential = fn => {
+export const sequential = <
-  let queue = []
+  TReturn,
-  return (...params) => {
+  TFunction extends (...args: any[]) => Promise<TReturn>
-    return new Promise((resolve, reject) => {
+>(
+  fn: TFunction
+): TFunction => {
+  let queue: (() => Promise<void>)[] = []
+  const result = (...params: Parameters<TFunction>) => {
+    return new Promise<TReturn>((resolve, reject) => {
       queue.push(async () => {
-        let data, error
+        let data: TReturn | undefined
+        let error: unknown
         try {
           data = await fn(...params)
         } catch (err) {
@@ -28,7 +36,7 @@ export const sequential = fn => {
         if (error) {
           reject(error)
         } else {
-          resolve(data)
+          resolve(data!)
         }
       })
       if (queue.length === 1) {
@@ -36,6 +44,7 @@ export const sequential = fn => {
       }
     })
   }
+  return result as TFunction
 }
 
 /**
@@ -45,9 +54,9 @@ export const sequential = fn => {
  * @param minDelay the minimum delay between invocations
  * @returns a debounced version of the callback
  */
-export const debounce = (callback, minDelay = 1000) => {
+export const debounce = (callback: Function, minDelay = 1000) => {
-  let timeout
+  let timeout: ReturnType<typeof setTimeout>
-  return async (...params) => {
+  return async (...params: any[]) => {
     return new Promise(resolve => {
       if (timeout) {
         clearTimeout(timeout)
@@ -70,11 +79,11 @@ export const debounce = (callback, minDelay = 1000) => {
  * @param minDelay
  * @returns {Function} a throttled version function
  */
-export const throttle = (callback, minDelay = 1000) => {
+export const throttle = (callback: Function, minDelay = 1000) => {
-  let lastParams
+  let lastParams: any[]
   let stalled = false
   let pending = false
-  const invoke = (...params) => {
+  const invoke = (...params: any[]) => {
     lastParams = params
     if (stalled) {
       pending = true
@@ -98,10 +107,10 @@ export const throttle = (callback, minDelay = 1000) => {
  * @param callback the function to run
  * @returns {Function}
  */
-export const domDebounce = callback => {
+export const domDebounce = (callback: Function) => {
   let active = false
-  let lastParams
+  let lastParams: any[]
-  return (...params) => {
+  return (...params: any[]) => {
     lastParams = params
     if (!active) {
       active = true
@@ -119,7 +128,17 @@ export const domDebounce = callback => {
  *
  * @param {any} props
  * */
-export const buildFormBlockButtonConfig = props => {
+export const buildFormBlockButtonConfig = (props?: {
+  _id?: string
+  actionType?: string
+  dataSource?: { resourceId: string }
+  notificationOverride?: boolean
+  actionUrl?: string
+  showDeleteButton?: boolean
+  deleteButtonLabel?: string
+  showSaveButton?: boolean
+  saveButtonLabel?: string
+}) => {
   const {
     _id,
     actionType,
@@ -227,7 +246,11 @@ export const buildFormBlockButtonConfig = props => {
 
   const defaultButtons = []
 
-  if (["Update", "Create"].includes(actionType) && showSaveButton !== false) {
+  if (
+    actionType &&
+    ["Update", "Create"].includes(actionType) &&
+    showSaveButton !== false
+  ) {
     defaultButtons.push({
       text: saveText || "Save",
       _id: Helpers.uuid(),
@@ -251,7 +274,13 @@ export const buildFormBlockButtonConfig = props => {
   return defaultButtons
 }
 
-export const buildMultiStepFormBlockDefaultProps = props => {
+export const buildMultiStepFormBlockDefaultProps = (props?: {
+  _id: string
+  stepCount: number
+  currentStep: number
+  actionType: string
+  dataSource: { resourceId: string }
+}) => {
   const { _id, stepCount, currentStep, actionType, dataSource } = props || {}
 
   // Sanity check
@@ -361,7 +390,7 @@ export const buildMultiStepFormBlockDefaultProps = props => {
  * @param {Object} filter UI filter
  * @returns {Object} parsed filter
  */
-export function parseFilter(filter) {
+export function parseFilter(filter: UISearchFilter) {
   if (!filter?.groups) {
     return filter
   }
@@ -369,13 +398,13 @@ export function parseFilter(filter) {
   const update = cloneDeep(filter)
 
   update.groups = update.groups
-    .map(group => {
+    ?.map(group => {
-      group.filters = group.filters.filter(filter => {
+      group.filters = group.filters?.filter((filter: any) => {
         return filter.field && filter.operator
       })
-      return group.filters.length ? group : null
+      return group.filters?.length ? group : null
     })
-    .filter(group => group)
+    .filter((group): group is SearchFilterGroup => !!group)
 
   return update
 }

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -634,6 +634,130 @@ if (descriptions.length) {
             }
           })
         })
+
+        it("should be able to select a ObjectId in a transformer", async () => {
+          const query = await createQuery({
+            fields: {
+              json: {},
+              extra: {
+                actionType: "find",
+              },
+            },
+            transformer: "return data.map(x => ({ id: x._id }))",
+          })
+
+          const result = await config.api.query.execute(query._id!)
+
+          expect(result.data).toEqual([
+            { id: expectValidId },
+            { id: expectValidId },
+            { id: expectValidId },
+            { id: expectValidId },
+            { id: expectValidId },
+          ])
+        })
+
+        it("can handle all bson field types with transformers", async () => {
+          collection = generator.guid()
+          await withCollection(async collection => {
+            await collection.insertOne({
+              _id: new BSON.ObjectId("65b0123456789abcdef01234"),
+              stringField: "This is a string",
+              numberField: 42,
+              doubleField: new BSON.Double(42.42),
+              integerField: new BSON.Int32(123),
+              longField: new BSON.Long("9223372036854775807"),
+              booleanField: true,
+              nullField: null,
+              arrayField: [1, 2, 3, "four", { nested: true }],
+              objectField: {
+                nestedString: "nested",
+                nestedNumber: 99,
+              },
+              dateField: new Date(Date.UTC(2025, 0, 30, 12, 30, 20)),
+              timestampField: new BSON.Timestamp({ t: 1706616000, i: 1 }),
+              binaryField: new BSON.Binary(
+                new TextEncoder().encode("bufferValue")
+              ),
+              objectIdField: new BSON.ObjectId("65b0123456789abcdef01235"),
+              regexField: new BSON.BSONRegExp("^Hello.*", "i"),
+              minKeyField: new BSON.MinKey(),
+              maxKeyField: new BSON.MaxKey(),
+              decimalField: new BSON.Decimal128("12345.6789"),
+              codeField: new BSON.Code(
+                "function() { return 'Hello, World!'; }"
+              ),
+              codeWithScopeField: new BSON.Code(
+                "function(x) { return x * 2; }",
+                { x: 10 }
+              ),
+            })
+          })
+
+          const query = await createQuery({
+            fields: {
+              json: {},
+              extra: {
+                actionType: "find",
+                collection,
+              },
+            },
+            transformer: `return data.map(x => ({ 
+                  ...x,
+                  binaryField: x.binaryField?.toString('utf8'),
+                  decimalField: x.decimalField.toString(),
+                  longField: x.longField.toString(),
+                  regexField: x.regexField.toString(),
+                  // TODO: currenlty not supported, it looks like there is bug in the library. Getting: Timestamp constructed from { t, i } must provide t as a number
+                  timestampField: null
+              }))`,
+          })
+
+          const result = await config.api.query.execute(query._id!)
+
+          expect(result.data).toEqual([
+            {
+              _id: "65b0123456789abcdef01234",
+              arrayField: [
+                1,
+                2,
+                3,
+                "four",
+                {
+                  nested: true,
+                },
+              ],
+              binaryField: "bufferValue",
+              booleanField: true,
+              codeField: {
+                code: "function() { return 'Hello, World!'; }",
+              },
+              codeWithScopeField: {
+                code: "function(x) { return x * 2; }",
+                scope: {
+                  x: 10,
+                },
+              },
+              dateField: "2025-01-30T12:30:20.000Z",
+              decimalField: "12345.6789",
+              doubleField: 42.42,
+              integerField: 123,
+              longField: "9223372036854775807",
+              maxKeyField: {},
+              minKeyField: {},
+              nullField: null,
+              numberField: 42,
+              objectField: {
+                nestedNumber: 99,
+                nestedString: "nested",
+              },
+              objectIdField: "65b0123456789abcdef01235",
+              regexField: "/^Hello.*/i",
+              stringField: "This is a string",
+              timestampField: null,
+            },
+          ])
+        })
       })
 
       it("should throw an error if the incorrect actionType is specified", async () => {

packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js

@@ -0,0 +1,122 @@
+if (typeof btoa !== "function") {
+  var chars = {
+    ascii: function () {
+      return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+    },
+    indices: function () {
+      if (!this.cache) {
+        this.cache = {}
+        var ascii = chars.ascii()
+
+        for (var c = 0; c < ascii.length; c++) {
+          var chr = ascii[c]
+          this.cache[chr] = c
+        }
+      }
+      return this.cache
+    },
+  }
+
+  function atob(b64) {
+    var indices = chars.indices(),
+      pos = b64.indexOf("="),
+      padded = pos > -1,
+      len = padded ? pos : b64.length,
+      i = -1,
+      data = ""
+
+    while (i < len) {
+      var code =
+        (indices[b64[++i]] << 18) |
+        (indices[b64[++i]] << 12) |
+        (indices[b64[++i]] << 6) |
+        indices[b64[++i]]
+      if (code !== 0) {
+        data += String.fromCharCode(
+          (code >>> 16) & 255,
+          (code >>> 8) & 255,
+          code & 255
+        )
+      }
+    }
+
+    if (padded) {
+      data = data.slice(0, pos - b64.length)
+    }
+
+    return data
+  }
+
+  function btoa(data) {
+    var ascii = chars.ascii(),
+      len = data.length - 1,
+      i = -1,
+      b64 = ""
+
+    while (i < len) {
+      var code =
+        (data.charCodeAt(++i) << 16) |
+        (data.charCodeAt(++i) << 8) |
+        data.charCodeAt(++i)
+      b64 +=
+        ascii[(code >>> 18) & 63] +
+        ascii[(code >>> 12) & 63] +
+        ascii[(code >>> 6) & 63] +
+        ascii[code & 63]
+    }
+
+    var pads = data.length % 3
+    if (pads > 0) {
+      b64 = b64.slice(0, pads - 3)
+
+      while (b64.length % 4 !== 0) {
+        b64 += "="
+      }
+    }
+
+    return b64
+  }
+}
+
+if (typeof TextDecoder === "undefined") {
+  globalThis.TextDecoder = class {
+    constructor(encoding = "utf8") {
+      if (encoding !== "utf8") {
+        throw new Error(
+          `Only UTF-8 is supported in this polyfill. Recieved: ${encoding}`
+        )
+      }
+    }
+    decode(buffer) {
+      return String.fromCharCode(...buffer)
+    }
+  }
+}
+
+if (typeof TextEncoder === "undefined") {
+  globalThis.TextEncoder = class {
+    encode(str) {
+      const utf8 = []
+      for (const i = 0; i < str.length; i++) {
+        const codePoint = str.charCodeAt(i)
+
+        if (codePoint < 0x80) {
+          utf8.push(codePoint)
+        } else if (codePoint < 0x800) {
+          utf8.push(0xc0 | (codePoint >> 6))
+          utf8.push(0x80 | (codePoint & 0x3f))
+        } else if (codePoint < 0x10000) {
+          utf8.push(0xe0 | (codePoint >> 12))
+          utf8.push(0x80 | ((codePoint >> 6) & 0x3f))
+          utf8.push(0x80 | (codePoint & 0x3f))
+        } else {
+          utf8.push(0xf0 | (codePoint >> 18))
+          utf8.push(0x80 | ((codePoint >> 12) & 0x3f))
+          utf8.push(0x80 | ((codePoint >> 6) & 0x3f))
+          utf8.push(0x80 | (codePoint & 0x3f))
+        }
+      }
+      return new Uint8Array(utf8)
+    }
+  }
+}

packages/server/src/jsRunner/bundles/index.ts

@@ -5,6 +5,7 @@ export const enum BundleType {
   BSON = "bson",
   SNIPPETS = "snippets",
   BUFFER = "buffer",
+  BSON_POLYFILLS = "bson_polyfills",
 }
 
 const bundleSourceFile: Record<BundleType, string> = {
@@ -12,6 +13,7 @@ const bundleSourceFile: Record<BundleType, string> = {
   [BundleType.BSON]: "./bson.ivm.bundle.js",
   [BundleType.SNIPPETS]: "./snippets.ivm.bundle.js",
   [BundleType.BUFFER]: "./buffer.ivm.bundle.js",
+  [BundleType.BSON_POLYFILLS]: "./bson-polyfills.ivm.bundle.js",
 }
 const bundleSourceCode: Partial<Record<BundleType, string>> = {}
 

packages/server/src/jsRunner/bundles/snippets.ts

@@ -1,6 +1,5 @@
 // @ts-ignore
-// eslint-disable-next-line local-rules/no-budibase-imports
+import { iifeWrapper } from "@budibase/string-templates"
-import { iifeWrapper } from "@budibase/string-templates/iife"
 
 export default new Proxy(
   {},

packages/server/src/jsRunner/vm/isolated-vm.ts

@@ -161,42 +161,10 @@ export class IsolatedVM implements VM {
 
     const bsonSource = loadBundle(BundleType.BSON)
 
-    this.addToContext({
+    const bsonPolyfills = loadBundle(BundleType.BSON_POLYFILLS)
-      textDecoderCb: new ivm.Callback(
-        (args: {
-          constructorArgs: any
-          functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
-        }) => {
-          const result = new TextDecoder(...args.constructorArgs).decode(
-            ...args.functionArgs
-          )
-          return result
-        }
-      ),
-    })
-
-    // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
-    const textDecoderPolyfill = class TextDecoderMock {
-      constructorArgs
-
-      constructor(...constructorArgs: any) {
-        this.constructorArgs = constructorArgs
-      }
-
-      decode(...input: any) {
-        // @ts-expect-error - this is going to run in the isolate, where this function will be available
-        // eslint-disable-next-line no-undef
-        return textDecoderCb({
-          constructorArgs: this.constructorArgs,
-          functionArgs: input,
-        })
-      }
-    }
-      .toString()
-      .replace(/TextDecoderMock/, "TextDecoder")
 
     const script = this.isolate.compileScriptSync(
-      `${textDecoderPolyfill};${bsonSource}`
+      `${bsonPolyfills};${bsonSource}`
     )
     script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
     new Promise(() => {

packages/string-templates/package.json

@@ -11,8 +11,7 @@
       "require": "./dist/bundle.cjs",
       "import": "./dist/bundle.mjs"
     },
-    "./package.json": "./package.json",
+    "./package.json": "./package.json"
-    "./iife": "./dist/iife.mjs"
   },
   "scripts": {
     "build": "tsc --emitDeclarationOnly && rollup -c",

packages/types/src/sdk/featureFlag.ts

@@ -1,6 +1,6 @@
 export enum FeatureFlag {
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
-  CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS = "CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS",
+  CHECK_COMPONENT_SETTINGS_ERRORS = "CHECK_COMPONENT_SETTINGS_ERRORS",
 
   // Account-portal
   DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL",
@@ -8,7 +8,7 @@ export enum FeatureFlag {
 
 export const FeatureFlagDefaults = {
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
-  [FeatureFlag.CHECK_SCREEN_COMPONENT_SETTINGS_ERRORS]: false,
+  [FeatureFlag.CHECK_COMPONENT_SETTINGS_ERRORS]: false,
 
   // Account-portal
   [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,