Merge branch 'master' into fix/use-direct-describe

examples/nextjs-api-sales/yarn.lock

@@ -423,9 +423,9 @@ core-js-pure@^3.20.2:
   integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ==
 
 cross-spawn@^7.0.2:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.5",
+  "version": "3.2.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/.npmignore

@@ -1,6 +1,4 @@
 *
 !dist/**/*
 dist/tsconfig.build.tsbuildinfo
-!package.json
-!src/**
-!tests/**
+!package.json

packages/backend-core/package.json

@@ -9,6 +9,13 @@
     "./tests": "./dist/tests/index.js",
     "./*": "./dist/*.js"
   },
+  "typesVersions": {
+    "*": {
+      "tests": [
+        "dist/tests/index.d.ts"
+      ]
+    }
+  },
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {

packages/backend-core/src/environment.ts

@@ -19,6 +19,12 @@ function isDev() {
   return process.env.NODE_ENV !== "production"
 }
 
+function parseIntSafe(number?: string) {
+  if (number) {
+    return parseInt(number)
+  }
+}
+
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
   require("dotenv").config()
@@ -231,6 +237,7 @@ const environment = {
   MIN_VERSION_WITHOUT_POWER_ROLE:
     process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
   DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
+  BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
 }
 
 export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

packages/client/src/utils/blocks.js

@@ -1,5 +1,7 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { API } from "../api/index.js"
+import { UILogicalOperator } from "@budibase/types"
+import { OnEmptyFilter } from "@budibase/frontend-core/src/constants.js"
 
 // Map of data types to component types for search fields inside blocks
 const schemaComponentMap = {
@@ -60,7 +62,11 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
  * @param formId the ID of the form containing the search fields
  */
 export const enrichFilter = (filter, columns, formId) => {
-  let enrichedFilter = [...(filter || [])]
+  if (!columns?.length) {
+    return filter
+  }
+
+  let newFilters = []
   columns?.forEach(column => {
     const safePath = column.name.split(".").map(safe).join(".")
     const stringType = column.type === "string" || column.type === "formula"
@@ -69,7 +75,7 @@ export const enrichFilter = (filter, columns, formId) => {
 
     // For dates, use a range of the entire day selected
     if (dateType) {
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: "rangeLow",
@@ -79,7 +85,7 @@ export const enrichFilter = (filter, columns, formId) => {
       const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
       let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
       hbs = `{{#if ${binding} }}${hbs}{{/if}}`
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: "rangeHigh",
@@ -90,7 +96,7 @@ export const enrichFilter = (filter, columns, formId) => {
 
     // For other fields, do an exact match
     else {
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: stringType ? "string" : "equal",
@@ -99,5 +105,16 @@ export const enrichFilter = (filter, columns, formId) => {
       })
     }
   })
-  return enrichedFilter
+
+  return {
+    logicalOperator: UILogicalOperator.ALL,
+    onEmptyFilter: OnEmptyFilter.RETURN_ALL,
+    groups: [
+      ...(filter?.groups || []),
+      {
+        logicalOperator: UILogicalOperator.ALL,
+        filters: newFilters,
+      },
+    ],
+  }
 }

packages/server/src/api/controllers/query/index.ts

@@ -23,6 +23,7 @@ import {
 } from "@budibase/types"
 import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
 import { findHBSBlocks } from "@budibase/string-templates"
+import { ObjectId } from "mongodb"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT,
@@ -223,6 +224,8 @@ export async function preview(
             } else {
               fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
             }
+          } else if (field instanceof ObjectId) {
+            fieldMetadata = makeQuerySchema(FieldType.STRING, key)
           } else {
             fieldMetadata = makeQuerySchema(FieldType.JSON, key)
           }

packages/server/src/startup/index.ts

@@ -28,6 +28,7 @@ import Koa from "koa"
 import { Server } from "http"
 import { AddressInfo } from "net"
 import fs from "fs"
+import bson from "bson"
 
 let STARTUP_RAN = false
 
@@ -193,6 +194,10 @@ export async function startup(
     })
   }
 
+  if (coreEnv.BSON_BUFFER_SIZE) {
+    bson.setInternalBufferSize(coreEnv.BSON_BUFFER_SIZE)
+  }
+
   console.log("Initialising JS runner")
   jsRunner.init()
 }

packages/server/src/threads/query.ts

@@ -136,21 +136,23 @@ class QueryRunner {
       pagination = output.pagination
     }
 
-    // transform as required
-    if (transformer) {
+    // We avoid invoking the transformer if it's trivial because there is a cost
+    // to passing data in and out of the isolate, especially for MongoDB where
+    // we have to bson serialise/deserialise the data.
+    const hasTransformer =
+      transformer != null &&
+      transformer.length > 0 &&
+      transformer.trim() !== "return data" &&
+      transformer.trim() !== "return data;"
+
+    if (transformer && hasTransformer) {
       transformer = iifeWrapper(transformer)
       let vm = new IsolatedVM()
       if (datasource.source === SourceName.MONGODB) {
         vm = vm.withParsingBson(rows)
       }
-
-      const ctx = {
-        data: rows,
-        params: enrichedParameters,
-      }
-      if (transformer != null) {
-        rows = vm.withContext(ctx, () => vm.execute(transformer!))
-      }
+      const ctx = { data: rows, params: enrichedParameters }
+      rows = vm.withContext(ctx, () => vm.execute(transformer!))
     }
 
     // if the request fails we retry once, invalidating the cached value

packages/server/tsconfig.json

@@ -1,6 +1,7 @@
 {
   "extends": "./tsconfig.build.json",
   "compilerOptions": {
+    "lib": ["es2020", "dom"],
     "composite": true,
     "baseUrl": "."
   },

packages/string-templates/.npmignore

@@ -0,0 +1,4 @@
+*
+!dist/**/*
+dist/tsconfig.build.tsbuildinfo
+!package.json

packages/string-templates/package.json

@@ -4,7 +4,7 @@
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "dist/bundle.cjs",
   "module": "dist/bundle.mjs",
-  "types": "src/index.ts",
+  "types": "dist/index.d.ts",
   "license": "MPL-2.0",
   "exports": {
     ".": {
@@ -12,12 +12,8 @@
       "import": "./dist/bundle.mjs"
     },
     "./package.json": "./package.json",
-    "./iife": "./src/iife.js"
+    "./iife": "./dist/iife.mjs"
   },
-  "files": [
-    "dist",
-    "src"
-  ],
   "scripts": {
     "build": "tsc --emitDeclarationOnly && rollup -c",
     "dev": "rollup -cw",

packages/string-templates/rollup.config.js

@@ -10,8 +10,8 @@ import inject from "@rollup/plugin-inject"
 
 const production = !process.env.ROLLUP_WATCH
 
-const config = (format, outputFile) => ({
-  input: "src/index.ts",
+const config = (input, outputFile, format) => ({
+  input,
   output: {
     sourcemap: !production,
     format,
@@ -42,6 +42,7 @@ const config = (format, outputFile) => ({
 })
 
 export default [
-  config("cjs", "./dist/bundle.cjs"),
-  config("esm", "./dist/bundle.mjs"),
+  config("src/index.ts", "./dist/bundle.cjs", "cjs"),
+  config("src/index.ts", "./dist/bundle.mjs", "esm"),
+  config("src/iife.ts", "./dist/iife.mjs", "esm"),
 ]

packages/worker/src/index.ts

@@ -99,6 +99,7 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
     startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"`
   }
   console.log(startupLog)
+
   await initPro()
   await redis.clients.init()
   features.init()