Merge pull request #15032 from Budibase/mongodb-transformer-tweaks

Bypass transformers that are trivial, allow users to configure bson buffer size.

packages/backend-core/src/environment.ts

@@ -19,6 +19,12 @@ function isDev() {
   return process.env.NODE_ENV !== "production"
 }
 
+function parseIntSafe(number?: string) {
+  if (number) {
+    return parseInt(number)
+  }
+}
+
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
   require("dotenv").config()
@@ -231,6 +237,7 @@ const environment = {
   MIN_VERSION_WITHOUT_POWER_ROLE:
     process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
   DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
+  BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
 }
 
 export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

packages/server/src/api/controllers/query/index.ts

@@ -23,6 +23,7 @@ import {
 } from "@budibase/types"
 import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
 import { findHBSBlocks } from "@budibase/string-templates"
+import { ObjectId } from "mongodb"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT,
@@ -223,6 +224,8 @@ export async function preview(
             } else {
               fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
             }
+          } else if (field instanceof ObjectId) {
+            fieldMetadata = makeQuerySchema(FieldType.STRING, key)
           } else {
             fieldMetadata = makeQuerySchema(FieldType.JSON, key)
           }

packages/server/src/startup/index.ts

@@ -28,6 +28,7 @@ import Koa from "koa"
 import { Server } from "http"
 import { AddressInfo } from "net"
 import fs from "fs"
+import bson from "bson"
 
 let STARTUP_RAN = false
 
@@ -193,6 +194,10 @@ export async function startup(
     })
   }
 
+  if (coreEnv.BSON_BUFFER_SIZE) {
+    bson.setInternalBufferSize(coreEnv.BSON_BUFFER_SIZE)
+  }
+
   console.log("Initialising JS runner")
   jsRunner.init()
 }

packages/server/src/threads/query.ts

@@ -136,21 +136,23 @@ class QueryRunner {
       pagination = output.pagination
     }
 
-    // transform as required
+    // We avoid invoking the transformer if it's trivial because there is a cost
-    if (transformer) {
+    // to passing data in and out of the isolate, especially for MongoDB where
+    // we have to bson serialise/deserialise the data.
+    const hasTransformer =
+      transformer != null &&
+      transformer.length > 0 &&
+      transformer.trim() !== "return data" &&
+      transformer.trim() !== "return data;"
+
+    if (transformer && hasTransformer) {
       transformer = iifeWrapper(transformer)
       let vm = new IsolatedVM()
       if (datasource.source === SourceName.MONGODB) {
         vm = vm.withParsingBson(rows)
       }
-
+      const ctx = { data: rows, params: enrichedParameters }
-      const ctx = {
+      rows = vm.withContext(ctx, () => vm.execute(transformer!))
-        data: rows,
-        params: enrichedParameters,
-      }
-      if (transformer != null) {
-        rows = vm.withContext(ctx, () => vm.execute(transformer!))
-      }
     }
 
     // if the request fails we retry once, invalidating the cached value

packages/worker/src/index.ts

@@ -24,6 +24,7 @@ import {
 db.init()
 import koaBody from "koa-body"
 import http from "http"
+import bson from "bson"
 import api from "./api"
 
 const koaSession = require("koa-session")
@@ -99,6 +100,11 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
     startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"`
   }
   console.log(startupLog)
+
+  if (coreEnv.BSON_BUFFER_SIZE) {
+    bson.setInternalBufferSize(coreEnv.BSON_BUFFER_SIZE)
+  }
+
   await initPro()
   await redis.clients.init()
   features.init()