Merge pull request #5877 from Budibase/fix/5613-5387

Misc MongoDB Connector Fixes

package.json

@@ -3,6 +3,8 @@
   "private": true,
   "devDependencies": {
     "@rollup/plugin-json": "^4.0.2",
+    "@types/mongodb": "3.6.3",
+    "@typescript-eslint/parser": "4.28.0",
     "babel-eslint": "^10.0.3",
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
@@ -16,7 +18,6 @@
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
     "svelte": "^3.38.2",
-    "@typescript-eslint/parser": "4.28.0",
     "typescript": "4.5.5"
   },
   "scripts": {

packages/server/src/integrations/mongodb.ts

@@ -4,10 +4,18 @@ import {
   QueryTypes,
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
+import {
+  MongoClient,
+  ObjectID,
+  FilterQuery,
+  UpdateQuery,
+  FindOneAndUpdateOption,
+  UpdateOneOptions,
+  UpdateManyOptions,
+  CommonOptions,
+} from "mongodb"
 
 module MongoDBModule {
-  const { MongoClient } = require("mongodb")
-
   interface MongoDBConfig {
     connectionString: string
     db: string
@@ -76,20 +84,76 @@ module MongoDBModule {
       return this.client.connect()
     }
 
+    createObjectIds(json: any): object {
+      const self = this
+      function interpolateObjectIds(json: any) {
+        for (let field of Object.keys(json)) {
+          if (json[field] instanceof Object) {
+            json[field] = self.createObjectIds(json[field])
+          }
+          if (field === "_id" && typeof json[field] === "string") {
+            const id = json["_id"].match(
+              /(?<=objectid\(['"]).*(?=['"]\))/gi
+            )?.[0]
+            if (id) {
+              json["_id"] = ObjectID.createFromHexString(id)
+            }
+          }
+        }
+        return json
+      }
+
+      if (Array.isArray(json)) {
+        for (let i = 0; i < json.length; i++) {
+          json[i] = interpolateObjectIds(json[i])
+        }
+        return json
+      }
+      return interpolateObjectIds(json)
+    }
+
+    parseQueryParams(params: string, mode: string) {
+      let queryParams = params.split(/(?<=(},)).*{/g)
+      let group1 = queryParams[0]
+      let group2 = queryParams[2]
+      let group3 = queryParams[4]
+      if (group1) {
+        group1 = JSON.parse(group1.replace(/,+$/, ""))
+      }
+      if (group2) {
+        group2 = JSON.parse("{" + group2.replace(/,+$/, ""))
+      }
+      if (group3) {
+        group3 = JSON.parse("{" + group3.replace(/,+$/, ""))
+      }
+      if (mode === "update") {
+        return {
+          filter: group1,
+          update: group2,
+          options: group3 ?? {},
+        }
+      }
+      return {
+        filter: group1,
+        options: group2 ?? {},
+      }
+    }
+
     async create(query: { json: object; extra: { [key: string]: string } }) {
       try {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let json = this.createObjectIds(query.json)
 
         // For mongodb we add an extra actionType to specify
         // which method we want to call on the collection
         switch (query.extra.actionTypes) {
           case "insertOne": {
-            return await collection.insertOne(query.json)
+            return await collection.insertOne(json)
           }
           case "insertMany": {
-            return await collection.insertOne(query.json).toArray()
+            return await collection.insertMany(json)
           }
           default: {
             throw new Error(
@@ -110,22 +174,32 @@ module MongoDBModule {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let json = this.createObjectIds(query.json)
 
         switch (query.extra.actionTypes) {
           case "find": {
-            return await collection.find(query.json).toArray()
+            return await collection.find(json).toArray()
           }
           case "findOne": {
-            return await collection.findOne(query.json)
+            return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            return await collection.findOneAndUpdate(query.json)
+            let findAndUpdateJson = json as {
+              filter: FilterQuery<any>
+              update: UpdateQuery<any>
+              options: FindOneAndUpdateOption<any>
+            }
+            return await collection.findOneAndUpdate(
+              findAndUpdateJson.filter,
+              findAndUpdateJson.update,
+              findAndUpdateJson.options
+            )
           }
           case "count": {
-            return await collection.countDocuments(query.json)
+            return await collection.countDocuments(json)
           }
           case "distinct": {
-            return await collection.distinct(query.json)
+            return await collection.distinct(json)
           }
           default: {
             throw new Error(
@@ -146,13 +220,30 @@ module MongoDBModule {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let queryJson = query.json
+        if (typeof queryJson === "string") {
+          queryJson = this.parseQueryParams(queryJson, "update")
+        }
+        let json = this.createObjectIds(queryJson) as {
+          filter: FilterQuery<any>
+          update: UpdateQuery<any>
+          options: object
+        }
 
         switch (query.extra.actionTypes) {
           case "updateOne": {
-            return await collection.updateOne(query.json)
+            return await collection.updateOne(
+              json.filter,
+              json.update,
+              json.options as UpdateOneOptions
+            )
           }
           case "updateMany": {
-            return await collection.updateMany(query.json).toArray()
+            return await collection.updateMany(
+              json.filter,
+              json.update,
+              json.options as UpdateManyOptions
+            )
           }
           default: {
             throw new Error(
@@ -173,13 +264,21 @@ module MongoDBModule {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let queryJson = query.json
+        if (typeof queryJson === "string") {
+          queryJson = this.parseQueryParams(queryJson, "delete")
+        }
+        let json = this.createObjectIds(queryJson) as {
+          filter: FilterQuery<any>
+          options: CommonOptions
+        }
 
         switch (query.extra.actionTypes) {
           case "deleteOne": {
-            return await collection.deleteOne(query.json)
+            return await collection.deleteOne(json.filter, json.options)
           }
           case "deleteMany": {
-            return await collection.deleteMany(query.json).toArray()
+            return await collection.deleteMany(json.filter, json.options)
           }
           default: {
             throw new Error(

packages/server/src/integrations/tests/mongo.spec.js

@@ -9,10 +9,10 @@ class TestConfiguration {
 }
 
 function disableConsole() {
-  jest.spyOn(console, 'error');
+  jest.spyOn(console, "error")
-  console.error.mockImplementation(() => {});
+  console.error.mockImplementation(() => {})
 
-  return console.error.mockRestore;
+  return console.error.mockRestore
 }
 
 describe("MongoDB Integration", () => {
@@ -25,12 +25,12 @@ describe("MongoDB Integration", () => {
 
   it("calls the create method with the correct params", async () => {
     const body = {
-      name: "Hello"
+      name: "Hello",
     }
     await config.integration.create({
       index: indexName,
       json: body,
-      extra: { collection: 'testCollection', actionTypes: 'insertOne'}
+      extra: { collection: "testCollection", actionTypes: "insertOne" },
     })
     expect(config.integration.client.insertOne).toHaveBeenCalledWith(body)
   })
@@ -38,9 +38,9 @@ describe("MongoDB Integration", () => {
   it("calls the read method with the correct params", async () => {
     const query = {
       json: {
-        address: "test"
+        address: "test",
       },
-      extra: { collection: 'testCollection', actionTypes: 'find'}
+      extra: { collection: "testCollection", actionTypes: "find" },
     }
     const response = await config.integration.read(query)
     expect(config.integration.client.find).toHaveBeenCalledWith(query.json)
@@ -50,30 +50,47 @@ describe("MongoDB Integration", () => {
   it("calls the delete method with the correct params", async () => {
     const query = {
       json: {
-        id: "test"
+        filter: {
+          id: "test",
         },
-      extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
+        options: {
+          opt: "option"
+        }
+      },
+      extra: { collection: "testCollection", actionTypes: "deleteOne" },
     }
     await config.integration.delete(query)
-    expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json)
+    expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json.filter, query.json.options)
   })
 
   it("calls the update method with the correct params", async () => {
     const query = {
       json: {
-        id: "test"
+        filter: {
+          id: "test",
         },
-      extra: { collection: 'testCollection', actionTypes: 'updateOne'}
+        update: {
+          name: "TestName",
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
     }
     await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json)
+    expect(config.integration.client.updateOne).toHaveBeenCalledWith(
+      query.json.filter,
+      query.json.update,
+      query.json.options
+    )
   })
 
   it("throws an error when an invalid query.extra.actionType is passed for each method", async () => {
     const restore = disableConsole()
 
     const query = {
-      extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
+      extra: { collection: "testCollection", actionTypes: "deleteOne" },
     }
 
     let error = null

yarn.lock

@@ -972,6 +972,13 @@
     estree-walker "^1.0.1"
     picomatch "^2.2.2"
 
+"@types/bson@*":
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337"
+  integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg==
+  dependencies:
+    bson "*"
+
 "@types/estree@0.0.39":
   version "0.0.39"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
@@ -982,6 +989,19 @@
   resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c"
   integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==
 
+"@types/mongodb@3.6.3":
+  version "3.6.3"
+  resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.3.tgz#5655af409d9e32d5d5ae9a653abf3e5f9c83eb7a"
+  integrity sha512-6YNqGP1hk5bjUFaim+QoFFuI61WjHiHE1BNeB41TA00Xd2K7zG4lcWyLLq/XtIp36uMavvS5hoAUJ+1u/GcX2Q==
+  dependencies:
+    "@types/bson" "*"
+    "@types/node" "*"
+
+"@types/node@*":
+  version "17.0.33"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.33.tgz#3c1879b276dc63e73030bb91165e62a4509cd506"
+  integrity sha512-miWq2m2FiQZmaHfdZNcbpp9PuXg34W5JZ5CrJ/BaS70VuhoJENBEQybeiYSaPBRNq6KQGnjfEnc/F3PN++D+XQ==
+
 "@types/node@>= 8":
   version "17.0.18"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.18.tgz#3b4fed5cfb58010e3a2be4b6e74615e4847f1074"
@@ -1300,6 +1320,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
+base64-js@^1.3.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
 base@^0.11.1:
   version "0.11.2"
   resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f"
@@ -1361,6 +1386,13 @@ braces@^3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
+bson@*:
+  version "4.6.3"
+  resolved "https://registry.yarnpkg.com/bson/-/bson-4.6.3.tgz#d1a9a0b84b9e84b62390811fc5580f6a8b1d858c"
+  integrity sha512-rAqP5hcUVJhXP2MCSNVsf0oM2OGU1So6A9pVRDYayvJ5+hygXHQApf87wd5NlhPM1J9RJnbqxIG/f8QTzRoQ4A==
+  dependencies:
+    buffer "^5.6.0"
+
 btoa-lite@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337"
@@ -1371,6 +1403,14 @@ buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
   integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
 
+buffer@^5.6.0:
+  version "5.7.1"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
+  integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.1.13"
+
 builtins@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88"
@@ -2969,6 +3009,11 @@ iconv-lite@^0.6.2:
   dependencies:
     safer-buffer ">= 2.1.2 < 3.0.0"
 
+ieee754@^1.1.13:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
+  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
+
 iferr@^0.1.5:
   version "0.1.5"
   resolved "https://registry.yarnpkg.com/iferr/-/iferr-0.1.5.tgz#c60eed69e6d8fdb6b3104a1fcbca1c192dc5b501"
@@ -4663,12 +4708,7 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
-picomatch@^2.2.2:
+picomatch@^2.2.2, picomatch@^2.3.1:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
-  integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
-
-picomatch@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==