Merge pull request #6895 from Budibase/bug/sev2/mongodb-fixes

Support ObjectId in MongoDB operators

packages/server/__mocks__/mongodb.ts

@@ -8,6 +8,7 @@ module MongoMock {
     this.insertMany = jest.fn(() => ({ toArray: () => [] }))
     this.find = jest.fn(() => ({ toArray: () => [] }))
     this.findOne = jest.fn()
+    this.findOneAndUpdate = jest.fn()
     this.count = jest.fn()
     this.deleteOne = jest.fn()
     this.deleteMany = jest.fn(() => ({ toArray: () => [] }))
@@ -19,6 +20,7 @@ module MongoMock {
       find: this.find,
       insertMany: this.insertMany,
       findOne: this.findOne,
+      findOneAndUpdate: this.findOneAndUpdate,
       count: this.count,
       deleteOne: this.deleteOne,
       deleteMany: this.deleteMany,
@@ -31,5 +33,7 @@ module MongoMock {
     })
   }
 
+  mongodb.ObjectID = require("mongodb").ObjectID
+
   module.exports = mongodb
 }

packages/server/src/integrations/mongodb.ts

@@ -92,12 +92,15 @@ module MongoDBModule {
           if (json[field] instanceof Object) {
             json[field] = self.createObjectIds(json[field])
           }
-          if (field === "_id" && typeof json[field] === "string") {
+          if (
-            const id = json["_id"].match(
+            (field === "_id" || field?.startsWith("$")) &&
+            typeof json[field] === "string"
+          ) {
+            const id = json[field].match(
               /(?<=objectid\(['"]).*(?=['"]\))/gi
             )?.[0]
             if (id) {
-              json["_id"] = ObjectID.createFromHexString(id)
+              json[field] = ObjectID.createFromHexString(id)
             }
           }
         }
@@ -114,10 +117,31 @@ module MongoDBModule {
     }
 
     parseQueryParams(params: string, mode: string) {
-      let queryParams = params.split(/(?<=}),[\n\s]*(?={)/g)
+      let queryParams = []
-      let group1 = queryParams[0] ? JSON.parse(queryParams[0]) : {}
+      let openCount = 0
-      let group2 = queryParams[1] ? JSON.parse(queryParams[1]) : {}
+      let inQuotes = false
-      let group3 = queryParams[2] ? JSON.parse(queryParams[2]) : {}
+      let i = 0
+      let startIndex = 0
+      for (let c of params) {
+        if (c === '"' && i > 0 && params[i - 1] !== "\\") {
+          inQuotes = !inQuotes
+        }
+        if (c === "{" && !inQuotes) {
+          openCount++
+          if (openCount === 1) {
+            startIndex = i
+          }
+        } else if (c === "}" && !inQuotes) {
+          if (openCount === 1) {
+            queryParams.push(JSON.parse(params.substring(startIndex, i + 1)))
+          }
+          openCount--
+        }
+        i++
+      }
+      let group1 = queryParams[0] ?? {}
+      let group2 = queryParams[1] ?? {}
+      let group3 = queryParams[2] ?? {}
       if (mode === "update") {
         return {
           filter: group1,
@@ -176,7 +200,10 @@ module MongoDBModule {
             return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            let findAndUpdateJson = json as {
+            if (typeof query.json === "string") {
+              json = this.parseQueryParams(query.json, "update")
+            }
+            let findAndUpdateJson = this.createObjectIds(json) as {
               filter: FilterQuery<any>
               update: UpdateQuery<any>
               options: FindOneAndUpdateOption<any>

packages/server/src/integrations/tests/mongo.spec.js

@@ -102,4 +102,222 @@ describe("MongoDB Integration", () => {
     expect(error).toBeDefined()
     restore()
   })
+
+  it("creates ObjectIds if the _id fields contains a match on ObjectId", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: "ObjectId('ACBD12345678ABCD12345678')",
+          name: "ObjectId('name')"
+        },
+        update: {
+          _id: "ObjectId('FFFF12345678ABCD12345678')",
+          name: "ObjectId('updatedName')",
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      name: "ObjectId('name')",
+    })
+    expect(args[1]).toEqual({
+      _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+      name: "ObjectId('updatedName')",
+    })
+    expect(args[2]).toEqual({
+      upsert: false
+    })
+  })
+
+  it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: {
+            $eq: "ObjectId('ACBD12345678ABCD12345678')",
+          }
+        },
+        update: {
+          $set: {
+            _id: "ObjectId('FFFF12345678ABCD12345678')",
+          },
+        },
+        options: {
+          upsert: true,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+      }
+    })
+    expect(args[2]).toEqual({
+      upsert: true
+    })
+  })
+
+  it("supports findOneAndUpdate", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: {
+            $eq: "ObjectId('ACBD12345678ABCD12345678')",
+          }
+        },
+        update: {
+          $set: {
+            name: "UPDATED",
+            age: 99
+          },
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" },
+    }
+    await config.integration.read(query)
+    expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
+    
+    const args = config.integration.client.findOneAndUpdate.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        name: "UPDATED",
+        age: 99
+      }
+    })
+    expect(args[2]).toEqual({
+      upsert: false
+    })
+  })
+
+  it("can parse nested objects with arrays", async () => {
+    const query = {
+      json: `{
+          "_id": {
+            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
+          }
+        },
+        {
+          "$set": {
+            "value": {
+              "data": [
+                { "cid": 1 },
+                { "cid": 2 },
+                { "nested": {
+                  "name": "test"
+                }}
+              ]
+            }
+          }
+        },
+        {
+          "upsert": true
+        }`,
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        value: {
+          data: [
+            { cid: 1 },
+            { cid: 2 },
+            { nested: {
+              name: "test"
+            }}
+          ]
+        },
+      },
+    })
+    expect(args[2]).toEqual({
+      upsert: true
+    })
+  })
+
+  it("ignores braces within strings when parsing nested objects", async () => {
+    const query = {
+      json: `{
+          "_id": {
+            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
+          }
+        },
+        {
+          "$set": {
+            "value": {
+              "data": [
+                { "cid": 1 },
+                { "cid": 2 },
+                { "nested": {
+                  "name": "te}st"
+                }}
+              ]
+            }
+          }
+        },
+        {
+          "upsert": true,
+          "extra": "ad\\"{\\"d"
+        }`,
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        value: {
+          data: [
+            { cid: 1 },
+            { cid: 2 },
+            { nested: {
+              name: "te}st"
+            }}
+          ]
+        },
+      },
+    })
+    expect(args[2]).toEqual({
+      upsert: true,
+      extra: "ad\"{\"d"
+    })
+  })
 })