From 6ff1e42899b833fb575a8e141085a17f3bb739f0 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 25 Aug 2022 11:02:46 +0100 Subject: [PATCH 01/21] Upgrade to mongodb@4.9 --- package.json | 1 - packages/server/__mocks__/mongodb.ts | 2 +- packages/server/package.json | 3 +- packages/server/src/integrations/mongodb.ts | 33 +++-- .../src/integrations/tests/mongo.spec.js | 18 +-- packages/server/yarn.lock | 130 +++++++++++------- 6 files changed, 110 insertions(+), 77 deletions(-) diff --git a/package.json b/package.json index 4c24e0025b..d5cc637fc2 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "private": true, "devDependencies": { "@rollup/plugin-json": "^4.0.2", - "@types/mongodb": "3.6.3", "@typescript-eslint/parser": "4.28.0", "babel-eslint": "^10.0.3", "eslint": "^7.28.0", diff --git a/packages/server/__mocks__/mongodb.ts b/packages/server/__mocks__/mongodb.ts index 92ec89227f..1655cfa96e 100644 --- a/packages/server/__mocks__/mongodb.ts +++ b/packages/server/__mocks__/mongodb.ts @@ -33,7 +33,7 @@ module MongoMock { }) } - mongodb.ObjectID = require("mongodb").ObjectID + mongodb.ObjectId = require("mongodb").ObjectId module.exports = mongodb } diff --git a/packages/server/package.json b/packages/server/package.json index 75c8f81d99..3966f2436d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -121,7 +121,7 @@ "koa2-ratelimit": "1.1.1", "lodash": "4.17.21", "memorystream": "0.3.1", - "mongodb": "3.6.3", + "mongodb": "4.9", "mssql": "6.2.3", "mysql2": "2.3.3", "node-fetch": "2.6.7", @@ -162,7 +162,6 @@ "@types/koa": "2.13.4", "@types/koa__router": "8.0.0", "@types/lodash": "4.14.180", - "@types/mongodb": "3.6.3", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/oracledb": "5.2.2", diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 9f1d41d2ec..0800c61cd8 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -6,13 +6,12 @@ import { } from "@budibase/types" import { MongoClient, - ObjectID, - FilterQuery, - UpdateQuery, - FindOneAndUpdateOption, - UpdateOneOptions, - UpdateManyOptions, - CommonOptions, + ObjectId, + Filter, + UpdateFilter, + FindOneAndUpdateOptions, + UpdateOptions, + OperationOptions, } from "mongodb" module MongoDBModule { @@ -100,7 +99,7 @@ module MongoDBModule { /(?<=objectid\(['"]).*(?=['"]\))/gi )?.[0] if (id) { - json[field] = ObjectID.createFromHexString(id) + json[field] = ObjectId.createFromHexString(id) } } } @@ -204,9 +203,9 @@ module MongoDBModule { json = this.parseQueryParams(query.json, "update") } let findAndUpdateJson = this.createObjectIds(json) as { - filter: FilterQuery - update: UpdateQuery - options: FindOneAndUpdateOption + filter: Filter + update: UpdateFilter + options: FindOneAndUpdateOptions } return await collection.findOneAndUpdate( findAndUpdateJson.filter, @@ -244,8 +243,8 @@ module MongoDBModule { queryJson = this.parseQueryParams(queryJson, "update") } let json = this.createObjectIds(queryJson) as { - filter: FilterQuery - update: UpdateQuery + filter: Filter + update: UpdateFilter options: object } @@ -254,14 +253,14 @@ module MongoDBModule { return await collection.updateOne( json.filter, json.update, - json.options as UpdateOneOptions + json.options as UpdateOptions ) } case "updateMany": { return await collection.updateMany( json.filter, json.update, - json.options as UpdateManyOptions + json.options as UpdateOptions ) } default: { @@ -288,8 +287,8 @@ module MongoDBModule { queryJson = this.parseQueryParams(queryJson, "delete") } let json = this.createObjectIds(queryJson) as { - filter: FilterQuery - options: CommonOptions + filter: Filter + options: OperationOptions } if (!json.options) { json = { diff --git a/packages/server/src/integrations/tests/mongo.spec.js b/packages/server/src/integrations/tests/mongo.spec.js index 40aa6dbb58..7b6626d9da 100644 --- a/packages/server/src/integrations/tests/mongo.spec.js +++ b/packages/server/src/integrations/tests/mongo.spec.js @@ -125,12 +125,12 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ - _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), - name: mongo.ObjectID.createFromHexString("BBBB12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), + name: mongo.ObjectId.createFromHexString("BBBB12345678ABCD12345678"), }) expect(args[1]).toEqual({ - _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"), - name: mongo.ObjectID.createFromHexString("CCCC12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), + name: mongo.ObjectId.createFromHexString("CCCC12345678ABCD12345678"), }) expect(args[2]).toEqual({ upsert: false @@ -162,12 +162,12 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ $set: { - _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), } }) expect(args[2]).toEqual({ @@ -201,7 +201,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.findOneAndUpdate.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ @@ -246,7 +246,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ @@ -299,7 +299,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 02df4a4d8e..78c2e8ae7a 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2632,7 +2632,7 @@ "@types/connect" "*" "@types/node" "*" -"@types/bson@*", "@types/bson@4.2.0": +"@types/bson@4.2.0": version "4.2.0" resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337" integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg== @@ -2848,14 +2848,6 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/mongodb@3.6.3": - version "3.6.3" - resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.3.tgz#5655af409d9e32d5d5ae9a653abf3e5f9c83eb7a" - integrity sha512-6YNqGP1hk5bjUFaim+QoFFuI61WjHiHE1BNeB41TA00Xd2K7zG4lcWyLLq/XtIp36uMavvS5hoAUJ+1u/GcX2Q== - dependencies: - "@types/bson" "*" - "@types/node" "*" - "@types/node-fetch@2.6.1": version "2.6.1" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" @@ -2990,6 +2982,19 @@ dependencies: "@types/node" "*" +"@types/webidl-conversions@*": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz#e33bc8ea812a01f63f90481c666334844b12a09e" + integrity sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q== + +"@types/whatwg-url@^8.2.1": + version "8.2.2" + resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63" + integrity sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA== + dependencies: + "@types/node" "*" + "@types/webidl-conversions" "*" + "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b" @@ -4056,14 +4061,6 @@ bl@^1.0.0: readable-stream "^2.3.5" safe-buffer "^5.1.1" -bl@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/bl/-/bl-2.2.1.tgz#8c11a7b730655c5d56898cdc871224f40fd901d5" - integrity sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g== - dependencies: - readable-stream "^2.3.5" - safe-buffer "^5.1.1" - bl@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.1.tgz#1cbb439299609e419b5a74d7fce2f8b37d8e5c6f" @@ -4206,10 +4203,12 @@ bson@*: dependencies: buffer "^5.6.0" -bson@^1.1.4: - version "1.1.6" - resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a" - integrity sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg== +bson@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-4.7.0.tgz#7874a60091ffc7a45c5dd2973b5cad7cded9718a" + integrity sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA== + dependencies: + buffer "^5.6.0" buffer-alloc-unsafe@^1.1.0: version "1.1.0" @@ -5194,7 +5193,7 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== -denque@^1.1.0, denque@^1.4.1: +denque@^1.1.0: version "1.5.1" resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== @@ -5204,6 +5203,11 @@ denque@^2.0.1: resolved "https://registry.yarnpkg.com/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a" integrity sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0, depd@^2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -7447,6 +7451,11 @@ ip-regex@^2.1.0: resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" integrity sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw== +ip@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" + integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== + is-accessor-descriptor@^0.1.6: version "0.1.6" resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" @@ -9977,18 +9986,25 @@ moment@^2.29.3: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== -mongodb@3.6.3: - version "3.6.3" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.6.3.tgz#eddaed0cc3598474d7a15f0f2a5b04848489fd05" - integrity sha512-rOZuR0QkodZiM+UbQE5kDsJykBqWi0CL4Ec2i1nrGrUI3KO11r6Fbxskqmq3JK2NH7aW4dcccBuUujAP0ERl5w== +mongodb-connection-string-url@^2.5.3: + version "2.5.3" + resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.3.tgz#c0c572b71570e58be2bd52b33dffd1330cfb6990" + integrity sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ== dependencies: - bl "^2.2.1" - bson "^1.1.4" - denque "^1.4.1" - require_optional "^1.0.1" - safe-buffer "^5.1.2" + "@types/whatwg-url" "^8.2.1" + whatwg-url "^11.0.0" + +mongodb@4.9: + version "4.9.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.9.0.tgz#58618439b721f2d6f7d38bb10a4612e29d7f1c8a" + integrity sha512-tJJEFJz7OQTQPZeVHZJIeSOjMRqc5eSyXTt86vSQENEErpkiG7279tM/GT5AVZ7TgXNh9HQxoa2ZkbrANz5GQw== + dependencies: + bson "^4.7.0" + denque "^2.1.0" + mongodb-connection-string-url "^2.5.3" + socks "^2.7.0" optionalDependencies: - saslprep "^1.0.0" + saslprep "^1.0.3" mri@1.1.4: version "1.1.4" @@ -11923,14 +11939,6 @@ require-main-filename@^2.0.0: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== -require_optional@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/require_optional/-/require_optional-1.0.1.tgz#4cf35a4247f64ca3df8c2ef208cc494b1ca8fc2e" - integrity sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g== - dependencies: - resolve-from "^2.0.0" - semver "^5.1.0" - requires-port@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" @@ -11943,11 +11951,6 @@ resolve-cwd@^3.0.0: dependencies: resolve-from "^5.0.0" -resolve-from@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-2.0.0.tgz#9480ab20e94ffa1d9e80a804c7ea147611966b57" - integrity sha512-qpFcKaXsq8+oRoLilkwyc7zHGF5i9Q2/25NIgLQQ/+VVv9rU4qvr6nXVAw1DsnXJyQkZsR4Ytfbtg5ehfcUssQ== - resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" @@ -12120,7 +12123,7 @@ sanitize-s3-objectkey@0.0.1: resolved "https://registry.yarnpkg.com/sanitize-s3-objectkey/-/sanitize-s3-objectkey-0.0.1.tgz#efa9887cd45275b40234fb4bb12fc5754fe64e7e" integrity sha512-ZTk7aqLxy4sD40GWcYWoLfbe05XLmkKvh6vGKe13ADlei24xlezcvjgKy1qRArlaIbIMYaqK7PCalvZtulZlaQ== -saslprep@^1.0.0: +saslprep@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== @@ -12182,7 +12185,7 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -12360,6 +12363,11 @@ slice-ansi@^2.1.0: astral-regex "^1.0.0" is-fullwidth-code-point "^2.0.0" +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -12435,6 +12443,14 @@ snowflake-sdk@^1.6.0: uuid "^3.3.2" winston "^3.1.0" +socks@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.0.tgz#f9225acdb841e874dca25f870e9130990f3913d0" + integrity sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA== + dependencies: + ip "^2.0.0" + smart-buffer "^4.2.0" + sonic-boom@^1.0.2: version "1.4.1" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-1.4.1.tgz#d35d6a74076624f12e6f917ade7b9d75e918f53e" @@ -13408,6 +13424,13 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -13949,6 +13972,11 @@ webidl-conversions@^6.1.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== + webpack-cli@^4.9.1: version "4.9.2" resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.9.2.tgz#77c1adaea020c3f9e2db8aad8ea78d235c83659d" @@ -14030,6 +14058,14 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" From 018f4db2e43ffefcb70329b4b79e5df7f638012d Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 25 Aug 2022 17:30:47 +0100 Subject: [PATCH 02/21] Add mongodb aggregate method --- .../src/components/integration/index.svelte | 130 +++++++++++++++++- packages/server/src/integrations/mongodb.ts | 25 ++++ packages/types/src/sdk/datasources.ts | 1 + 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index a0e1b8c1c4..f87883d991 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -1,12 +1,21 @@ {#if schema} @@ -93,8 +101,9 @@ Stage {index + 1} { + updateEditors(index) query.fields.steps.splice(index, 1) - query.fields.steps = query.fields.steps + query.fields.steps = [...query.fields.steps] }} icon="DeleteOutline" /> @@ -108,18 +117,17 @@ query.fields.steps[index].key = detail }} /> - {#key query.fields.steps.length} - { - query.fields.steps[index].value = detail - }} - /> - {/key} + { + query.fields.steps[index].value = detail + }} + /> From dbf58fcfe9cd8440d340504fe80898a8cb1769ad Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 4 Oct 2022 11:03:24 +0100 Subject: [PATCH 09/21] Added support for JSON array stages --- .../src/components/integration/index.svelte | 7 ++- packages/server/src/integrations/mongodb.ts | 50 +++++++++++-------- packages/types/src/sdk/datasources.ts | 1 - 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index a83b9d4db0..c98746ae6e 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -41,6 +41,9 @@ flowEditors[i].update(query.fields.steps[i + 1].value.value) } } + + $: shouldDisplayJsonBox = + schema.type === QueryTypes.JSON && query.fields.extra?.actionType !== "flow" {#if schema} @@ -55,7 +58,7 @@ value={query.fields.sql} parameters={query.parameters} /> - {:else if schema.type === QueryTypes.JSON} + {:else if shouldDisplayJsonBox} {/if} - {:else if schema.type === QueryTypes.FLOW} + {:else if query.fields.extra?.actionType === "flow"}
{#if (query.fields.steps?.length ?? 0) === 0}
diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index b5fed323a5..aae345c981 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -57,7 +57,7 @@ const SCHEMA: Integration = { type: QueryType.JSON, }, aggregate: { - type: QueryType.FLOW, + type: QueryType.JSON, }, }, extra: { @@ -66,8 +66,8 @@ const SCHEMA: Integration = { type: DatasourceFieldType.STRING, required: true, }, - actionTypes: { - displayName: "Action Types", + actionType: { + displayName: "Action Type", type: DatasourceFieldType.LIST, required: true, data: { @@ -75,6 +75,7 @@ const SCHEMA: Integration = { create: ["insertOne", "insertMany"], update: ["updateOne", "updateMany"], delete: ["deleteOne", "deleteMany"], + aggregate: ["json", "flow"], }, }, }, @@ -170,7 +171,7 @@ class MongoIntegration implements IntegrationBase { // For mongodb we add an extra actionType to specify // which method we want to call on the collection - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "insertOne": { return await collection.insertOne(json) } @@ -179,7 +180,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for create` + `actionType ${query.extra.actionType} does not exist on DB for create` ) } } @@ -198,7 +199,7 @@ class MongoIntegration implements IntegrationBase { const collection = db.collection(query.extra.collection) let json = this.createObjectIds(query.json) - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "find": { return await collection.find(json).toArray() } @@ -228,7 +229,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for read` + `actionType ${query.extra.actionType} does not exist on DB for read` ) } } @@ -255,7 +256,7 @@ class MongoIntegration implements IntegrationBase { options: object } - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "updateOne": { return await collection.updateOne( json.filter, @@ -272,7 +273,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for update` + `actionType ${query.extra.actionType} does not exist on DB for update` ) } } @@ -304,7 +305,7 @@ class MongoIntegration implements IntegrationBase { } } - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "deleteOne": { return await collection.deleteOne(json.filter, json.options) } @@ -313,7 +314,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for delete` + `actionType ${query.extra.actionType} does not exist on DB for delete` ) } } @@ -325,20 +326,29 @@ class MongoIntegration implements IntegrationBase { } } - async aggregate(query: { steps: any[]; extra: { [key: string]: string } }) { + async aggregate(query: { json: object; steps: any[]; extra: { [key: string]: string } }) { try { await this.connect() const db = this.client.db(this.config.db) const collection = db.collection(query.extra.collection) let response = [] - for await (const doc of collection.aggregate( - query.steps.map(({ key, value }) => { - let temp: any = {} - temp[key] = JSON.parse(value.value) - return temp - }) - )) { - response.push(doc) + if (query.extra?.actionType === "flow") { + for await (const doc of collection.aggregate( + query.steps.map(({ key, value }) => { + let temp: any = {} + temp[key] = JSON.parse(value.value) + return temp + }) + )) { + response.push(doc) + } + } else { + const stages: Array = query.json as Array + for await (const doc of collection.aggregate( + stages ? stages : [] + )) { + response.push(doc) + } } return response } catch (err) { diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 1dbdc6e423..d01d636b86 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -20,7 +20,6 @@ export enum QueryType { SQL = "sql", JSON = "json", FIELDS = "fields", - FLOW = "flow", } export enum DatasourceFieldType { From 9a5b3b27aaade48afef330d8a84f5559215caae2 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 4 Oct 2022 11:03:54 +0100 Subject: [PATCH 10/21] lint --- packages/server/src/integrations/mongodb.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index aae345c981..85aa341935 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -326,7 +326,11 @@ class MongoIntegration implements IntegrationBase { } } - async aggregate(query: { json: object; steps: any[]; extra: { [key: string]: string } }) { + async aggregate(query: { + json: object + steps: any[] + extra: { [key: string]: string } + }) { try { await this.connect() const db = this.client.db(this.config.db) @@ -344,9 +348,7 @@ class MongoIntegration implements IntegrationBase { } } else { const stages: Array = query.json as Array - for await (const doc of collection.aggregate( - stages ? stages : [] - )) { + for await (const doc of collection.aggregate(stages ? stages : [])) { response.push(doc) } } From 12f0ebba7a4f8a920126e966c513faac351d17d3 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 4 Oct 2022 11:29:13 +0100 Subject: [PATCH 11/21] Rename actionTypes to actionType --- .../src/integrations/tests/mongo.spec.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/server/src/integrations/tests/mongo.spec.ts b/packages/server/src/integrations/tests/mongo.spec.ts index b06709d8e2..b941d7ae66 100644 --- a/packages/server/src/integrations/tests/mongo.spec.ts +++ b/packages/server/src/integrations/tests/mongo.spec.ts @@ -34,7 +34,7 @@ describe("MongoDB Integration", () => { await config.integration.create({ index: indexName, json: body, - extra: { collection: "testCollection", actionTypes: "insertOne" }, + extra: { collection: "testCollection", actionType: "insertOne" }, }) expect(config.integration.client.insertOne).toHaveBeenCalledWith(body) }) @@ -44,7 +44,7 @@ describe("MongoDB Integration", () => { json: { address: "test", }, - extra: { collection: "testCollection", actionTypes: "find" }, + extra: { collection: "testCollection", actionType: "find" }, } const response = await config.integration.read(query) expect(config.integration.client.find).toHaveBeenCalledWith(query.json) @@ -61,7 +61,7 @@ describe("MongoDB Integration", () => { opt: "option", }, }, - extra: { collection: "testCollection", actionTypes: "deleteOne" }, + extra: { collection: "testCollection", actionType: "deleteOne" }, } await config.integration.delete(query) expect(config.integration.client.deleteOne).toHaveBeenCalledWith( @@ -83,7 +83,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalledWith( @@ -97,7 +97,7 @@ describe("MongoDB Integration", () => { const restore = disableConsole() const query = { - extra: { collection: "testCollection", actionTypes: "deleteOne" }, + extra: { collection: "testCollection", actionType: "deleteOne" }, } let error = null @@ -125,7 +125,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -161,7 +161,7 @@ describe("MongoDB Integration", () => { upsert: true, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -200,7 +200,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" }, + extra: { collection: "testCollection", actionType: "findOneAndUpdate" }, } await config.integration.read(query) expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled() @@ -245,7 +245,7 @@ describe("MongoDB Integration", () => { { "upsert": true }`, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -300,7 +300,7 @@ describe("MongoDB Integration", () => { "upsert": true, "extra": "ad\\"{\\"d" }`, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() From 9ee47881477b35e88b929199d873ca217a0bec28 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 15:31:53 +0100 Subject: [PATCH 12/21] Include stage templates --- .../src/components/integration/index.svelte | 19 +++++++++++++++++-- packages/server/src/integrations/mongodb.ts | 19 +++++++++++++++++++ packages/types/src/sdk/datasources.ts | 6 ++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index c98746ae6e..d07f395a01 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -4,6 +4,7 @@ import { Label, Input, + Select, Divider, Layout, Icon, @@ -42,6 +43,19 @@ } } + function setEditorTemplate(fromKey, toKey, index) { + if ( + schema.steps.filter(step => step.key === fromKey)[0]?.template === + query.fields.steps[index].value.value + ) { + query.fields.steps[index].value.value = schema.steps.filter( + step => step.key === toKey + )[0]?.template + flowEditors[index].update(query.fields.steps[index].value.value) + } + query.fields.steps[index].key = toKey + } + $: shouldDisplayJsonBox = schema.type === QueryTypes.JSON && query.fields.extra?.actionType !== "flow" @@ -114,10 +128,11 @@
- s.key)} on:change={({ detail }) => { - query.fields.steps[index].key = detail + setEditorTemplate(step.key, detail, index) }} /> } export interface ExtraQueryConfig { From 8fd592d6d092c3f7dd480ceb032691cfd509b109 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 16:03:51 +0100 Subject: [PATCH 13/21] Add all the stage templates --- packages/server/src/integrations/mongodb.ts | 201 +++++++++++++++++++- 1 file changed, 200 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 1794615541..1cb24921b7 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -71,12 +71,211 @@ const SCHEMA: Integration = { "default": "", "output": {} }`, + }, + { + key: "$bucketAuto", + template: `{ + "groupBy": "", + "buckets": 1, + "output": {}, + "granularity": "R5" +}`, + }, + { + key: "$changeStream", + template: `{ + "allChangesForCluster": true, + "fullDocument": "", + "fullDocumentBeforeChange": "", + "resumeAfter": 1, + "showExpandedEvents": true, + "startAfter": {}, + "startAtOperationTime": "" +}`, + }, + { + key: "$collStats", + template: `{ + "latencyStats": { "histograms": true } }, + "storageStats": { "scale": 1 } }, + "count": {}, + "queryExecStats": {} +}`, + }, + { + key: "$count", + template: ``, + }, + { + key: "$densify", + template: `{ + "field": "", + "partitionByFields": [], + "range": { + "step": 1, + "unit": 1, + "bounds": "full" + } +}`, + }, + { + key: "$documents", + template: `[]`, + }, + { + key: "$facet", + template: `{\n\t\n}`, + }, + { + key: "$fill", + template: `{ + "partitionBy": "", + "partitionByFields": [], + "sortBy": {}, + "output": {} +}`, + }, + { + key: "$geoNear", + template: `{ + "near": { "type": "Point", "coordinates": [ -73.98142 , 40.71782 ] }, + "key": "location", + "distanceField": "dist.calculated", + "query": { "category": "Parks" } +}` + }, + { + key: "$graphLookup", + template: `{ + "from": "", + "startWith": "", + "connectFromField": "", + "connectToField": "", + "as": "", + "maxDepth": 1, + "depthField": "", + "restrictSearchWithMatch": {} +}` + }, + { + key: "$group", + template: `{ + "_id": "" +}`, + }, + { + key: "$indexStats", + template: "{\n\t\n}", + }, + { + key: "$limit", + template: `1`, + }, + { + key: "$listLocalSessions", + template: `{\n\t\n}`, + }, + { + key: "$listSessions", + template: `{\n\t\n}`, + }, + { + key: "$lookup", + template: `{ + "from": "", + "localField": "", + "foreignField": "", + "as": "" +}` }, { key: "$match", template: "{\n\t\n}", }, - ] + { + key: "$merge", + template: `{ + "into": {}, + "on": "_id", + "whenMatched": "replace", + "whenNotMatched": "insert" +}` + }, + { + key: "$out", + template: `{ + "db": "", + "coll": "" +}` + }, + { + key: "$planCacheStats", + template: "{\n\t\n}", + }, + { + key: "$project", + template: "{\n\t\n}", + }, + { + key: "$redact", + template: "", + }, + { + key: "$replaceRoot", + template: `{ "newRoot": "" }`, + }, + { + key: "$replaceWith", + template: ``, + }, + { + key: "$sample", + template: `{ "size": 3 }`, + }, + { + key: "$set", + template: "{\n\t\n}", + }, + { + key: "$setWindowFields", + template: `{ + "partitionBy": "", + "sortBy": {}, + "output": {} +}`, + }, + { + key: "$skip", + template: `1`, + }, + { + key: "$sort", + template: "{\n\t\n}", + }, + { + key: "$sortByCount", + template: "", + }, + { + key: "$unionWith", + template: `{ + "coll": "", + "pipeline": [] +}` + }, + { + key: "$unset", + template: "", + }, + { + key: "$unwind", + template: `{ + "path": "", + "includeArrayIndex": "", + "preserveNullAndEmptyArrays": true +}` + }, + ], }, }, extra: { From 910991eb8c4fc62faac70976166b4c44b300b1ed Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 16:20:11 +0100 Subject: [PATCH 14/21] Allow template to be used when changed body blank --- .../src/components/integration/index.svelte | 5 ++++- packages/server/src/integrations/mongodb.ts | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index d07f395a01..e3d9eab414 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -44,9 +44,12 @@ } function setEditorTemplate(fromKey, toKey, index) { + const currentValue = query.fields.steps[index].value.value if ( + !currentValue || + currentValue.toString().replace("\\s", "").length < 3 || schema.steps.filter(step => step.key === fromKey)[0]?.template === - query.fields.steps[index].value.value + currentValue ) { query.fields.steps[index].value.value = schema.steps.filter( step => step.key === toKey diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 1cb24921b7..eeff616cbd 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -138,11 +138,16 @@ const SCHEMA: Integration = { { key: "$geoNear", template: `{ - "near": { "type": "Point", "coordinates": [ -73.98142 , 40.71782 ] }, + "near": { + "type": "Point", + "coordinates": [ + -73.98142, 40.71782 + ] + }, "key": "location", "distanceField": "dist.calculated", "query": { "category": "Parks" } -}` +}`, }, { key: "$graphLookup", @@ -155,7 +160,7 @@ const SCHEMA: Integration = { "maxDepth": 1, "depthField": "", "restrictSearchWithMatch": {} -}` +}`, }, { key: "$group", @@ -186,7 +191,7 @@ const SCHEMA: Integration = { "localField": "", "foreignField": "", "as": "" -}` +}`, }, { key: "$match", @@ -199,14 +204,14 @@ const SCHEMA: Integration = { "on": "_id", "whenMatched": "replace", "whenNotMatched": "insert" -}` +}`, }, { key: "$out", template: `{ "db": "", "coll": "" -}` +}`, }, { key: "$planCacheStats", @@ -261,7 +266,7 @@ const SCHEMA: Integration = { template: `{ "coll": "", "pipeline": [] -}` +}`, }, { key: "$unset", @@ -273,7 +278,7 @@ const SCHEMA: Integration = { "path": "", "includeArrayIndex": "", "preserveNullAndEmptyArrays": true -}` +}`, }, ], }, From 48d84d67bfeb5c6134646bac55cdf4ff1ce3a5e4 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 18:42:35 +0100 Subject: [PATCH 15/21] Allow stages to be swapped in flow --- .../src/components/integration/index.svelte | 58 ++++++++++++++++--- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index e3d9eab414..114cec98e2 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -37,11 +37,16 @@ query.fields[schema.type] = detail.value } - function updateEditors(deleteIndex) { + function updateEditorsOnDelete(deleteIndex) { for (let i = deleteIndex; i < query.fields.steps?.length - 1; i++) { flowEditors[i].update(query.fields.steps[i + 1].value.value) } } + function updateEditorsOnSwap(actionIndex, targetIndex) { + const target = query.fields.steps[targetIndex].value.value + flowEditors[targetIndex].update(query.fields.steps[actionIndex].value.value) + flowEditors[actionIndex].update(target) + } function setEditorTemplate(fromKey, toKey, index) { const currentValue = query.fields.steps[index].value.value @@ -119,14 +124,44 @@
Stage {index + 1} - { - updateEditors(index) - query.fields.steps.splice(index, 1) - query.fields.steps = [...query.fields.steps] - }} - icon="DeleteOutline" - /> +
+
+ {#if index > 0} + { + updateEditorsOnSwap(index, index - 1) + const target = query.fields.steps[index - 1].key + query.fields.steps[index - 1].key = + query.fields.steps[index].key + query.fields.steps[index].key = target + }} + icon="ChevronUp" + /> + {/if} + {#if index < query.fields.steps.length - 1} + { + updateEditorsOnSwap(index, index + 1) + const target = query.fields.steps[index + 1].key + query.fields.steps[index + 1].key = + query.fields.steps[index].key + query.fields.steps[index].key = target + }} + icon="ChevronDown" + /> + {/if} +
+ { + updateEditorsOnDelete(index) + query.fields.steps.splice(index, 1) + query.fields.steps = [...query.fields.steps] + }} + icon="DeleteOutline" + /> +
@@ -209,6 +244,11 @@ align-items: center; padding-bottom: 24px; } + .block-actions { + justify-content: space-between; + display: flex; + align-items: right; + } .fields { display: flex; From 98871064a973c0f4b8f9d0c569791f185c52fcdc Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 18:54:53 +0100 Subject: [PATCH 16/21] Add support for ObjectId in aggregations --- packages/server/src/integrations/mongodb.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index eeff616cbd..70419be58b 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -564,14 +564,14 @@ class MongoIntegration implements IntegrationBase { query.steps.map(({ key, value }) => { let temp: any = {} temp[key] = JSON.parse(value.value) - return temp + return this.createObjectIds(temp) }) )) { response.push(doc) } } else { const stages: Array = query.json as Array - for await (const doc of collection.aggregate(stages ? stages : [])) { + for await (const doc of collection.aggregate(stages ? this.createObjectIds(stages) : [])) { response.push(doc) } } From cd5930450ef3db660c5eb919e695e6c7c83dd701 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 18:55:22 +0100 Subject: [PATCH 17/21] lint --- packages/server/src/integrations/mongodb.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 70419be58b..f602b78093 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -571,7 +571,9 @@ class MongoIntegration implements IntegrationBase { } } else { const stages: Array = query.json as Array - for await (const doc of collection.aggregate(stages ? this.createObjectIds(stages) : [])) { + for await (const doc of collection.aggregate( + stages ? this.createObjectIds(stages) : [] + )) { response.push(doc) } } From 34a1d5e36b736854fadebb08f15c82f3c9bc62d3 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 13 Oct 2022 13:05:45 +0100 Subject: [PATCH 18/21] Rename flow to pipeline --- packages/builder/src/components/integration/index.svelte | 6 +++--- packages/server/src/integrations/mongodb.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index 114cec98e2..2faa3c0ce5 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -16,7 +16,6 @@ SQL: "sql", JSON: "json", FIELDS: "fields", - FLOW: "flow", } export let query @@ -65,7 +64,8 @@ } $: shouldDisplayJsonBox = - schema.type === QueryTypes.JSON && query.fields.extra?.actionType !== "flow" + schema.type === QueryTypes.JSON && + query.fields.extra?.actionType !== "pipeline" {#if schema} @@ -98,7 +98,7 @@
{/if} - {:else if query.fields.extra?.actionType === "flow"} + {:else if query.fields.extra?.actionType === "pipeline"}
{#if (query.fields.steps?.length ?? 0) === 0}
diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index f602b78093..d7709a08c7 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -290,7 +290,7 @@ const SCHEMA: Integration = { required: true, }, actionType: { - displayName: "Action Type", + displayName: "Query Type", type: DatasourceFieldType.LIST, required: true, data: { @@ -298,7 +298,7 @@ const SCHEMA: Integration = { create: ["insertOne", "insertMany"], update: ["updateOne", "updateMany"], delete: ["deleteOne", "deleteMany"], - aggregate: ["json", "flow"], + aggregate: ["json", "pipeline"], }, }, }, @@ -559,7 +559,7 @@ class MongoIntegration implements IntegrationBase { const db = this.client.db(this.config.db) const collection = db.collection(query.extra.collection) let response = [] - if (query.extra?.actionType === "flow") { + if (query.extra?.actionType === "pipeline") { for await (const doc of collection.aggregate( query.steps.map(({ key, value }) => { let temp: any = {} From 881d5d53b0885215a720321292fdfcd233e6faad Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 13 Oct 2022 13:07:53 +0100 Subject: [PATCH 19/21] Rename flowEditors to stepEditors --- .../builder/src/components/integration/index.svelte | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index 2faa3c0ce5..57050a27db 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -24,7 +24,7 @@ export let editable = true export let height = 500 - let flowEditors = [] + let stepEditors = [] $: urlDisplay = schema.urlDisplay && @@ -38,13 +38,13 @@ function updateEditorsOnDelete(deleteIndex) { for (let i = deleteIndex; i < query.fields.steps?.length - 1; i++) { - flowEditors[i].update(query.fields.steps[i + 1].value.value) + stepEditors[i].update(query.fields.steps[i + 1].value.value) } } function updateEditorsOnSwap(actionIndex, targetIndex) { const target = query.fields.steps[targetIndex].value.value - flowEditors[targetIndex].update(query.fields.steps[actionIndex].value.value) - flowEditors[actionIndex].update(target) + stepEditors[targetIndex].update(query.fields.steps[actionIndex].value.value) + stepEditors[actionIndex].update(target) } function setEditorTemplate(fromKey, toKey, index) { @@ -58,7 +58,7 @@ query.fields.steps[index].value.value = schema.steps.filter( step => step.key === toKey )[0]?.template - flowEditors[index].update(query.fields.steps[index].value.value) + stepEditors[index].update(query.fields.steps[index].value.value) } query.fields.steps[index].key = toKey } @@ -174,7 +174,7 @@ }} /> Date: Thu, 13 Oct 2022 16:13:56 +0100 Subject: [PATCH 20/21] Update packages/builder/src/components/integration/index.svelte Co-authored-by: Martin McKeaveney --- packages/builder/src/components/integration/index.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index 57050a27db..50c350e9c6 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -100,7 +100,7 @@ {/if} {:else if query.fields.extra?.actionType === "pipeline"}
- {#if (query.fields.steps?.length ?? 0) === 0} + {#if query.fields.steps?.length === 0}