From 8738beb907841ecd90663054faa0ac21f8fa001d Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 25 Aug 2022 11:02:46 +0100 Subject: [PATCH 001/101] Upgrade to mongodb@4.9 --- package.json | 1 - packages/server/__mocks__/mongodb.ts | 2 +- packages/server/package.json | 3 +- packages/server/src/integrations/mongodb.ts | 33 +++-- .../src/integrations/tests/mongo.spec.js | 18 +-- packages/server/yarn.lock | 130 +++++++++++------- 6 files changed, 110 insertions(+), 77 deletions(-) diff --git a/package.json b/package.json index 4c24e0025b..d5cc637fc2 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "private": true, "devDependencies": { "@rollup/plugin-json": "^4.0.2", - "@types/mongodb": "3.6.3", "@typescript-eslint/parser": "4.28.0", "babel-eslint": "^10.0.3", "eslint": "^7.28.0", diff --git a/packages/server/__mocks__/mongodb.ts b/packages/server/__mocks__/mongodb.ts index 92ec89227f..1655cfa96e 100644 --- a/packages/server/__mocks__/mongodb.ts +++ b/packages/server/__mocks__/mongodb.ts @@ -33,7 +33,7 @@ module MongoMock { }) } - mongodb.ObjectID = require("mongodb").ObjectID + mongodb.ObjectId = require("mongodb").ObjectId module.exports = mongodb } diff --git a/packages/server/package.json b/packages/server/package.json index 75c8f81d99..3966f2436d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -121,7 +121,7 @@ "koa2-ratelimit": "1.1.1", "lodash": "4.17.21", "memorystream": "0.3.1", - "mongodb": "3.6.3", + "mongodb": "4.9", "mssql": "6.2.3", "mysql2": "2.3.3", "node-fetch": "2.6.7", @@ -162,7 +162,6 @@ "@types/koa": "2.13.4", "@types/koa__router": "8.0.0", "@types/lodash": "4.14.180", - "@types/mongodb": "3.6.3", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/oracledb": "5.2.2", diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 9f1d41d2ec..0800c61cd8 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -6,13 +6,12 @@ import { } from "@budibase/types" import { MongoClient, - ObjectID, - FilterQuery, - UpdateQuery, - FindOneAndUpdateOption, - UpdateOneOptions, - UpdateManyOptions, - CommonOptions, + ObjectId, + Filter, + UpdateFilter, + FindOneAndUpdateOptions, + UpdateOptions, + OperationOptions, } from "mongodb" module MongoDBModule { @@ -100,7 +99,7 @@ module MongoDBModule { /(?<=objectid\(['"]).*(?=['"]\))/gi )?.[0] if (id) { - json[field] = ObjectID.createFromHexString(id) + json[field] = ObjectId.createFromHexString(id) } } } @@ -204,9 +203,9 @@ module MongoDBModule { json = this.parseQueryParams(query.json, "update") } let findAndUpdateJson = this.createObjectIds(json) as { - filter: FilterQuery - update: UpdateQuery - options: FindOneAndUpdateOption + filter: Filter + update: UpdateFilter + options: FindOneAndUpdateOptions } return await collection.findOneAndUpdate( findAndUpdateJson.filter, @@ -244,8 +243,8 @@ module MongoDBModule { queryJson = this.parseQueryParams(queryJson, "update") } let json = this.createObjectIds(queryJson) as { - filter: FilterQuery - update: UpdateQuery + filter: Filter + update: UpdateFilter options: object } @@ -254,14 +253,14 @@ module MongoDBModule { return await collection.updateOne( json.filter, json.update, - json.options as UpdateOneOptions + json.options as UpdateOptions ) } case "updateMany": { return await collection.updateMany( json.filter, json.update, - json.options as UpdateManyOptions + json.options as UpdateOptions ) } default: { @@ -288,8 +287,8 @@ module MongoDBModule { queryJson = this.parseQueryParams(queryJson, "delete") } let json = this.createObjectIds(queryJson) as { - filter: FilterQuery - options: CommonOptions + filter: Filter + options: OperationOptions } if (!json.options) { json = { diff --git a/packages/server/src/integrations/tests/mongo.spec.js b/packages/server/src/integrations/tests/mongo.spec.js index 40aa6dbb58..7b6626d9da 100644 --- a/packages/server/src/integrations/tests/mongo.spec.js +++ b/packages/server/src/integrations/tests/mongo.spec.js @@ -125,12 +125,12 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ - _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), - name: mongo.ObjectID.createFromHexString("BBBB12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), + name: mongo.ObjectId.createFromHexString("BBBB12345678ABCD12345678"), }) expect(args[1]).toEqual({ - _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"), - name: mongo.ObjectID.createFromHexString("CCCC12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), + name: mongo.ObjectId.createFromHexString("CCCC12345678ABCD12345678"), }) expect(args[2]).toEqual({ upsert: false @@ -162,12 +162,12 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ $set: { - _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), } }) expect(args[2]).toEqual({ @@ -201,7 +201,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.findOneAndUpdate.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ @@ -246,7 +246,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ @@ -299,7 +299,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), } }) expect(args[1]).toEqual({ diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 02df4a4d8e..78c2e8ae7a 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2632,7 +2632,7 @@ "@types/connect" "*" "@types/node" "*" -"@types/bson@*", "@types/bson@4.2.0": +"@types/bson@4.2.0": version "4.2.0" resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337" integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg== @@ -2848,14 +2848,6 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/mongodb@3.6.3": - version "3.6.3" - resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.3.tgz#5655af409d9e32d5d5ae9a653abf3e5f9c83eb7a" - integrity sha512-6YNqGP1hk5bjUFaim+QoFFuI61WjHiHE1BNeB41TA00Xd2K7zG4lcWyLLq/XtIp36uMavvS5hoAUJ+1u/GcX2Q== - dependencies: - "@types/bson" "*" - "@types/node" "*" - "@types/node-fetch@2.6.1": version "2.6.1" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" @@ -2990,6 +2982,19 @@ dependencies: "@types/node" "*" +"@types/webidl-conversions@*": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz#e33bc8ea812a01f63f90481c666334844b12a09e" + integrity sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q== + +"@types/whatwg-url@^8.2.1": + version "8.2.2" + resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63" + integrity sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA== + dependencies: + "@types/node" "*" + "@types/webidl-conversions" "*" + "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b" @@ -4056,14 +4061,6 @@ bl@^1.0.0: readable-stream "^2.3.5" safe-buffer "^5.1.1" -bl@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/bl/-/bl-2.2.1.tgz#8c11a7b730655c5d56898cdc871224f40fd901d5" - integrity sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g== - dependencies: - readable-stream "^2.3.5" - safe-buffer "^5.1.1" - bl@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.1.tgz#1cbb439299609e419b5a74d7fce2f8b37d8e5c6f" @@ -4206,10 +4203,12 @@ bson@*: dependencies: buffer "^5.6.0" -bson@^1.1.4: - version "1.1.6" - resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a" - integrity sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg== +bson@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-4.7.0.tgz#7874a60091ffc7a45c5dd2973b5cad7cded9718a" + integrity sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA== + dependencies: + buffer "^5.6.0" buffer-alloc-unsafe@^1.1.0: version "1.1.0" @@ -5194,7 +5193,7 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== -denque@^1.1.0, denque@^1.4.1: +denque@^1.1.0: version "1.5.1" resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== @@ -5204,6 +5203,11 @@ denque@^2.0.1: resolved "https://registry.yarnpkg.com/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a" integrity sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0, depd@^2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -7447,6 +7451,11 @@ ip-regex@^2.1.0: resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" integrity sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw== +ip@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" + integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== + is-accessor-descriptor@^0.1.6: version "0.1.6" resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" @@ -9977,18 +9986,25 @@ moment@^2.29.3: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== -mongodb@3.6.3: - version "3.6.3" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.6.3.tgz#eddaed0cc3598474d7a15f0f2a5b04848489fd05" - integrity sha512-rOZuR0QkodZiM+UbQE5kDsJykBqWi0CL4Ec2i1nrGrUI3KO11r6Fbxskqmq3JK2NH7aW4dcccBuUujAP0ERl5w== +mongodb-connection-string-url@^2.5.3: + version "2.5.3" + resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.3.tgz#c0c572b71570e58be2bd52b33dffd1330cfb6990" + integrity sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ== dependencies: - bl "^2.2.1" - bson "^1.1.4" - denque "^1.4.1" - require_optional "^1.0.1" - safe-buffer "^5.1.2" + "@types/whatwg-url" "^8.2.1" + whatwg-url "^11.0.0" + +mongodb@4.9: + version "4.9.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.9.0.tgz#58618439b721f2d6f7d38bb10a4612e29d7f1c8a" + integrity sha512-tJJEFJz7OQTQPZeVHZJIeSOjMRqc5eSyXTt86vSQENEErpkiG7279tM/GT5AVZ7TgXNh9HQxoa2ZkbrANz5GQw== + dependencies: + bson "^4.7.0" + denque "^2.1.0" + mongodb-connection-string-url "^2.5.3" + socks "^2.7.0" optionalDependencies: - saslprep "^1.0.0" + saslprep "^1.0.3" mri@1.1.4: version "1.1.4" @@ -11923,14 +11939,6 @@ require-main-filename@^2.0.0: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== -require_optional@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/require_optional/-/require_optional-1.0.1.tgz#4cf35a4247f64ca3df8c2ef208cc494b1ca8fc2e" - integrity sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g== - dependencies: - resolve-from "^2.0.0" - semver "^5.1.0" - requires-port@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" @@ -11943,11 +11951,6 @@ resolve-cwd@^3.0.0: dependencies: resolve-from "^5.0.0" -resolve-from@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-2.0.0.tgz#9480ab20e94ffa1d9e80a804c7ea147611966b57" - integrity sha512-qpFcKaXsq8+oRoLilkwyc7zHGF5i9Q2/25NIgLQQ/+VVv9rU4qvr6nXVAw1DsnXJyQkZsR4Ytfbtg5ehfcUssQ== - resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" @@ -12120,7 +12123,7 @@ sanitize-s3-objectkey@0.0.1: resolved "https://registry.yarnpkg.com/sanitize-s3-objectkey/-/sanitize-s3-objectkey-0.0.1.tgz#efa9887cd45275b40234fb4bb12fc5754fe64e7e" integrity sha512-ZTk7aqLxy4sD40GWcYWoLfbe05XLmkKvh6vGKe13ADlei24xlezcvjgKy1qRArlaIbIMYaqK7PCalvZtulZlaQ== -saslprep@^1.0.0: +saslprep@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== @@ -12182,7 +12185,7 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -12360,6 +12363,11 @@ slice-ansi@^2.1.0: astral-regex "^1.0.0" is-fullwidth-code-point "^2.0.0" +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -12435,6 +12443,14 @@ snowflake-sdk@^1.6.0: uuid "^3.3.2" winston "^3.1.0" +socks@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.0.tgz#f9225acdb841e874dca25f870e9130990f3913d0" + integrity sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA== + dependencies: + ip "^2.0.0" + smart-buffer "^4.2.0" + sonic-boom@^1.0.2: version "1.4.1" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-1.4.1.tgz#d35d6a74076624f12e6f917ade7b9d75e918f53e" @@ -13408,6 +13424,13 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -13949,6 +13972,11 @@ webidl-conversions@^6.1.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== + webpack-cli@^4.9.1: version "4.9.2" resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.9.2.tgz#77c1adaea020c3f9e2db8aad8ea78d235c83659d" @@ -14030,6 +14058,14 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" From 52c8fda064bb3e53ea9ea32fa50e3a1db5af2c40 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 25 Aug 2022 17:30:47 +0100 Subject: [PATCH 002/101] Add mongodb aggregate method --- .../src/components/integration/index.svelte | 130 +++++++++++++++++- packages/server/src/integrations/mongodb.ts | 25 ++++ packages/types/src/sdk/datasources.ts | 1 + 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index a0e1b8c1c4..f87883d991 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -1,12 +1,21 @@ {#if schema} @@ -93,8 +101,9 @@ Stage {index + 1} { + updateEditors(index) query.fields.steps.splice(index, 1) - query.fields.steps = query.fields.steps + query.fields.steps = [...query.fields.steps] }} icon="DeleteOutline" /> @@ -108,18 +117,17 @@ query.fields.steps[index].key = detail }} /> - {#key query.fields.steps.length} - { - query.fields.steps[index].value = detail - }} - /> - {/key} + { + query.fields.steps[index].value = detail + }} + /> From 51f2a3b86c44dc4ad7ac85c68b0c82838203ad60 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 4 Oct 2022 10:31:40 +0100 Subject: [PATCH 013/101] Types for sending quota triggered email --- packages/types/src/api/account/index.ts | 1 + packages/types/src/api/account/license.ts | 11 +++++++++++ packages/types/src/documents/account/account.ts | 2 ++ .../TestConfiguration/InternalAPIClient.ts | 6 ++---- .../internal-api/TestConfiguration/applications.ts | 8 ++------ .../config/internal-api/TestConfiguration/auth.ts | 6 +++--- .../config/internal-api/fixtures/applications.ts | 9 ++++----- .../tests/internal-api/applications/create.spec.ts | 13 +++++++------ 8 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 packages/types/src/api/account/license.ts diff --git a/packages/types/src/api/account/index.ts b/packages/types/src/api/account/index.ts index 50c6bf22c6..0cbc487bcc 100644 --- a/packages/types/src/api/account/index.ts +++ b/packages/types/src/api/account/index.ts @@ -1 +1,2 @@ export * from "./user" +export * from "./license" diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts new file mode 100644 index 0000000000..67791e5d6f --- /dev/null +++ b/packages/types/src/api/account/license.ts @@ -0,0 +1,11 @@ +import { QuotaUsage } from "../../documents" + +export interface GetLicenseRequest { + quotaUsage: QuotaUsage +} + +export interface QuotaUsageTriggeredRequest { + percentage: number + name: string + resetDate: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e7dcf2d89f..eb12f7754b 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,4 +1,5 @@ import { Feature, Hosting, PlanType, Quotas } from "../../sdk" +import { QuotaUsage } from "../global" export interface CreateAccount { email: string @@ -42,6 +43,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { diff --git a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts index bfcbb9f4e2..dafc2b1ff2 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts @@ -16,9 +16,7 @@ class InternalAPIClient { constructor(appId?: string) { if (!env.BUDIBASE_SERVER_URL) { - throw new Error( - "Must set BUDIBASE_SERVER_URL env var" - ) + throw new Error("Must set BUDIBASE_SERVER_URL env var") } this.host = `${env.BUDIBASE_SERVER_URL}/api` this.appId = appId @@ -55,4 +53,4 @@ class InternalAPIClient { put = this.apiCall("PUT") } -export default InternalAPIClient \ No newline at end of file +export default InternalAPIClient diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 10e4a6657b..0c51487122 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -1,6 +1,4 @@ -import { - Application, -} from "@budibase/server/api/controllers/public/mapping/types" +import { Application } from "@budibase/server/api/controllers/public/mapping/types" import { App } from "@budibase/types" import { Response } from "node-fetch" import InternalAPIClient from "./InternalAPIClient" @@ -37,9 +35,7 @@ export default class AppApi { return [response, json] } - async create( - body: any - ): Promise<[Response, Partial]> { + async create(body: any): Promise<[Response, Partial]> { const response = await this.api.post(`/applications`, { body }) const json = await response.json() return [response, json] diff --git a/qa-core/src/config/internal-api/TestConfiguration/auth.ts b/qa-core/src/config/internal-api/TestConfiguration/auth.ts index 6ac53f24b6..d83c859ab3 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/auth.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/auth.ts @@ -9,11 +9,11 @@ export default class AuthApi { } async login(): Promise<[Response, any]> { - const response = await this.api.post(`/global/auth/default/login`, { + const response = await this.api.post(`/global/auth/default/login`, { body: { username: process.env.BB_ADMIN_USER_EMAIL, - password: process.env.BB_ADMIN_USER_PASSWORD - } + password: process.env.BB_ADMIN_USER_PASSWORD, + }, }) const cookie = response.headers.get("set-cookie") this.api.cookie = cookie as any diff --git a/qa-core/src/config/internal-api/fixtures/applications.ts b/qa-core/src/config/internal-api/fixtures/applications.ts index dfad7e0b46..9076a05e1b 100644 --- a/qa-core/src/config/internal-api/fixtures/applications.ts +++ b/qa-core/src/config/internal-api/fixtures/applications.ts @@ -1,10 +1,9 @@ import generator from "../../generator" -import { - Application, -} from "@budibase/server/api/controllers/public/mapping/types" +import { Application } from "@budibase/server/api/controllers/public/mapping/types" - -const generate = (overrides: Partial = {}): Partial => ({ +const generate = ( + overrides: Partial = {} +): Partial => ({ name: generator.word(), url: `/${generator.word()}`, ...overrides, diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index 81d43d9c91..2c934e0bd7 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -24,14 +24,14 @@ describe("Internal API - /applications endpoints", () => { useTemplate: "true", templateName: "Near Miss Register", templateKey: "app/near-miss-register", - templateFile: undefined + templateFile: undefined, }) } it("GET - fetch applications", async () => { await config.applications.create({ ...generateApp(), - useTemplate: false + useTemplate: false, }) const [response, apps] = await config.applications.fetch() expect(response).toHaveStatusCode(200) @@ -57,7 +57,7 @@ describe("Internal API - /applications endpoints", () => { expect(publish).toEqual({ _id: expect.any(String), appUrl: app.url, - status: "SUCCESS" + status: "SUCCESS", }) }) @@ -70,7 +70,8 @@ describe("Internal API - /applications endpoints", () => { config.applications.api.appId = app.appId // check preview renders - const [previewResponse, previewRenders] = await config.applications.canRender() + const [previewResponse, previewRenders] = + await config.applications.canRender() expect(previewResponse).toHaveStatusCode(200) expect(previewRenders).toBe(true) @@ -79,8 +80,8 @@ describe("Internal API - /applications endpoints", () => { // check published app renders config.applications.api.appId = db.getProdAppID(app.appId) - const [publishedAppResponse, publishedAppRenders] = await config.applications.canRender() + const [publishedAppResponse, publishedAppRenders] = + await config.applications.canRender() expect(publishedAppRenders).toBe(true) }) - }) From 55c112677eff75d5994f91e457cbb33970750a5c Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 4 Oct 2022 11:03:24 +0100 Subject: [PATCH 014/101] Added support for JSON array stages --- .../src/components/integration/index.svelte | 7 ++- packages/server/src/integrations/mongodb.ts | 50 +++++++++++-------- packages/types/src/sdk/datasources.ts | 1 - 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/integration/index.svelte b/packages/builder/src/components/integration/index.svelte index a83b9d4db0..c98746ae6e 100644 --- a/packages/builder/src/components/integration/index.svelte +++ b/packages/builder/src/components/integration/index.svelte @@ -41,6 +41,9 @@ flowEditors[i].update(query.fields.steps[i + 1].value.value) } } + + $: shouldDisplayJsonBox = + schema.type === QueryTypes.JSON && query.fields.extra?.actionType !== "flow" {#if schema} @@ -55,7 +58,7 @@ value={query.fields.sql} parameters={query.parameters} /> - {:else if schema.type === QueryTypes.JSON} + {:else if shouldDisplayJsonBox} {/if} - {:else if schema.type === QueryTypes.FLOW} + {:else if query.fields.extra?.actionType === "flow"}
{#if (query.fields.steps?.length ?? 0) === 0}
diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index b5fed323a5..aae345c981 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -57,7 +57,7 @@ const SCHEMA: Integration = { type: QueryType.JSON, }, aggregate: { - type: QueryType.FLOW, + type: QueryType.JSON, }, }, extra: { @@ -66,8 +66,8 @@ const SCHEMA: Integration = { type: DatasourceFieldType.STRING, required: true, }, - actionTypes: { - displayName: "Action Types", + actionType: { + displayName: "Action Type", type: DatasourceFieldType.LIST, required: true, data: { @@ -75,6 +75,7 @@ const SCHEMA: Integration = { create: ["insertOne", "insertMany"], update: ["updateOne", "updateMany"], delete: ["deleteOne", "deleteMany"], + aggregate: ["json", "flow"], }, }, }, @@ -170,7 +171,7 @@ class MongoIntegration implements IntegrationBase { // For mongodb we add an extra actionType to specify // which method we want to call on the collection - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "insertOne": { return await collection.insertOne(json) } @@ -179,7 +180,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for create` + `actionType ${query.extra.actionType} does not exist on DB for create` ) } } @@ -198,7 +199,7 @@ class MongoIntegration implements IntegrationBase { const collection = db.collection(query.extra.collection) let json = this.createObjectIds(query.json) - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "find": { return await collection.find(json).toArray() } @@ -228,7 +229,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for read` + `actionType ${query.extra.actionType} does not exist on DB for read` ) } } @@ -255,7 +256,7 @@ class MongoIntegration implements IntegrationBase { options: object } - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "updateOne": { return await collection.updateOne( json.filter, @@ -272,7 +273,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for update` + `actionType ${query.extra.actionType} does not exist on DB for update` ) } } @@ -304,7 +305,7 @@ class MongoIntegration implements IntegrationBase { } } - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "deleteOne": { return await collection.deleteOne(json.filter, json.options) } @@ -313,7 +314,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for delete` + `actionType ${query.extra.actionType} does not exist on DB for delete` ) } } @@ -325,20 +326,29 @@ class MongoIntegration implements IntegrationBase { } } - async aggregate(query: { steps: any[]; extra: { [key: string]: string } }) { + async aggregate(query: { json: object; steps: any[]; extra: { [key: string]: string } }) { try { await this.connect() const db = this.client.db(this.config.db) const collection = db.collection(query.extra.collection) let response = [] - for await (const doc of collection.aggregate( - query.steps.map(({ key, value }) => { - let temp: any = {} - temp[key] = JSON.parse(value.value) - return temp - }) - )) { - response.push(doc) + if (query.extra?.actionType === "flow") { + for await (const doc of collection.aggregate( + query.steps.map(({ key, value }) => { + let temp: any = {} + temp[key] = JSON.parse(value.value) + return temp + }) + )) { + response.push(doc) + } + } else { + const stages: Array = query.json as Array + for await (const doc of collection.aggregate( + stages ? stages : [] + )) { + response.push(doc) + } } return response } catch (err) { diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 1dbdc6e423..d01d636b86 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -20,7 +20,6 @@ export enum QueryType { SQL = "sql", JSON = "json", FIELDS = "fields", - FLOW = "flow", } export enum DatasourceFieldType { From 9580e83ae38c2872935208559c148caaf806faa3 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 4 Oct 2022 11:03:54 +0100 Subject: [PATCH 015/101] lint --- packages/server/src/integrations/mongodb.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index aae345c981..85aa341935 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -326,7 +326,11 @@ class MongoIntegration implements IntegrationBase { } } - async aggregate(query: { json: object; steps: any[]; extra: { [key: string]: string } }) { + async aggregate(query: { + json: object + steps: any[] + extra: { [key: string]: string } + }) { try { await this.connect() const db = this.client.db(this.config.db) @@ -344,9 +348,7 @@ class MongoIntegration implements IntegrationBase { } } else { const stages: Array = query.json as Array - for await (const doc of collection.aggregate( - stages ? stages : [] - )) { + for await (const doc of collection.aggregate(stages ? stages : [])) { response.push(doc) } } From a95f32c5de8c1c8622255709e24df4aa647c232a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 4 Oct 2022 11:29:13 +0100 Subject: [PATCH 016/101] Rename actionTypes to actionType --- .../src/integrations/tests/mongo.spec.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/server/src/integrations/tests/mongo.spec.ts b/packages/server/src/integrations/tests/mongo.spec.ts index b06709d8e2..b941d7ae66 100644 --- a/packages/server/src/integrations/tests/mongo.spec.ts +++ b/packages/server/src/integrations/tests/mongo.spec.ts @@ -34,7 +34,7 @@ describe("MongoDB Integration", () => { await config.integration.create({ index: indexName, json: body, - extra: { collection: "testCollection", actionTypes: "insertOne" }, + extra: { collection: "testCollection", actionType: "insertOne" }, }) expect(config.integration.client.insertOne).toHaveBeenCalledWith(body) }) @@ -44,7 +44,7 @@ describe("MongoDB Integration", () => { json: { address: "test", }, - extra: { collection: "testCollection", actionTypes: "find" }, + extra: { collection: "testCollection", actionType: "find" }, } const response = await config.integration.read(query) expect(config.integration.client.find).toHaveBeenCalledWith(query.json) @@ -61,7 +61,7 @@ describe("MongoDB Integration", () => { opt: "option", }, }, - extra: { collection: "testCollection", actionTypes: "deleteOne" }, + extra: { collection: "testCollection", actionType: "deleteOne" }, } await config.integration.delete(query) expect(config.integration.client.deleteOne).toHaveBeenCalledWith( @@ -83,7 +83,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalledWith( @@ -97,7 +97,7 @@ describe("MongoDB Integration", () => { const restore = disableConsole() const query = { - extra: { collection: "testCollection", actionTypes: "deleteOne" }, + extra: { collection: "testCollection", actionType: "deleteOne" }, } let error = null @@ -125,7 +125,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -161,7 +161,7 @@ describe("MongoDB Integration", () => { upsert: true, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -200,7 +200,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" }, + extra: { collection: "testCollection", actionType: "findOneAndUpdate" }, } await config.integration.read(query) expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled() @@ -245,7 +245,7 @@ describe("MongoDB Integration", () => { { "upsert": true }`, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -300,7 +300,7 @@ describe("MongoDB Integration", () => { "upsert": true, "extra": "ad\\"{\\"d" }`, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() From 2c2cd4c9f6777646344f1ff899e8668fe2365fb3 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 4 Oct 2022 16:11:19 +0100 Subject: [PATCH 017/101] Add Sync endpoint --- .../config/internal-api/TestConfiguration/applications.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 10e4a6657b..4247b06d2a 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -50,4 +50,11 @@ export default class AppApi { const json = await response.json() return [response, json.data] } + + async sync(appId: string): Promise<[Response, any]> { + const response = await this.api.post(`/applications/${appId}/sync`) + const json = await response.json() + return [response, json] + } + } From ea8122d5a0da6f48be964b3b93203e560d56db48 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 4 Oct 2022 17:12:38 +0100 Subject: [PATCH 018/101] Add missing applications endpoints --- .../TestConfiguration/applications.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 26375154e9..82b85372c6 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -53,4 +53,29 @@ export default class AppApi { return [response, json] } + async update(appId: string, body: any): Promise<[Response, Application]> { + const response = await this.api.put(`/applications/${appId}/client/update`, { body }) + const json = await response.json() + return [response, json] + } + + async revert(appId: string): Promise<[Response, Application]> { + const response = await this.api.post(`/applications/${appId}/client/revert`) + const json = await response.json() + return [response, json] + } + + async delete(appId: string): Promise<[Response, any]> { + const response = await this.api.del(`/applications/${appId}`) + const json = await response.json() + return [response, json] + } + + async getAppDefinition(appId: string): Promise<[Response, any]> { + const response = await this.api.get(`/applications/${appId}/definition`) + const json = await response.json() + return [response, json] + } + + } From 2ec21741d1c889df4c5b8d14d069d2f574e58010 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 5 Oct 2022 09:28:07 +0100 Subject: [PATCH 019/101] Initial commit of QR Reader field --- .../screenTemplates/utils/commonComponents.js | 1 + .../DataTable/modals/CreateEditColumn.svelte | 1 + .../TableNavigator/TableDataImport.svelte | 4 + .../design/settings/componentSettings.js | 1 + .../settings/controls/FormFieldSelect.svelte | 2 +- .../builder/src/constants/backend/index.js | 10 ++ .../new/_components/componentStructure.json | 3 +- packages/client/manifest.json | 36 +++++ packages/client/package.json | 1 + .../src/components/app/CodeScanner.svelte | 141 ++++++++++++++++++ .../app/forms/CodeScannerField.svelte | 55 +++++++ .../client/src/components/app/forms/index.js | 1 + packages/client/src/constants.js | 1 + packages/server/src/constants/index.js | 2 + .../server/src/integrations/base/sqlTable.ts | 1 + packages/server/src/utilities/csvParser.js | 1 + .../src/utilities/rowProcessor/index.js | 5 + packages/types/src/sdk/datasources.ts | 1 + 18 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 packages/client/src/components/app/CodeScanner.svelte create mode 100644 packages/client/src/components/app/forms/CodeScannerField.svelte diff --git a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js index 9176d535ab..cc8c5ef712 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js +++ b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js @@ -138,6 +138,7 @@ const fieldTypeToComponentMap = { attachment: "attachmentfield", link: "relationshipfield", json: "jsonfield", + code: "codescanner", } export function makeDatasourceFormComponents(datasource) { diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 21059b32dd..d9993539ce 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -261,6 +261,7 @@ } else { return [ FIELDS.STRING, + FIELDS.CODE, FIELDS.LONGFORM, FIELDS.OPTIONS, FIELDS.DATETIME, diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index 774aac0677..2b53a75152 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -124,6 +124,10 @@ label: "Multi-select", value: FIELDS.ARRAY.type, }, + { + label: "Code", + value: FIELDS.CODE.type, + }, ] diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index 56ae3de490..21dbeb4beb 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -51,6 +51,7 @@ const componentMap = { "field/link": FormFieldSelect, "field/array": FormFieldSelect, "field/json": FormFieldSelect, + "field/code": FormFieldSelect, // Some validation types are the same as others, so not all types are // explicitly listed here. e.g. options uses string validation "validation/string": ValidationEditor, diff --git a/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte index 1f08c56ff5..48e8213ee1 100644 --- a/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte @@ -26,7 +26,7 @@ let entries = Object.entries(schema ?? {}) let types = [] - if (type === "field/options") { + if ((type === "field/options", type === "field/code")) { // allow options to be used on both options and string fields types = [type, "field/string"] } else { diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 28ce35d9f7..bd2d5bc4e3 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -8,6 +8,15 @@ export const FIELDS = { presence: false, }, }, + CODE: { + name: "Code", + type: "code", + constraints: { + type: "string", + length: {}, + presence: false, + }, + }, LONGFORM: { name: "Long Form Text", type: "longform", @@ -148,6 +157,7 @@ export const ALLOWABLE_STRING_OPTIONS = [ FIELDS.STRING, FIELDS.OPTIONS, FIELDS.LONGFORM, + FIELDS.CODE, ] export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map( opt => opt.type diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json index 671637f381..be17dbe868 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json @@ -65,7 +65,8 @@ "relationshipfield", "datetimefield", "multifieldselect", - "s3upload" + "s3upload", + "codescanner" ] }, { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index b6d4941e4c..5ccb31409d 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3157,6 +3157,42 @@ } ] }, + "codescanner": { + "name": "Code Scanner", + "icon": "Camera", + "styles": [ + "size" + ], + "draggable": true, + "illegalChildren": [ + "section" + ], + "settings": [ + { + "type": "text", + "label": "Label", + "key": "label", + "defaultValue": true + }, + { + "type": "field/code", + "label": "Field", + "key": "field", + "required": true + }, + { + "type": "boolean", + "label": "Disabled", + "key": "disabled", + "defaultValue": false + }, + { + "type": "validation/string", + "label": "Validation", + "key": "validation" + } + ] + }, "embeddedmap": { "name": "Embedded Map", "icon": "Location", diff --git a/packages/client/package.json b/packages/client/package.json index 1e2049ac27..aa8c6da1ea 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -33,6 +33,7 @@ "apexcharts": "^3.22.1", "dayjs": "^1.10.5", "downloadjs": "1.4.7", + "html5-qrcode": "^2.2.1", "leaflet": "^1.7.1", "regexparam": "^1.3.0", "sanitize-html": "^2.7.0", diff --git a/packages/client/src/components/app/CodeScanner.svelte b/packages/client/src/components/app/CodeScanner.svelte new file mode 100644 index 0000000000..7203b30aef --- /dev/null +++ b/packages/client/src/components/app/CodeScanner.svelte @@ -0,0 +1,141 @@ + + +
+ {#if code} +
+ + {code} +
+ {/if} + +
+ + +
+ +
+ + +
+ + diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte new file mode 100644 index 0000000000..034e6eacce --- /dev/null +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -0,0 +1,55 @@ + + + + {#if fieldState} + + {/if} + diff --git a/packages/client/src/components/app/forms/index.js b/packages/client/src/components/app/forms/index.js index 0ff82cea94..44c1516885 100644 --- a/packages/client/src/components/app/forms/index.js +++ b/packages/client/src/components/app/forms/index.js @@ -13,3 +13,4 @@ export { default as passwordfield } from "./PasswordField.svelte" export { default as formstep } from "./FormStep.svelte" export { default as jsonfield } from "./JSONField.svelte" export { default as s3upload } from "./S3Upload.svelte" +export { default as codescanner } from "./CodeScannerField.svelte" diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js index 51ef3fd124..9bd6d27bb0 100644 --- a/packages/client/src/constants.js +++ b/packages/client/src/constants.js @@ -1,5 +1,6 @@ export const FieldTypes = { STRING: "string", + CODE: "code", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index c002c10f7b..ef15711cb7 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -31,6 +31,7 @@ exports.NoEmptyFilterStrings = [ exports.FieldTypes = { STRING: "string", + CODE: "code", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", @@ -51,6 +52,7 @@ exports.CanSwitchTypes = [ exports.FieldTypes.STRING, exports.FieldTypes.OPTIONS, exports.FieldTypes.LONGFORM, + exports.FieldTypes.CODE, ], [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER], ] diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index af561e81c5..4626297ddc 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -40,6 +40,7 @@ function generateSchema( case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: + case FieldTypes.CODE: schema.text(key) break case FieldTypes.NUMBER: diff --git a/packages/server/src/utilities/csvParser.js b/packages/server/src/utilities/csvParser.js index 8f9b3373c9..7eba0493eb 100644 --- a/packages/server/src/utilities/csvParser.js +++ b/packages/server/src/utilities/csvParser.js @@ -4,6 +4,7 @@ const { FieldTypes } = require("../constants") const VALIDATORS = { [FieldTypes.STRING]: () => true, [FieldTypes.OPTIONS]: () => true, + [FieldTypes.CODE]: () => true, [FieldTypes.NUMBER]: attribute => { // allow not to be present if (!attribute) { diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index e4c364eaf3..55b1c09538 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -48,6 +48,11 @@ const TYPE_TRANSFORM_MAP = { [null]: "", [undefined]: undefined, }, + [FieldTypes.CODE]: { + "": "", + [null]: "", + [undefined]: undefined, + }, [FieldTypes.FORMULA]: { "": "", [null]: "", diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index d01d636b86..c76ea376b6 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -24,6 +24,7 @@ export enum QueryType { export enum DatasourceFieldType { STRING = "string", + CODE = "code", LONGFORM = "longForm", BOOLEAN = "boolean", NUMBER = "number", From 617d443890473bb4d49313168cbc57b308f3016e Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 6 Oct 2022 09:54:43 +0100 Subject: [PATCH 020/101] UX/UI updates and rollup config changes to exclude common warnings --- packages/client/manifest.json | 27 ++- packages/client/rollup.config.js | 9 + .../src/components/app/CodeScanner.svelte | 226 +++++++++++++----- .../app/forms/CodeScannerField.svelte | 34 +-- 4 files changed, 202 insertions(+), 94 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 5ccb31409d..acb05d58bc 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3168,24 +3168,39 @@ "section" ], "settings": [ - { - "type": "text", - "label": "Label", - "key": "label", - "defaultValue": true - }, { "type": "field/code", "label": "Field", "key": "field", "required": true }, + { + "type": "text", + "label": "Label", + "key": "label" + }, + { + "type": "text", + "label": "Button", + "key": "scanButtonText" + }, + { + "type": "text", + "label": "Default value", + "key": "defaultValue" + }, { "type": "boolean", "label": "Disabled", "key": "disabled", "defaultValue": false }, + { + "type": "boolean", + "label": "Allow manual entry", + "key": "allowManualEntry", + "defaultValue": false + }, { "type": "validation/string", "label": "Validation", diff --git a/packages/client/rollup.config.js b/packages/client/rollup.config.js index 5206b63884..be94ace90c 100644 --- a/packages/client/rollup.config.js +++ b/packages/client/rollup.config.js @@ -27,6 +27,15 @@ export default { file: `./dist/budibase-client.js`, }, ], + onwarn(warning, warn) { + if ( + warning.code === "THIS_IS_UNDEFINED" || + warning.code === "CIRCULAR_DEPENDENCY" + ) { + return + } + warn(warning) + }, plugins: [ alias({ entries: [ diff --git a/packages/client/src/components/app/CodeScanner.svelte b/packages/client/src/components/app/CodeScanner.svelte index 7203b30aef..3215875faa 100644 --- a/packages/client/src/components/app/CodeScanner.svelte +++ b/packages/client/src/components/app/CodeScanner.svelte @@ -1,123 +1,189 @@
- {#if code} -
+ {#if value && !manualMode} +
- {code} + {value}
{/if} - + + {#if allowManualEntry && manualMode} +
+ { + dispatch("change", value) + }} + /> +
+ {/if} + + {#if value} + { + dispatch("change", "") + }} + {disabled} + > + Clear + + {:else} + { + showReaderModal() + }} + {disabled} + > + {scanButtonText} + + {/if}
diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index 034e6eacce..e2da89d1b5 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -9,34 +9,18 @@ export let validation export let defaultValue = "" export let onChange + export let allowManualEntry + export let scanButtonText let fieldState let fieldApi - let scannedCode - let loaded = false - - const handleInput = () => { - const changed = fieldApi.setValue(scannedCode) + const handleUpdate = e => { + const changed = fieldApi.setValue(e.detail) if (onChange && changed) { - onChange({ value: scannedCode }) + onChange({ value: e.detail }) } } - - $: if (!loaded && !scannedCode && fieldState?.value) { - scannedCode = fieldState.value + "" - loaded = true - } - - /* - QR Nimiq has rollup issues? - QR qrcodejs 12b bundle? - https://github.com/davidshimjs/qrcodejs - BOTH html5-qrcode has a 330k bundle - https://github.com/mebjas/html5-qrcode - BOTH zxing 360k bundle size - https://github.com/zxing-js/library - */ {#if fieldState} - + {/if} From e251c914ff04a5a2d234d092c8e240d47b266a15 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 6 Oct 2022 13:02:17 +0100 Subject: [PATCH 021/101] Review feedback changes. Changed 'Code' labelling to 'Scanned Code' --- .../store/screenTemplates/utils/commonComponents.js | 2 +- .../backend/DataTable/modals/CreateEditColumn.svelte | 2 +- .../backend/TableNavigator/TableDataImport.svelte | 4 ++-- .../src/components/design/settings/componentSettings.js | 2 +- .../design/settings/controls/FormFieldSelect.svelte | 2 +- packages/builder/src/constants/backend/index.js | 8 ++++---- packages/client/manifest.json | 2 +- .../src/components/app/forms/CodeScannerField.svelte | 2 +- packages/client/src/constants.js | 2 +- packages/server/src/constants/index.js | 4 ++-- packages/server/src/integrations/base/sqlTable.ts | 2 +- packages/server/src/utilities/csvParser.js | 2 +- packages/server/src/utilities/rowProcessor/index.js | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js index cc8c5ef712..a6c71ae54e 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js +++ b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js @@ -138,7 +138,7 @@ const fieldTypeToComponentMap = { attachment: "attachmentfield", link: "relationshipfield", json: "jsonfield", - code: "codescanner", + scannedcode: "codescanner", } export function makeDatasourceFormComponents(datasource) { diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 8f6b4a746e..a4cddf512f 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -261,7 +261,7 @@ } else { return [ FIELDS.STRING, - FIELDS.CODE, + FIELDS.SCANNEDCODE, FIELDS.LONGFORM, FIELDS.OPTIONS, FIELDS.DATETIME, diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index 2b53a75152..4d827b59b5 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -125,8 +125,8 @@ value: FIELDS.ARRAY.type, }, { - label: "Code", - value: FIELDS.CODE.type, + label: "Scanned Code", + value: FIELDS.SCANNEDCODE.type, }, ] diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index 21dbeb4beb..eadaecd8cc 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -51,7 +51,7 @@ const componentMap = { "field/link": FormFieldSelect, "field/array": FormFieldSelect, "field/json": FormFieldSelect, - "field/code": FormFieldSelect, + "field/scannedcode": FormFieldSelect, // Some validation types are the same as others, so not all types are // explicitly listed here. e.g. options uses string validation "validation/string": ValidationEditor, diff --git a/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte index 48e8213ee1..81b01b2b77 100644 --- a/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte @@ -26,7 +26,7 @@ let entries = Object.entries(schema ?? {}) let types = [] - if ((type === "field/options", type === "field/code")) { + if ((type === "field/options", type === "field/scannedcode")) { // allow options to be used on both options and string fields types = [type, "field/string"] } else { diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index bd2d5bc4e3..93bb90344e 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -8,9 +8,9 @@ export const FIELDS = { presence: false, }, }, - CODE: { - name: "Code", - type: "code", + SCANNEDCODE: { + name: "Scanned Code", + type: "scannedcode", constraints: { type: "string", length: {}, @@ -157,7 +157,7 @@ export const ALLOWABLE_STRING_OPTIONS = [ FIELDS.STRING, FIELDS.OPTIONS, FIELDS.LONGFORM, - FIELDS.CODE, + FIELDS.SCANNEDCODE, ] export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map( opt => opt.type diff --git a/packages/client/manifest.json b/packages/client/manifest.json index acb05d58bc..64d90875a7 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3169,7 +3169,7 @@ ], "settings": [ { - "type": "field/code", + "type": "field/scannedcode", "label": "Field", "key": "field", "required": true diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index e2da89d1b5..38eab79dc8 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -4,7 +4,7 @@ export let field export let label - export let type = "code" + export let type = "scannedcode" export let disabled = false export let validation export let defaultValue = "" diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js index 9bd6d27bb0..bad0c30371 100644 --- a/packages/client/src/constants.js +++ b/packages/client/src/constants.js @@ -1,6 +1,6 @@ export const FieldTypes = { STRING: "string", - CODE: "code", + SCANNEDCODE: "scannedcode", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index ef15711cb7..4b54424efa 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -31,7 +31,7 @@ exports.NoEmptyFilterStrings = [ exports.FieldTypes = { STRING: "string", - CODE: "code", + SCANNEDCODE: "scannedcode", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", @@ -52,7 +52,7 @@ exports.CanSwitchTypes = [ exports.FieldTypes.STRING, exports.FieldTypes.OPTIONS, exports.FieldTypes.LONGFORM, - exports.FieldTypes.CODE, + exports.FieldTypes.SCANNEDCODE, ], [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER], ] diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 4626297ddc..52eb25be9d 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -40,7 +40,7 @@ function generateSchema( case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: - case FieldTypes.CODE: + case FieldTypes.SCANNEDCODE: schema.text(key) break case FieldTypes.NUMBER: diff --git a/packages/server/src/utilities/csvParser.js b/packages/server/src/utilities/csvParser.js index 7eba0493eb..08962032b6 100644 --- a/packages/server/src/utilities/csvParser.js +++ b/packages/server/src/utilities/csvParser.js @@ -4,7 +4,7 @@ const { FieldTypes } = require("../constants") const VALIDATORS = { [FieldTypes.STRING]: () => true, [FieldTypes.OPTIONS]: () => true, - [FieldTypes.CODE]: () => true, + [FieldTypes.SCANNEDCODE]: () => true, [FieldTypes.NUMBER]: attribute => { // allow not to be present if (!attribute) { diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index 55b1c09538..b61fbe66e1 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -48,7 +48,7 @@ const TYPE_TRANSFORM_MAP = { [null]: "", [undefined]: undefined, }, - [FieldTypes.CODE]: { + [FieldTypes.SCANNEDCODE]: { "": "", [null]: "", [undefined]: undefined, From aff6e5cbbb0dbed12d0d3993466d576c1f8104f8 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 6 Oct 2022 16:03:47 +0100 Subject: [PATCH 022/101] Process quota emails in account-portal --- packages/backend-core/package.json | 2 + .../tests/utilities/structures/accounts.ts | 23 +++++++ .../tests/utilities/structures/common.ts | 1 + .../tests/utilities/structures/index.ts | 7 ++ .../tests/utilities/structures/licenses.ts | 18 +++++ packages/backend-core/yarn.lock | 10 +++ .../src/api/controllers/deploy/index.ts | 1 - packages/types/src/api/account/license.ts | 6 -- .../types/src/documents/account/account.ts | 36 +++++++++- packages/types/src/documents/global/quotas.ts | 13 ++-- packages/types/src/sdk/licensing/quota.ts | 35 ++++++---- .../src/api/controllers/system/tenants.js | 58 ---------------- .../src/api/controllers/system/tenants.ts | 66 +++++++++++++++++++ packages/worker/src/sdk/users/events.ts | 1 - 14 files changed, 190 insertions(+), 87 deletions(-) create mode 100644 packages/backend-core/tests/utilities/structures/accounts.ts create mode 100644 packages/backend-core/tests/utilities/structures/common.ts create mode 100644 packages/backend-core/tests/utilities/structures/licenses.ts delete mode 100644 packages/worker/src/api/controllers/system/tenants.js create mode 100644 packages/worker/src/api/controllers/system/tenants.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index a07db6fca0..ed31e3c0c8 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -62,6 +62,7 @@ ] }, "devDependencies": { + "@types/chance": "1.1.3", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", @@ -73,6 +74,7 @@ "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", "ioredis-mock": "5.8.0", + "chance": "1.1.3", "jest": "27.5.1", "koa": "2.7.0", "nodemon": "2.0.16", diff --git a/packages/backend-core/tests/utilities/structures/accounts.ts b/packages/backend-core/tests/utilities/structures/accounts.ts new file mode 100644 index 0000000000..5d23962575 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/accounts.ts @@ -0,0 +1,23 @@ +import { generator, uuid } from "." +import { AuthType, CloudAccount, Hosting } from "@budibase/types" +import * as db from "../../../src/db/utils" + +export const cloudAccount = (): CloudAccount => { + return { + accountId: uuid(), + createdAt: Date.now(), + verified: true, + verificationSent: true, + tier: "", + email: generator.email(), + tenantId: generator.word(), + hosting: Hosting.CLOUD, + authType: AuthType.PASSWORD, + password: generator.word(), + tenantName: generator.word(), + name: generator.name(), + size: "10+", + profession: "Software Engineer", + budibaseUserId: db.generateGlobalUserID(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/common.ts b/packages/backend-core/tests/utilities/structures/common.ts new file mode 100644 index 0000000000..51ae220254 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/common.ts @@ -0,0 +1 @@ +export { v4 as uuid } from "uuid" diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 12b6ab7ad6..68064b9715 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1 +1,8 @@ +export * from "./common" + +import Chance from "chance" +export const generator = new Chance() + export * as koa from "./koa" +export * as accounts from "./accounts" +export * as licenses from "./licenses" diff --git a/packages/backend-core/tests/utilities/structures/licenses.ts b/packages/backend-core/tests/utilities/structures/licenses.ts new file mode 100644 index 0000000000..a541e91860 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/licenses.ts @@ -0,0 +1,18 @@ +import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" + +const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { + return { + type, + } +} + +export const newLicense = (opts: { + quotas: Quotas + planType?: PlanType +}): License => { + return { + features: [], + quotas: opts.quotas, + plan: newPlan(opts.planType), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 2e62aea734..6bc9b63728 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -663,6 +663,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/chance@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" + integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw== + "@types/connect@*": version "3.4.35" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chance@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" + integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== + char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 5edf862706..a51e7ad6ec 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -17,7 +17,6 @@ import { getProdAppDB, getDevAppDB, } from "@budibase/backend-core/context" -import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" // the max time we can wait for an invalidation to complete before considering it failed diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts index 67791e5d6f..80481eb8f5 100644 --- a/packages/types/src/api/account/license.ts +++ b/packages/types/src/api/account/license.ts @@ -3,9 +3,3 @@ import { QuotaUsage } from "../../documents" export interface GetLicenseRequest { quotaUsage: QuotaUsage } - -export interface QuotaUsageTriggeredRequest { - percentage: number - name: string - resetDate: string -} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index eb12f7754b..e3fcf75ecf 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,5 +1,12 @@ -import { Feature, Hosting, PlanType, Quotas } from "../../sdk" -import { QuotaUsage } from "../global" +import { + Feature, + Hosting, + MonthlyQuotaName, + PlanType, + Quotas, + StaticQuotaName, +} from "../../sdk" +import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global" export interface CreateAccount { email: string @@ -43,7 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides - quotaUsage?: QuotaUsage + quotaUsage?: AccountQuotaUsage } export interface PasswordAccount extends Account { @@ -86,3 +93,26 @@ export interface OAuthTokens { accessToken: string refreshToken: string } + +export type QuotaTriggers = { + [key: string]: string | null +} + +export interface AccountStaticUsage extends StaticUsage { + triggers?: { + [key in StaticQuotaName]?: QuotaTriggers + } +} + +export interface AccountMonthlyUsage extends MonthlyUsage { + triggers?: { + [key in MonthlyQuotaName]?: QuotaTriggers + } +} + +export interface AccountQuotaUsage extends QuotaUsage { + usageQuota: AccountStaticUsage + monthly: { + [key: string]: AccountMonthlyUsage + } +} diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index eb1d77c228..d9cc83b46a 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,7 +24,14 @@ export interface UsageBreakdown { } } -export type MonthlyUsage = { +export interface StaticUsage { + [StaticQuotaName.APPS]: number + [StaticQuotaName.PLUGINS]: number + [StaticQuotaName.USER_GROUPS]: number + [StaticQuotaName.ROWS]: number +} + +export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number @@ -34,9 +41,7 @@ export type MonthlyUsage = { } export interface BaseQuotaUsage { - usageQuota: { - [key in StaticQuotaName]: number - } + usageQuota: StaticUsage monthly: { [key: string]: MonthlyUsage } diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 49dd561db0..6e58734925 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,26 +61,33 @@ export type PlanQuotas = { [PlanType.ENTERPRISE]: Quotas } +export type MonthlyQuotas = { + [MonthlyQuotaName.QUERIES]: Quota + [MonthlyQuotaName.AUTOMATIONS]: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota +} + +export type StaticQuotas = { + [StaticQuotaName.ROWS]: Quota + [StaticQuotaName.APPS]: Quota + [StaticQuotaName.USER_GROUPS]: Quota + [StaticQuotaName.PLUGINS]: Quota +} + +export type ConstantQuotas = { + [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota +} + export type Quotas = { [QuotaType.USAGE]: { - [QuotaUsageType.MONTHLY]: { - [MonthlyQuotaName.QUERIES]: Quota - [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota - } - [QuotaUsageType.STATIC]: { - [StaticQuotaName.ROWS]: Quota - [StaticQuotaName.APPS]: Quota - [StaticQuotaName.USER_GROUPS]: Quota - [StaticQuotaName.PLUGINS]: Quota - } - } - [QuotaType.CONSTANT]: { - [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota + [QuotaUsageType.MONTHLY]: MonthlyQuotas + [QuotaUsageType.STATIC]: StaticQuotas } + [QuotaType.CONSTANT]: ConstantQuotas } export interface Quota { name: string value: number + triggers: number[] } diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js deleted file mode 100644 index c54a3d9834..0000000000 --- a/packages/worker/src/api/controllers/system/tenants.js +++ /dev/null @@ -1,58 +0,0 @@ -const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") -const { getTenantId } = require("@budibase/backend-core/tenancy") -const { deleteTenant } = require("@budibase/backend-core/deprovision") -const { quotas } = require("@budibase/pro") - -exports.exists = async ctx => { - const tenantId = ctx.request.params - ctx.body = { - exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let exists = false - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 - } - } catch (err) { - // if error it doesn't exist - } - return exists - }), - } -} - -exports.fetch = async ctx => { - ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let tenants = [] - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - tenants = tenantsDoc.tenantIds - } - } catch (err) { - // if error it doesn't exist - } - return tenants - }) -} - -exports.delete = async ctx => { - const tenantId = getTenantId() - - if (ctx.params.tenantId !== tenantId) { - ctx.throw(403, "Unauthorized") - } - - try { - await deleteTenant(tenantId) - await quotas.bustCache() - ctx.status = 204 - } catch (err) { - ctx.log.error(err) - throw err - } -} diff --git a/packages/worker/src/api/controllers/system/tenants.ts b/packages/worker/src/api/controllers/system/tenants.ts new file mode 100644 index 0000000000..d6e6261c22 --- /dev/null +++ b/packages/worker/src/api/controllers/system/tenants.ts @@ -0,0 +1,66 @@ +const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") +const { getTenantId } = require("@budibase/backend-core/tenancy") +const { deleteTenant } = require("@budibase/backend-core/deprovision") +import { quotas } from "@budibase/pro" + +export const exists = async (ctx: any) => { + const tenantId = ctx.request.params + ctx.body = { + exists: await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let exists = false + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 + } + } catch (err) { + // if error it doesn't exist + } + return exists + } + ), + } +} + +export const fetch = async (ctx: any) => { + ctx.body = await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let tenants = [] + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + tenants = tenantsDoc.tenantIds + } + } catch (err) { + // if error it doesn't exist + } + return tenants + } + ) +} + +const _delete = async (ctx: any) => { + const tenantId = getTenantId() + + if (ctx.params.tenantId !== tenantId) { + ctx.throw(403, "Unauthorized") + } + + try { + await deleteTenant(tenantId) + await quotas.bustCache() + ctx.status = 204 + } catch (err) { + ctx.log.error(err) + throw err + } +} + +export { _delete as delete } diff --git a/packages/worker/src/sdk/users/events.ts b/packages/worker/src/sdk/users/events.ts index 0094c6fd84..3046442393 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/worker/src/sdk/users/events.ts @@ -1,7 +1,6 @@ import env from "../../environment" import { events, accounts, tenancy } from "@budibase/backend-core" import { User, UserRoles, CloudAccount } from "@budibase/types" -import { users as pro } from "@budibase/pro" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) From 32e6c5bd7a1e844285b24c6753d042ff3f1b0cdb Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 6 Oct 2022 18:19:00 +0100 Subject: [PATCH 023/101] UI fixes --- .../src/components/backend/DataTable/modals/EditRoles.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte index e2ccab11af..c2e3db497a 100644 --- a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte @@ -37,6 +37,7 @@ const id = event?.detail const role = $roles.find(role => role._id === id) if (role) { + console.log("INH ", role.inherits) selectedRole = { ...role, inherits: role.inherits ?? "", @@ -124,7 +125,7 @@ role._id} getOptionLabel={role => role.name} disabled={builtInRoles.includes(selectedRole.name)} From c9f3dc7fe4aa1e49560fe86297c0b424b0eb392a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 6 Oct 2022 19:22:18 +0100 Subject: [PATCH 025/101] Don't allow duplicate roles --- .../backend/DataTable/modals/EditRoles.svelte | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte index 881d9dda7c..600e331d3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte @@ -17,12 +17,21 @@ $: selectedRoleId = selectedRole._id $: otherRoles = editableRoles.filter(role => role._id !== selectedRoleId) $: isCreating = selectedRoleId == null || selectedRoleId === "" + + $: hasUniqueRoleName = !otherRoles + ?.map(role => role.name) + ?.includes(selectedRole.name) + $: valid = selectedRole.name && selectedRole.inherits && selectedRole.permissionId && !builtInRoles.includes(selectedRole.name) + $: shouldDisableRoleInput = + builtInRoles.includes(selectedRole.name) && + selectedRole.name?.toLowerCase() === selectedRoleId?.toLowerCase() + const fetchBasePermissions = async () => { try { basePermissions = await API.getBasePermissions() @@ -99,7 +108,7 @@ title="Edit Roles" confirmText={isCreating ? "Create" : "Save"} onConfirm={saveRole} - disabled={!valid} + disabled={!valid || !hasUniqueRoleName} > {#if errors.length} @@ -119,7 +128,8 @@ x._id} getOptionLabel={x => x.name} - disabled={builtInRoles.includes(selectedRole.name)} + disabled={shouldDisableRoleInput} /> {/if}
From c09154b03ece74b2f405c2a93eb9a8c42f0147f1 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 7 Oct 2022 11:00:25 +0100 Subject: [PATCH 026/101] Renamed the 'Scanned Code' field type to 'Barcode/QR'. The component is also now listed as 'Barcode/QR Scanner'. Minor fix to include longform text columns in the table csv import list --- .../store/screenTemplates/utils/commonComponents.js | 2 +- .../backend/DataTable/modals/CreateEditColumn.svelte | 2 +- .../backend/TableNavigator/TableDataImport.svelte | 8 ++++++-- .../src/components/design/settings/componentSettings.js | 2 +- .../design/settings/controls/FormFieldSelect.svelte | 2 +- packages/builder/src/constants/backend/index.js | 8 ++++---- packages/client/manifest.json | 4 ++-- .../src/components/app/forms/CodeScannerField.svelte | 2 +- packages/client/src/constants.js | 2 +- packages/server/src/constants/index.js | 4 ++-- packages/server/src/integrations/base/sqlTable.ts | 2 +- packages/server/src/utilities/csvParser.js | 2 +- packages/server/src/utilities/rowProcessor/index.js | 2 +- 13 files changed, 23 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js index a6c71ae54e..ac40a10fea 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js +++ b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js @@ -138,7 +138,7 @@ const fieldTypeToComponentMap = { attachment: "attachmentfield", link: "relationshipfield", json: "jsonfield", - scannedcode: "codescanner", + barcodeqr: "codescanner", } export function makeDatasourceFormComponents(datasource) { diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a4cddf512f..b7249ad60c 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -261,7 +261,7 @@ } else { return [ FIELDS.STRING, - FIELDS.SCANNEDCODE, + FIELDS.BARCODEQR, FIELDS.LONGFORM, FIELDS.OPTIONS, FIELDS.DATETIME, diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index 4d827b59b5..ea0ce59169 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -125,8 +125,12 @@ value: FIELDS.ARRAY.type, }, { - label: "Scanned Code", - value: FIELDS.SCANNEDCODE.type, + label: "Barcode/QR", + value: FIELDS.BARCODEQR.type, + }, + { + label: "Long Form Text", + value: FIELDS.LONGFORM.type, }, ] diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index eadaecd8cc..4e9b624f12 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -51,7 +51,7 @@ const componentMap = { "field/link": FormFieldSelect, "field/array": FormFieldSelect, "field/json": FormFieldSelect, - "field/scannedcode": FormFieldSelect, + "field/barcodeqr": FormFieldSelect, // Some validation types are the same as others, so not all types are // explicitly listed here. e.g. options uses string validation "validation/string": ValidationEditor, diff --git a/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte index 81b01b2b77..18653f51a3 100644 --- a/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte @@ -26,7 +26,7 @@ let entries = Object.entries(schema ?? {}) let types = [] - if ((type === "field/options", type === "field/scannedcode")) { + if ((type === "field/options", type === "field/barcodeqr")) { // allow options to be used on both options and string fields types = [type, "field/string"] } else { diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 93bb90344e..427e8d80a1 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -8,9 +8,9 @@ export const FIELDS = { presence: false, }, }, - SCANNEDCODE: { - name: "Scanned Code", - type: "scannedcode", + BARCODEQR: { + name: "Barcode/QR", + type: "barcodeqr", constraints: { type: "string", length: {}, @@ -157,7 +157,7 @@ export const ALLOWABLE_STRING_OPTIONS = [ FIELDS.STRING, FIELDS.OPTIONS, FIELDS.LONGFORM, - FIELDS.SCANNEDCODE, + FIELDS.BARCODEQR, ] export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map( opt => opt.type diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 64d90875a7..5b4edc8f85 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3158,7 +3158,7 @@ ] }, "codescanner": { - "name": "Code Scanner", + "name": "Barcode/QR Scanner", "icon": "Camera", "styles": [ "size" @@ -3169,7 +3169,7 @@ ], "settings": [ { - "type": "field/scannedcode", + "type": "field/barcodeqr", "label": "Field", "key": "field", "required": true diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte index 38eab79dc8..7f7d82f304 100644 --- a/packages/client/src/components/app/forms/CodeScannerField.svelte +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -4,7 +4,7 @@ export let field export let label - export let type = "scannedcode" + export let type = "barcodeqr" export let disabled = false export let validation export let defaultValue = "" diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js index bad0c30371..1560552dc4 100644 --- a/packages/client/src/constants.js +++ b/packages/client/src/constants.js @@ -1,6 +1,6 @@ export const FieldTypes = { STRING: "string", - SCANNEDCODE: "scannedcode", + BARCODEQR: "barcodeqr", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 4b54424efa..2a92e87ff8 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -31,7 +31,7 @@ exports.NoEmptyFilterStrings = [ exports.FieldTypes = { STRING: "string", - SCANNEDCODE: "scannedcode", + BARCODEQR: "barcodeqr", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", @@ -52,7 +52,7 @@ exports.CanSwitchTypes = [ exports.FieldTypes.STRING, exports.FieldTypes.OPTIONS, exports.FieldTypes.LONGFORM, - exports.FieldTypes.SCANNEDCODE, + exports.FieldTypes.BARCODEQR, ], [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER], ] diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 52eb25be9d..02f7ad8718 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -40,7 +40,7 @@ function generateSchema( case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: - case FieldTypes.SCANNEDCODE: + case FieldTypes.BARCODEQR: schema.text(key) break case FieldTypes.NUMBER: diff --git a/packages/server/src/utilities/csvParser.js b/packages/server/src/utilities/csvParser.js index 08962032b6..09449f6fc1 100644 --- a/packages/server/src/utilities/csvParser.js +++ b/packages/server/src/utilities/csvParser.js @@ -4,7 +4,7 @@ const { FieldTypes } = require("../constants") const VALIDATORS = { [FieldTypes.STRING]: () => true, [FieldTypes.OPTIONS]: () => true, - [FieldTypes.SCANNEDCODE]: () => true, + [FieldTypes.BARCODEQR]: () => true, [FieldTypes.NUMBER]: attribute => { // allow not to be present if (!attribute) { diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index b61fbe66e1..91daa1b5a0 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -48,7 +48,7 @@ const TYPE_TRANSFORM_MAP = { [null]: "", [undefined]: undefined, }, - [FieldTypes.SCANNEDCODE]: { + [FieldTypes.BARCODEQR]: { "": "", [null]: "", [undefined]: undefined, From ea029291309b5df1bc4284b6af0c22022a8ab74e Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 7 Oct 2022 12:19:00 +0100 Subject: [PATCH 027/101] Add tests for endpoints --- .../TestConfiguration/applications.ts | 8 ++- .../internal-api/applications/create.spec.ts | 67 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 82b85372c6..3b577abf5e 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -53,7 +53,7 @@ export default class AppApi { return [response, json] } - async update(appId: string, body: any): Promise<[Response, Application]> { + async updateClient(appId: string, body: any): Promise<[Response, Application]> { const response = await this.api.put(`/applications/${appId}/client/update`, { body }) const json = await response.json() return [response, json] @@ -77,5 +77,11 @@ export default class AppApi { return [response, json] } + async update(appId: string, body: any): Promise<[Response, Application]> { + const response = await this.api.put(`/applications/${appId}`, { body }) + const json = await response.json() + return [response, json] + } + } diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index 2c934e0bd7..b447a95459 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -84,4 +84,71 @@ describe("Internal API - /applications endpoints", () => { await config.applications.canRender() expect(publishedAppRenders).toBe(true) }) + + it("POST - Sync application before deployment", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + const [syncResponse, sync] = await config.applications.sync(app.appId ? app.appId : "") + expect(syncResponse).toHaveStatusCode(200) + expect(sync).toEqual({ + message: "App sync not required, app not deployed." + }) + }) + + it("POST - Sync application after deployment", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + // publish app + await config.applications.publish() + + const [syncResponse, sync] = await config.applications.sync(app.appId ? app.appId : "") + expect(syncResponse).toHaveStatusCode(200) + expect(sync).toEqual({ + message: "App sync completed successfully." + }) + }) + + it("PUT - Update an application", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + + }) + + + it.skip("POST - Revert Changes", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + + // publish app + await config.applications.publish() + + const [updateResponse, updatedApp] = await config.applications.update(app.appId ? app.appId : "", { + name: generator.word(), + }) + expect(updateResponse).toHaveStatusCode(200) + expect(updatedApp.name).not.toEqual(app.name) + + const [revertResponse, revert] = await config.applications.revert(app.appId ? app.appId : "") + expect(revertResponse).toHaveStatusCode(200) + expect(revert).toEqual({ + message: "App reverted successfully." + }) + }) + + it("DELETE - Delete an application", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + + const [deleteResponse] = await config.applications.delete(app.appId ? app.appId : "") + expect(deleteResponse).toHaveStatusCode(200) + }) }) From 783c012a2676b9bfd06fedb245dd75f20965545f Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Fri, 7 Oct 2022 13:57:10 +0100 Subject: [PATCH 028/101] Move trigger logic inside pro --- packages/types/src/api/account/license.ts | 6 +++++ .../types/src/documents/account/account.ts | 25 +------------------ packages/types/src/documents/global/quotas.ts | 17 +++++++++++++ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts index 80481eb8f5..40ee79c3e3 100644 --- a/packages/types/src/api/account/license.ts +++ b/packages/types/src/api/account/license.ts @@ -3,3 +3,9 @@ import { QuotaUsage } from "../../documents" export interface GetLicenseRequest { quotaUsage: QuotaUsage } + +export interface QuotaTriggeredRequest { + percentage: number + name: string + resetDate?: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e3fcf75ecf..70c3061c3f 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -50,7 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides - quotaUsage?: AccountQuotaUsage + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { @@ -93,26 +93,3 @@ export interface OAuthTokens { accessToken: string refreshToken: string } - -export type QuotaTriggers = { - [key: string]: string | null -} - -export interface AccountStaticUsage extends StaticUsage { - triggers?: { - [key in StaticQuotaName]?: QuotaTriggers - } -} - -export interface AccountMonthlyUsage extends MonthlyUsage { - triggers?: { - [key in MonthlyQuotaName]?: QuotaTriggers - } -} - -export interface AccountQuotaUsage extends QuotaUsage { - usageQuota: AccountStaticUsage - monthly: { - [key: string]: AccountMonthlyUsage - } -} diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index d9cc83b46a..84e5af3996 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,17 +24,27 @@ export interface UsageBreakdown { } } +export type QuotaTriggers = { + [key: string]: string | undefined +} + export interface StaticUsage { [StaticQuotaName.APPS]: number [StaticQuotaName.PLUGINS]: number [StaticQuotaName.USER_GROUPS]: number [StaticQuotaName.ROWS]: number + triggers: { + [key in StaticQuotaName]?: QuotaTriggers + } } export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number + triggers: { + [key in MonthlyQuotaName]?: QuotaTriggers + } breakdown?: { [key in BreakdownQuotaName]?: UsageBreakdown } @@ -56,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage { } } +export type SetUsageValues = { + total: number + app?: number + breakdown?: number + triggers?: QuotaTriggers +} + export type UsageValues = { total: number app?: number From 0c4a40c79582e65efd2cd37e0841d1f2ddc7fbd6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 7 Oct 2022 16:05:01 +0100 Subject: [PATCH 029/101] Switching from scan for app locks to mget - which is a fast O(N) operation that only retrieves what we need. --- packages/backend-core/src/redis/index.ts | 25 +++++++++++++++++++ .../server/src/api/controllers/application.ts | 17 ++++++------- packages/server/src/utilities/redis.js | 8 ++---- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 206110366f..62e718d8ad 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -214,6 +214,31 @@ export = class RedisWrapper { } } + async bulkGet(keys: string[]) { + const db = this._db + const prefixedKeys = keys.map(key => addDbPrefix(db, key)) + let response = await this.getClient().mget(prefixedKeys) + if (Array.isArray(response)) { + let final: any = {} + let count = 0 + for (let result of response) { + if (result) { + let parsed + try { + parsed = JSON.parse(result) + } catch (err) { + parsed = result + } + final[keys[count]] = parsed + } + count++ + } + return final + } else { + throw new Error(`Invalid response: ${response}`) + } + } + async store(key: string, value: any, expirySeconds: number | null = null) { const db = this._db if (typeof value === "object") { diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index b6377a61a2..e8af2f54f8 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -32,7 +32,7 @@ const { import { USERS_TABLE_SCHEMA } from "../../constants" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { clientLibraryPath, stringToReadStream } from "../../utilities" -import { getAllLocks } from "../../utilities/redis" +import { getLocksById } from "../../utilities/redis" import { updateClientLibrary, backupClientLibrary, @@ -45,11 +45,10 @@ import { cleanupAutomations } from "../../automations/utils" import { context } from "@budibase/backend-core" import { checkAppMetadata } from "../../automations/logging" import { getUniqueRows } from "../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" +import { quotas, groups } from "@budibase/pro" import { errors, events, migrations } from "@budibase/backend-core" import { App, Layout, Screen, MigrationType } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" -import { groups } from "@budibase/pro" import { enrichPluginURLs } from "../../utilities/plugins" const URL_REGEX_SLASH = /\/|\\/g @@ -172,16 +171,16 @@ export const fetch = async (ctx: any) => { const all = ctx.query && ctx.query.status === AppStatus.ALL const apps = await getAllApps({ dev, all }) + const appIds = apps + .filter((app: any) => app.status === "development") + .map((app: any) => app.appId) // get the locks for all the dev apps if (dev || all) { - const locks = await getAllLocks() + const locks = await getLocksById(appIds) for (let app of apps) { - if (app.status !== "development") { - continue - } - const lock = locks.find((lock: any) => lock.appId === app.appId) + const lock = locks[app.appId] if (lock) { - app.lockedBy = lock.user + app.lockedBy = lock } else { // make sure its definitely not present delete app.lockedBy diff --git a/packages/server/src/utilities/redis.js b/packages/server/src/utilities/redis.js index 4eddca6e4a..b39b7cae55 100644 --- a/packages/server/src/utilities/redis.js +++ b/packages/server/src/utilities/redis.js @@ -34,12 +34,8 @@ exports.doesUserHaveLock = async (devAppId, user) => { return expected === userId } -exports.getAllLocks = async () => { - const locks = await devAppClient.scan() - return locks.map(lock => ({ - appId: lock.key, - user: lock.value, - })) +exports.getLocksById = async appIds => { + return await devAppClient.bulkGet(appIds) } exports.updateLock = async (devAppId, user) => { From 39410a07af0e959cf7f3776c2caf8260e5f8fedd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 7 Oct 2022 16:24:04 +0100 Subject: [PATCH 030/101] Fixing test mocks. --- packages/server/src/api/routes/tests/application.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index dcfc2c6d9b..f62665d184 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -1,7 +1,7 @@ jest.mock("../../../utilities/redis", () => ({ init: jest.fn(), - getAllLocks: () => { - return [] + getLocksById: () => { + return {} }, doesUserHaveLock: () => { return true From 3616bfd297d50612189c7608cc6719b431889b40 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 7 Oct 2022 16:45:27 +0000 Subject: [PATCH 031/101] v2.0.24-alpha.3 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index f35cc59762..a47d8fe604 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 2d7cfece80..8df964912e 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "2.0.24-alpha.2", + "@budibase/types": "2.0.24-alpha.3", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index c178a8c331..0583237a45 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "2.0.24-alpha.2", + "@budibase/string-templates": "2.0.24-alpha.3", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 3f2219482e..900b33fb88 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.0.24-alpha.2", - "@budibase/client": "2.0.24-alpha.2", - "@budibase/frontend-core": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", + "@budibase/bbui": "2.0.24-alpha.3", + "@budibase/client": "2.0.24-alpha.3", + "@budibase/frontend-core": "2.0.24-alpha.3", + "@budibase/string-templates": "2.0.24-alpha.3", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index ebf28a4b40..18eff4ce2a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", - "@budibase/types": "2.0.24-alpha.2", + "@budibase/backend-core": "2.0.24-alpha.3", + "@budibase/string-templates": "2.0.24-alpha.3", + "@budibase/types": "2.0.24-alpha.3", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 341479676e..1676a722ba 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.0.24-alpha.2", - "@budibase/frontend-core": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", + "@budibase/bbui": "2.0.24-alpha.3", + "@budibase/frontend-core": "2.0.24-alpha.3", + "@budibase/string-templates": "2.0.24-alpha.3", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4c22e3f219..7d3355f34e 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.0.24-alpha.2", + "@budibase/bbui": "2.0.24-alpha.3", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index f7f7ab6d6a..53e89077a7 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index ec34d9024b..c798551278 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.0.24-alpha.2", - "@budibase/client": "2.0.24-alpha.2", + "@budibase/backend-core": "2.0.24-alpha.3", + "@budibase/client": "2.0.24-alpha.3", "@budibase/pro": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", - "@budibase/types": "2.0.24-alpha.2", + "@budibase/string-templates": "2.0.24-alpha.3", + "@budibase/types": "2.0.24-alpha.3", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index e8679e5a06..44c8fc560e 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index c9c4607407..f5a83e2c3b 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index f9e4929cde..39593000d1 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.24-alpha.2", + "version": "2.0.24-alpha.3", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.0.24-alpha.2", + "@budibase/backend-core": "2.0.24-alpha.3", "@budibase/pro": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", - "@budibase/types": "2.0.24-alpha.2", + "@budibase/string-templates": "2.0.24-alpha.3", + "@budibase/types": "2.0.24-alpha.3", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From ba50fa56b64faa482bc352ec2602223de02197b5 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 7 Oct 2022 16:48:36 +0000 Subject: [PATCH 032/101] Update pro version to 2.0.24-alpha.3 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index c798551278..2f2409850e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "2.0.24-alpha.3", "@budibase/client": "2.0.24-alpha.3", - "@budibase/pro": "2.0.24-alpha.2", + "@budibase/pro": "2.0.24-alpha.3", "@budibase/string-templates": "2.0.24-alpha.3", "@budibase/types": "2.0.24-alpha.3", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 7cfb82a072..18ab07c17f 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24-alpha.2.tgz#8677856bc4230c30c209ca3cf4abd6d72ac1712c" - integrity sha512-TVf6al/KtOdilwWnaU2ijhUZEZ9c2WNEIN03HZDjHTKM3ur8gNg5znma2Vd0YiBLmnA3a+cQqyu0UkJKxezbVg== +"@budibase/backend-core@2.0.24-alpha.3": + version "2.0.24-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24-alpha.3.tgz#49a92082d1ca6bed0eb82519a6c02d7d14ead751" + integrity sha512-Zm/ddRDMzMuXCoEXZa0CzA/B1SnpQ+yjZCNAPH7Y4yYTIKfeE/DQ6COLlUixQxVMAUN1L5+GXML+px6fRigA5w== dependencies: - "@budibase/types" "2.0.24-alpha.2" + "@budibase/types" "2.0.24-alpha.3" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24-alpha.2.tgz#023b943d9b457f363b132e841bb6daa306cc2bbe" - integrity sha512-uAW93A0r2SyUl40gTUJoyAYGu0tgh9CikFFf5EddJytkOd+iAKQJbTjbpUTdBsyivBeZTNIVKCfIf8wxx4XYXQ== +"@budibase/pro@2.0.24-alpha.3": + version "2.0.24-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24-alpha.3.tgz#16605029663b07d0e3bcf0c08893878b542b6752" + integrity sha512-gBdqJzvtEAMJkhB2YTAyVYKoCANA25F1wT/K7kQtTOy5LrjDmcXah7CZoetfZQGbsKxUYEAZwJzxLSXmIxvWmQ== dependencies: - "@budibase/backend-core" "2.0.24-alpha.2" - "@budibase/types" "2.0.24-alpha.2" + "@budibase/backend-core" "2.0.24-alpha.3" + "@budibase/types" "2.0.24-alpha.3" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24-alpha.2.tgz#3945eb9b869ff7a91fe32cf443a0aa7daf6c7a8d" - integrity sha512-Bna7JkI2UTPrWH63xrIjfBjgiD5T09n4ITcxxgPw3W81VZ6BJnqiBLCLP6wM5c5xIhKag3VAHQX9n8OREANYAA== +"@budibase/types@2.0.24-alpha.3": + version "2.0.24-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24-alpha.3.tgz#84eec5f991a2cfaf48d968b07a075b343f847289" + integrity sha512-f9PhtqzmqPI76ITXttuvxsvqMUJtkrDYf/4MHlI2v5ssNL9r0C/hbQEXllff3L3JqViEHWxkKFmfvfnDTV8rRQ== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index 39593000d1..46927f8725 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "2.0.24-alpha.3", - "@budibase/pro": "2.0.24-alpha.2", + "@budibase/pro": "2.0.24-alpha.3", "@budibase/string-templates": "2.0.24-alpha.3", "@budibase/types": "2.0.24-alpha.3", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index e33d793690..12086ae67d 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24-alpha.2.tgz#8677856bc4230c30c209ca3cf4abd6d72ac1712c" - integrity sha512-TVf6al/KtOdilwWnaU2ijhUZEZ9c2WNEIN03HZDjHTKM3ur8gNg5znma2Vd0YiBLmnA3a+cQqyu0UkJKxezbVg== +"@budibase/backend-core@2.0.24-alpha.3": + version "2.0.24-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24-alpha.3.tgz#49a92082d1ca6bed0eb82519a6c02d7d14ead751" + integrity sha512-Zm/ddRDMzMuXCoEXZa0CzA/B1SnpQ+yjZCNAPH7Y4yYTIKfeE/DQ6COLlUixQxVMAUN1L5+GXML+px6fRigA5w== dependencies: - "@budibase/types" "2.0.24-alpha.2" + "@budibase/types" "2.0.24-alpha.3" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24-alpha.2.tgz#023b943d9b457f363b132e841bb6daa306cc2bbe" - integrity sha512-uAW93A0r2SyUl40gTUJoyAYGu0tgh9CikFFf5EddJytkOd+iAKQJbTjbpUTdBsyivBeZTNIVKCfIf8wxx4XYXQ== +"@budibase/pro@2.0.24-alpha.3": + version "2.0.24-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24-alpha.3.tgz#16605029663b07d0e3bcf0c08893878b542b6752" + integrity sha512-gBdqJzvtEAMJkhB2YTAyVYKoCANA25F1wT/K7kQtTOy5LrjDmcXah7CZoetfZQGbsKxUYEAZwJzxLSXmIxvWmQ== dependencies: - "@budibase/backend-core" "2.0.24-alpha.2" - "@budibase/types" "2.0.24-alpha.2" + "@budibase/backend-core" "2.0.24-alpha.3" + "@budibase/types" "2.0.24-alpha.3" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24-alpha.2.tgz#3945eb9b869ff7a91fe32cf443a0aa7daf6c7a8d" - integrity sha512-Bna7JkI2UTPrWH63xrIjfBjgiD5T09n4ITcxxgPw3W81VZ6BJnqiBLCLP6wM5c5xIhKag3VAHQX9n8OREANYAA== +"@budibase/types@2.0.24-alpha.3": + version "2.0.24-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24-alpha.3.tgz#84eec5f991a2cfaf48d968b07a075b343f847289" + integrity sha512-f9PhtqzmqPI76ITXttuvxsvqMUJtkrDYf/4MHlI2v5ssNL9r0C/hbQEXllff3L3JqViEHWxkKFmfvfnDTV8rRQ== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From 4e04fc84b2907e6ea03299d88d0c93d80126be2d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 7 Oct 2022 18:04:18 +0100 Subject: [PATCH 033/101] only prevent rollback in cypress env --- packages/server/src/api/controllers/dev.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index 8438175ca8..e5582be5b3 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -103,7 +103,7 @@ exports.revert = async ctx => { target: appId, }) try { - if (!env.isTest()) { + if (!env.isCypress()) { // in-memory db stalls on rollback await replication.rollback() } From 197d195cb464ee44f7e515e64a575404262f053b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 7 Oct 2022 16:05:01 +0100 Subject: [PATCH 034/101] Switching from scan for app locks to mget - which is a fast O(N) operation that only retrieves what we need. --- packages/backend-core/src/redis/index.ts | 25 +++++++++++++++++++ .../server/src/api/controllers/application.ts | 17 ++++++------- packages/server/src/utilities/redis.js | 8 ++---- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 206110366f..62e718d8ad 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -214,6 +214,31 @@ export = class RedisWrapper { } } + async bulkGet(keys: string[]) { + const db = this._db + const prefixedKeys = keys.map(key => addDbPrefix(db, key)) + let response = await this.getClient().mget(prefixedKeys) + if (Array.isArray(response)) { + let final: any = {} + let count = 0 + for (let result of response) { + if (result) { + let parsed + try { + parsed = JSON.parse(result) + } catch (err) { + parsed = result + } + final[keys[count]] = parsed + } + count++ + } + return final + } else { + throw new Error(`Invalid response: ${response}`) + } + } + async store(key: string, value: any, expirySeconds: number | null = null) { const db = this._db if (typeof value === "object") { diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index b6377a61a2..e8af2f54f8 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -32,7 +32,7 @@ const { import { USERS_TABLE_SCHEMA } from "../../constants" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { clientLibraryPath, stringToReadStream } from "../../utilities" -import { getAllLocks } from "../../utilities/redis" +import { getLocksById } from "../../utilities/redis" import { updateClientLibrary, backupClientLibrary, @@ -45,11 +45,10 @@ import { cleanupAutomations } from "../../automations/utils" import { context } from "@budibase/backend-core" import { checkAppMetadata } from "../../automations/logging" import { getUniqueRows } from "../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" +import { quotas, groups } from "@budibase/pro" import { errors, events, migrations } from "@budibase/backend-core" import { App, Layout, Screen, MigrationType } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" -import { groups } from "@budibase/pro" import { enrichPluginURLs } from "../../utilities/plugins" const URL_REGEX_SLASH = /\/|\\/g @@ -172,16 +171,16 @@ export const fetch = async (ctx: any) => { const all = ctx.query && ctx.query.status === AppStatus.ALL const apps = await getAllApps({ dev, all }) + const appIds = apps + .filter((app: any) => app.status === "development") + .map((app: any) => app.appId) // get the locks for all the dev apps if (dev || all) { - const locks = await getAllLocks() + const locks = await getLocksById(appIds) for (let app of apps) { - if (app.status !== "development") { - continue - } - const lock = locks.find((lock: any) => lock.appId === app.appId) + const lock = locks[app.appId] if (lock) { - app.lockedBy = lock.user + app.lockedBy = lock } else { // make sure its definitely not present delete app.lockedBy diff --git a/packages/server/src/utilities/redis.js b/packages/server/src/utilities/redis.js index 4eddca6e4a..b39b7cae55 100644 --- a/packages/server/src/utilities/redis.js +++ b/packages/server/src/utilities/redis.js @@ -34,12 +34,8 @@ exports.doesUserHaveLock = async (devAppId, user) => { return expected === userId } -exports.getAllLocks = async () => { - const locks = await devAppClient.scan() - return locks.map(lock => ({ - appId: lock.key, - user: lock.value, - })) +exports.getLocksById = async appIds => { + return await devAppClient.bulkGet(appIds) } exports.updateLock = async (devAppId, user) => { From d80c966cf137a0e6fd5a59c93edeb5d614c61009 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 7 Oct 2022 16:24:04 +0100 Subject: [PATCH 035/101] Fixing test mocks. --- packages/server/src/api/routes/tests/application.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index dcfc2c6d9b..f62665d184 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -1,7 +1,7 @@ jest.mock("../../../utilities/redis", () => ({ init: jest.fn(), - getAllLocks: () => { - return [] + getLocksById: () => { + return {} }, doesUserHaveLock: () => { return true From 970ba0315e5059cf7a934e0542b3584e8167fefe Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 7 Oct 2022 18:05:58 +0100 Subject: [PATCH 036/101] Add tests for revert endpoint --- .../TestConfiguration/applications.ts | 13 ++++- .../config/internal-api/fixtures/screens.ts | 34 +++++++++++++ .../internal-api/applications/create.spec.ts | 51 +++++++++++++++---- 3 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 qa-core/src/config/internal-api/fixtures/screens.ts diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 3b577abf5e..8c88819ef5 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -29,7 +29,7 @@ export default class AppApi { return [response, json] } - async publish(): Promise<[Response, string]> { + async publish(): Promise<[Response, any]> { const response = await this.api.post("/deploy") const json = await response.json() return [response, json] @@ -82,6 +82,17 @@ export default class AppApi { const json = await response.json() return [response, json] } + async addScreentoApp(body: any): Promise<[Response, Application]> { + const response = await this.api.post(`/screens`, { body }) + const json = await response.json() + return [response, json] + } + + async getRoutes(): Promise<[Response, any]> { + const response = await this.api.get(`/routing`) + const json = await response.json() + return [response, json] + } } diff --git a/qa-core/src/config/internal-api/fixtures/screens.ts b/qa-core/src/config/internal-api/fixtures/screens.ts new file mode 100644 index 0000000000..2a2cc1eada --- /dev/null +++ b/qa-core/src/config/internal-api/fixtures/screens.ts @@ -0,0 +1,34 @@ +import generator from "../../generator" + +const randomId = generator.guid() + +const generateScreen = (): any => ({ + showNavigation: true, + width: "Large", + props: { + _id: randomId, + _component: + "@budibase/standard-components/container", + _styles: { + normal: {}, + hover: {}, + active: {}, + selected: {} + }, + _children: [], + _instanceName: "New Screen", + direction: "column", + hAlign: "stretch", + vAlign: "top", + size: "grow", + gap: "M" + }, routing: { + route: "/test", + roleId: "BASIC", + homeScreen: false + }, + name: randomId, + template: "createFromScratch" +}) + +export default generateScreen diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index b447a95459..4b9a21319d 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -4,6 +4,7 @@ import { db } from "@budibase/backend-core" import InternalAPIClient from "../../../config/internal-api/TestConfiguration/InternalAPIClient" import generateApp from "../../../config/internal-api/fixtures/applications" import generator from "../../../config/generator" +import generateScreen from "../../../config/internal-api/fixtures/screens" describe("Internal API - /applications endpoints", () => { const api = new InternalAPIClient() @@ -118,29 +119,57 @@ describe("Internal API - /applications endpoints", () => { const [response, app] = await config.applications.create(generateApp()) expect(response).toHaveStatusCode(200) expect(app.appId).toBeDefined() - - }) - - - it.skip("POST - Revert Changes", async () => { - const [response, app] = await config.applications.create(generateApp()) - expect(response).toHaveStatusCode(200) - expect(app.appId).toBeDefined() - - // publish app - await config.applications.publish() + config.applications.api.appId = app.appId const [updateResponse, updatedApp] = await config.applications.update(app.appId ? app.appId : "", { name: generator.word(), }) expect(updateResponse).toHaveStatusCode(200) expect(updatedApp.name).not.toEqual(app.name) + }) + + // Skip this test because of the if line 44 in InternalAPIClient.ts + it.skip("POST - Revert Changes without changes", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + const [revertResponse, revert] = await config.applications.revert(app.appId ? app.appId : "") + expect(revertResponse).toHaveStatusCode(400) + expect(revert).toEqual({ + message: "There is no version to revert to", + }) + }) + + it("POST - Revert Changes", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + // publish app + const [publishResponse, publish] = await config.applications.publish() + expect(publishResponse).toHaveStatusCode(200) + expect(publish.status).toEqual("SUCCESS") + + // Change/add component to the app + const [screenResponse, screen] = await config.applications.addScreentoApp(generateScreen()) + expect(screenResponse).toHaveStatusCode(200) + expect(screen._id).toBeDefined() + const [revertResponse, revert] = await config.applications.revert(app.appId ? app.appId : "") expect(revertResponse).toHaveStatusCode(200) expect(revert).toEqual({ message: "App reverted successfully." }) + + // Check screen is removed + const [routesResponse, routes] = await config.applications.getRoutes() + expect(routesResponse).toHaveStatusCode(200) + expect(routes.routes["/test"]).toBeUndefined() + }) it("DELETE - Delete an application", async () => { From 15723e38d81bca73c04fa9b53ef20195c3e650ae Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 7 Oct 2022 17:16:16 +0000 Subject: [PATCH 037/101] v2.0.24 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index bc62b3c368..b2ce7d106f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.23", + "version": "2.0.24", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index a9c1e3eb59..163cf61187 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.0.23", + "@budibase/types": "^2.0.24", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 3ca5585e34..86b1990e80 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.23", + "version": "2.0.24", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.0.23", + "@budibase/string-templates": "^2.0.24", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 6f289d3433..07f53bcd05 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.23", + "version": "2.0.24", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.0.23", - "@budibase/client": "^2.0.23", - "@budibase/frontend-core": "^2.0.23", - "@budibase/string-templates": "^2.0.23", + "@budibase/bbui": "^2.0.24", + "@budibase/client": "^2.0.24", + "@budibase/frontend-core": "^2.0.24", + "@budibase/string-templates": "^2.0.24", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index e054839f25..a4d98d9daf 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.0.23", - "@budibase/string-templates": "^2.0.23", - "@budibase/types": "^2.0.23", + "@budibase/backend-core": "^2.0.24", + "@budibase/string-templates": "^2.0.24", + "@budibase/types": "^2.0.24", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index f22039ae4f..9c1222c0c3 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.0.23", + "version": "2.0.24", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.0.23", - "@budibase/frontend-core": "^2.0.23", - "@budibase/string-templates": "^2.0.23", + "@budibase/bbui": "^2.0.24", + "@budibase/frontend-core": "^2.0.24", + "@budibase/string-templates": "^2.0.24", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 3942ad481b..d028ae43c7 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.0.23", + "@budibase/bbui": "^2.0.24", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 59bd24e4ca..ebfd81e7b2 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index e9eca4136d..8d085e2ab7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.0.23", - "@budibase/client": "^2.0.23", + "@budibase/backend-core": "^2.0.24", + "@budibase/client": "^2.0.24", "@budibase/pro": "2.0.23", - "@budibase/string-templates": "^2.0.23", - "@budibase/types": "^2.0.23", + "@budibase/string-templates": "^2.0.24", + "@budibase/types": "^2.0.24", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 651034e3fd..23a7109cd4 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.23", + "version": "2.0.24", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 8c1b53a33b..a36cabc46e 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 9763d42abf..c2e690fe01 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.23", + "version": "2.0.24", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.0.23", + "@budibase/backend-core": "^2.0.24", "@budibase/pro": "2.0.23", - "@budibase/string-templates": "^2.0.23", - "@budibase/types": "^2.0.23", + "@budibase/string-templates": "^2.0.24", + "@budibase/types": "^2.0.24", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From a0cc4a1b2194c60da44971a568b2e852b6855a12 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 7 Oct 2022 17:19:31 +0000 Subject: [PATCH 038/101] Update pro version to 2.0.24 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 8d085e2ab7..36e723d0e9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.0.24", "@budibase/client": "^2.0.24", - "@budibase/pro": "2.0.23", + "@budibase/pro": "2.0.24", "@budibase/string-templates": "^2.0.24", "@budibase/types": "^2.0.24", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index d70aabbcf7..e005d36235 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.23": - version "2.0.23" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.23.tgz#915d4ea2e78547795894b2aebd81b8a8c9b568db" - integrity sha512-wovJF+kS/vVs+cR6lHieEEs99vmXsOt5qKjMAkkMUVXmWBWPYQaLppz+dxTV9xXd8Ht9yFhnkbOo7QJxFE8+fA== +"@budibase/backend-core@2.0.24": + version "2.0.24" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24.tgz#305be50481095441aeb663add4271c06b0d98435" + integrity sha512-yq/TwV1GWGeAZgn8FrSVAk0tNXFQLvduoKrwy4f2EsR26R5y6uxp2B0igP0sKMvUJPjTNPBONJM0MdYx9JBSmA== dependencies: - "@budibase/types" "^2.0.23" + "@budibase/types" "^2.0.24" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.23": - version "2.0.23" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.23.tgz#215759f07470d37b6571f54ad80d7fb38a2f902e" - integrity sha512-RP7lS076VP3W+9AI9d/ylM6k1YBotGlqkBrshgzX/pl1e2O9eg0MmG2/dP6X+w4LzGv2ruC7lfwyf3MgWgYFyQ== +"@budibase/pro@2.0.24": + version "2.0.24" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24.tgz#d678753a28fc44f4f476120c8fb7b01365b5e18b" + integrity sha512-j/wsd1RCvTGodRAvD9GJZ9JDivGxMuFDAmrQlvcq5ffrqX2VDm0dSP/F+8YZFck4hdSmcL5VeqxZCPh2kx74vA== dependencies: - "@budibase/backend-core" "2.0.23" - "@budibase/types" "2.0.23" + "@budibase/backend-core" "2.0.24" + "@budibase/types" "2.0.24" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.23", "@budibase/types@^2.0.23": - version "2.0.23" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.23.tgz#2467342b9c48c965319182ef4cbb7166c8bf1e43" - integrity sha512-Oyq9tVwV+zl38III1or4Cr8VOKemMv3L/o95CBXdyZRDmmVfdzQgJ7AByUMOj9quEe6IjctdkOI4ssE8Fds9Dw== +"@budibase/types@2.0.24", "@budibase/types@^2.0.24": + version "2.0.24" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24.tgz#8ac0615b17d3f46809b10897b3217a86b6d2a62e" + integrity sha512-Lo1NGnu+4PH0OLjwxRU5dqEAADEBOCkikQtAjapVFIZWcBit0ZC/bXTs6meIaey6JqqEKOlDR5V+VwrzcmdUIA== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index c2e690fe01..ad7db4639b 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.0.24", - "@budibase/pro": "2.0.23", + "@budibase/pro": "2.0.24", "@budibase/string-templates": "^2.0.24", "@budibase/types": "^2.0.24", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index f885ab690e..e0d3c1cb1c 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.23": - version "2.0.23" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.23.tgz#915d4ea2e78547795894b2aebd81b8a8c9b568db" - integrity sha512-wovJF+kS/vVs+cR6lHieEEs99vmXsOt5qKjMAkkMUVXmWBWPYQaLppz+dxTV9xXd8Ht9yFhnkbOo7QJxFE8+fA== +"@budibase/backend-core@2.0.24": + version "2.0.24" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24.tgz#305be50481095441aeb663add4271c06b0d98435" + integrity sha512-yq/TwV1GWGeAZgn8FrSVAk0tNXFQLvduoKrwy4f2EsR26R5y6uxp2B0igP0sKMvUJPjTNPBONJM0MdYx9JBSmA== dependencies: - "@budibase/types" "^2.0.23" + "@budibase/types" "^2.0.24" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.23": - version "2.0.23" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.23.tgz#215759f07470d37b6571f54ad80d7fb38a2f902e" - integrity sha512-RP7lS076VP3W+9AI9d/ylM6k1YBotGlqkBrshgzX/pl1e2O9eg0MmG2/dP6X+w4LzGv2ruC7lfwyf3MgWgYFyQ== +"@budibase/pro@2.0.24": + version "2.0.24" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24.tgz#d678753a28fc44f4f476120c8fb7b01365b5e18b" + integrity sha512-j/wsd1RCvTGodRAvD9GJZ9JDivGxMuFDAmrQlvcq5ffrqX2VDm0dSP/F+8YZFck4hdSmcL5VeqxZCPh2kx74vA== dependencies: - "@budibase/backend-core" "2.0.23" - "@budibase/types" "2.0.23" + "@budibase/backend-core" "2.0.24" + "@budibase/types" "2.0.24" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.23", "@budibase/types@^2.0.23": - version "2.0.23" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.23.tgz#2467342b9c48c965319182ef4cbb7166c8bf1e43" - integrity sha512-Oyq9tVwV+zl38III1or4Cr8VOKemMv3L/o95CBXdyZRDmmVfdzQgJ7AByUMOj9quEe6IjctdkOI4ssE8Fds9Dw== +"@budibase/types@2.0.24", "@budibase/types@^2.0.24": + version "2.0.24" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24.tgz#8ac0615b17d3f46809b10897b3217a86b6d2a62e" + integrity sha512-Lo1NGnu+4PH0OLjwxRU5dqEAADEBOCkikQtAjapVFIZWcBit0ZC/bXTs6meIaey6JqqEKOlDR5V+VwrzcmdUIA== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From 052bc0aeedcff577e6cf148c246a34863d9c9ac5 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Sun, 9 Oct 2022 19:31:35 +0100 Subject: [PATCH 039/101] ensure correct revert endpoint being hit, update call to revert --- packages/server/src/api/controllers/dev.js | 2 +- packages/server/src/api/controllers/table/internal.ts | 3 ++- .../config/internal-api/TestConfiguration/applications.ts | 6 +++--- qa-core/src/tests/internal-api/applications/create.spec.ts | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index e5582be5b3..c8f134756b 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -103,7 +103,7 @@ exports.revert = async ctx => { target: appId, }) try { - if (!env.isCypress()) { + if (env.COUCH_DB_URL) { // in-memory db stalls on rollback await replication.rollback() } diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 22c9b6dc55..7e55c71aea 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -18,6 +18,7 @@ import { Table } from "@budibase/types" import { quotas } from "@budibase/pro" import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" +import env from "../../../environment" function checkAutoColumns(table: Table, oldTable: Table) { if (!table.schema) { @@ -167,7 +168,7 @@ export async function destroy(ctx: any) { await db.remove(tableToDelete) // remove table search index - if (!isTest()) { + if (!isTest() || env.COUCH_DB_URL) { const currentIndexes = await db.getIndexes() const existingIndex = currentIndexes.indexes.find( (existing: any) => existing.name === `search:${ctx.params.tableId}` diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 8c88819ef5..0372e50b0b 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -59,8 +59,8 @@ export default class AppApi { return [response, json] } - async revert(appId: string): Promise<[Response, Application]> { - const response = await this.api.post(`/applications/${appId}/client/revert`) + async revert(appId: string): Promise<[Response, { message: string }]> { + const response = await this.api.post(`/dev/${appId}/revert`) const json = await response.json() return [response, json] } @@ -82,8 +82,8 @@ export default class AppApi { const json = await response.json() return [response, json] } - async addScreentoApp(body: any): Promise<[Response, Application]> { + async addScreentoApp(body: any): Promise<[Response, Application]> { const response = await this.api.post(`/screens`, { body }) const json = await response.json() return [response, json] diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index 4b9a21319d..fc7901cc6f 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -158,11 +158,11 @@ describe("Internal API - /applications endpoints", () => { expect(screenResponse).toHaveStatusCode(200) expect(screen._id).toBeDefined() - - const [revertResponse, revert] = await config.applications.revert(app.appId ? app.appId : "") + // // Revert the app to published state + const [revertResponse, revert] = await config.applications.revert(app.appId as string) expect(revertResponse).toHaveStatusCode(200) expect(revert).toEqual({ - message: "App reverted successfully." + message: "Reverted changes successfully." }) // Check screen is removed From 84685d3340968809a692b5a99194919c16c0188e Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 10 Oct 2022 08:21:17 +0100 Subject: [PATCH 040/101] Add locking framework --- packages/backend-core/src/index.ts | 1 + packages/backend-core/src/pkg/redis.ts | 2 + packages/backend-core/src/redis/init.js | 25 ++++---- packages/backend-core/src/redis/redlock.ts | 74 ++++++++++++++++++++-- packages/backend-core/src/redis/utils.js | 1 + packages/server/src/migrations/index.ts | 47 +++++--------- packages/types/src/sdk/index.ts | 1 + packages/types/src/sdk/locks.ts | 31 +++++++++ packages/worker/src/migrations/index.ts | 47 +++++--------- 9 files changed, 149 insertions(+), 80 deletions(-) create mode 100644 packages/types/src/sdk/locks.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index cb1e6d1c82..275a954a78 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { locks, migrations } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index bae566b42e..0c374dd105 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -7,3 +7,4 @@ export * from "./datasources" export * from "./search" export * from "./koa" export * from "./auth" +export * from "./locks" diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts new file mode 100644 index 0000000000..3aa067bea1 --- /dev/null +++ b/packages/types/src/sdk/locks.ts @@ -0,0 +1,31 @@ +export enum LockType { + /** + * If this lock is already held the attempted operation will not be performed. + * No retries will take place and no error will be thrown. + */ + TRY_ONCE = "try_once", +} + +export enum LockName { + MIGRATIONS = "migrations", + TRIGGER_QUOTA = "trigger_quota", +} + +export interface LockOptions { + /** + * The lock type determines which client to use + */ + type: LockType + /** + * The name for the lock + */ + name: LockName + /** + * The ttl to auto-expire the lock if not unlocked manually + */ + ttl: number + /** + * The suffix to add to the lock name for additional uniqueness + */ + nameSuffix?: string +} diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts index 6900596216..19ef076a52 100644 --- a/packages/worker/src/migrations/index.ts +++ b/packages/worker/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { migrations, locks } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } From 5838eb70b4a6bbfacb4b7d3b7fdb3e45e1d69b5c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 10 Oct 2022 09:02:17 +0100 Subject: [PATCH 041/101] turn on gsheets --- packages/server/src/api/controllers/integration.js | 8 -------- packages/server/src/integrations/index.ts | 5 +---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/server/src/api/controllers/integration.js b/packages/server/src/api/controllers/integration.js index 3d1643601b..2f11ec19ed 100644 --- a/packages/server/src/api/controllers/integration.js +++ b/packages/server/src/api/controllers/integration.js @@ -1,17 +1,9 @@ const { getDefinitions } = require("../../integrations") -const { SourceName } = require("@budibase/types") -const googlesheets = require("../../integrations/googlesheets") -const { featureFlags } = require("@budibase/backend-core") exports.fetch = async function (ctx) { ctx.status = 200 const defs = await getDefinitions() - // for google sheets integration google verification - if (featureFlags.isEnabled(featureFlags.TenantFeatureFlag.GOOGLE_SHEETS)) { - defs[SourceName.GOOGLE_SHEETS] = googlesheets.schema - } - ctx.body = defs } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 240f68ed91..c926aeb992 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -33,6 +33,7 @@ const DEFINITIONS: { [key: string]: Integration } = { [SourceName.ARANGODB]: arangodb.schema, [SourceName.REST]: rest.schema, [SourceName.FIRESTORE]: firebase.schema, + [SourceName.GOOGLE_SHEETS]: googlesheets.schema, [SourceName.REDIS]: redis.schema, [SourceName.SNOWFLAKE]: snowflake.schema, } @@ -66,10 +67,6 @@ if ( INTEGRATIONS[SourceName.ORACLE] = oracle.integration } -if (environment.SELF_HOSTED) { - DEFINITIONS[SourceName.GOOGLE_SHEETS] = googlesheets.schema -} - module.exports = { getDefinitions: async () => { const pluginSchemas: { [key: string]: Integration } = {} From 43befea5e9eee81b59ca6c4e9e3f4e15121e3db4 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 10 Oct 2022 10:39:34 +0100 Subject: [PATCH 042/101] Code review updates --- packages/client/manifest.json | 13 +++++++++---- .../components/app/{ => forms}/CodeScanner.svelte | 12 ++++++------ .../components/app/forms/CodeScannerField.svelte | 6 ++++-- 3 files changed, 19 insertions(+), 12 deletions(-) rename packages/client/src/components/app/{ => forms}/CodeScanner.svelte (94%) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 0c114ab703..f19fbd7c64 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3163,7 +3163,6 @@ "styles": [ "size" ], - "draggable": true, "illegalChildren": [ "section" ], @@ -3181,7 +3180,7 @@ }, { "type": "text", - "label": "Button", + "label": "Button text", "key": "scanButtonText" }, { @@ -4450,7 +4449,9 @@ "formblock": { "name": "Form Block", "icon": "Form", - "styles": ["size"], + "styles": [ + "size" + ], "block": true, "info": "Form blocks are only compatible with internal or SQL tables", "settings": [ @@ -4458,7 +4459,11 @@ "type": "select", "label": "Type", "key": "actionType", - "options": ["Create", "Update", "View"], + "options": [ + "Create", + "Update", + "View" + ], "defaultValue": "Create" }, { diff --git a/packages/client/src/components/app/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte similarity index 94% rename from packages/client/src/components/app/CodeScanner.svelte rename to packages/client/src/components/app/forms/CodeScanner.svelte index 3215875faa..8728e707dd 100644 --- a/packages/client/src/components/app/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -6,7 +6,7 @@ export let value export let disabled = false export let allowManualEntry = false - export let scanButtonText = "Scan Code" + export let scanButtonText = "Scan code" import { createEventDispatcher } from "svelte" const dispatch = createEventDispatcher() @@ -161,17 +161,17 @@