From b1167fd444601577f343d5a8dae2d21a31484e21 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Mar 2022 11:36:30 +0000 Subject: [PATCH 1/6] Adding public API rate limiting, with env variable option, defaults to 120 requests per minute. --- packages/server/package.json | 1 + .../server/src/api/routes/public/index.ts | 13 +++++ packages/server/src/environment.js | 1 + packages/server/yarn.lock | 47 ++++++++++--------- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 43cf9b9dbe..9846d5752c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -111,6 +111,7 @@ "koa-send": "5.0.0", "koa-session": "5.12.0", "koa-static": "5.0.0", + "koa2-ratelimit": "^1.1.0", "lodash": "4.17.21", "memorystream": "^0.3.1", "mongodb": "3.6.3", diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 7e48e1a6a7..438ef2c1e9 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -8,18 +8,31 @@ import authorized from "../../../middleware/authorized" import { paramResource, paramSubResource } from "../../../middleware/resourceId" import { CtxFn } from "./utils/Endpoint" import mapperMiddleware from "./middleware/mapper" +import env from "../../../environment" +// below imports don't have declaration files const Router = require("@koa/router") +const RateLimit = require("koa2-ratelimit").RateLimit const { PermissionLevels, PermissionTypes, } = require("@budibase/backend-core/permissions") const PREFIX = "/api/public/v1" +const DEFAULT_API_LIMITING = 120 + +// rate limiting, allows for 2 requests per second +const limiter = RateLimit.middleware({ + interval: { min: 1 }, + // per ip, per interval + max: env.API_RATE_LIMITING || DEFAULT_API_LIMITING, +}) const publicRouter = new Router({ prefix: PREFIX, }) +publicRouter.use(limiter) + function addMiddleware( endpoints: any, middleware: CtxFn, diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 7ed8b16b6f..e3e321b795 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -45,6 +45,7 @@ module.exports = { INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, + API_RATE_LIMITING: process.env.API_RATE_LIMITING, // environment NODE_ENV: process.env.NODE_ENV, JEST_WORKER_ID: process.env.JEST_WORKER_ID, diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index b91c23776a..ce4213c322 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -995,10 +995,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@^1.0.79-alpha.5": - version "1.0.79-alpha.5" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.79-alpha.5.tgz#dac73ccfcd2e6e63415cde6d76e4bf09043dc6b6" - integrity sha512-m7/z55fp+EYVYEAyuQ2K//AcrfgzLBcR4EVjP+rrmbIhGFbLV2ASl5IBg9bcAYp0z2m816skJrY2asx9raWhgw== +"@budibase/backend-core@^1.0.79-alpha.7": + version "1.0.79-alpha.7" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.79-alpha.7.tgz#57e8319118b425cc228173d1ec8bf19843e1a417" + integrity sha512-Ao4dR6zwnJa4mYiRyl5lULYm+wsYwOi3sxDIjt5vmurqUL2JTuVUrGt1MAq1N6K11xQpbQQJAFvgfxBgn9aEMg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1068,7 +1068,7 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/bbui@^1.0.79-alpha.5": +"@budibase/bbui@^1.0.79-alpha.7": version "1.58.13" resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.58.13.tgz#59df9c73def2d81c75dcbd2266c52c19db88dbd7" integrity sha512-Zk6CKXdBfKsTVzA1Xs5++shdSSZLfphVpZuKVbjfzkgtuhyH7ruucexuSHEpFsxjW5rEKgKIBoRFzCK5vPvN0w== @@ -1080,14 +1080,14 @@ svelte-portal "^1.0.0" turndown "^7.0.0" -"@budibase/client@^1.0.79-alpha.5": - version "1.0.79-alpha.5" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.79-alpha.5.tgz#d729858b10e6cd2a506fb63364a0e7ab3149780e" - integrity sha512-OrBErU97YL67GggsLmcD46AUElSgtyFjZdCXi++3s4zaZYZxT4Ix2iFMrnslcpF87bv8xyiSt3vsyCPGKCU5wQ== +"@budibase/client@^1.0.79-alpha.7": + version "1.0.79-alpha.7" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.79-alpha.7.tgz#d225ac5bd68fa9ecb81114791e6d931246da9637" + integrity sha512-7faCcIlXyOf660PwpOMCt9/X2liiTuCsPGUpLsJQu2j9CcVZ5vV+au0CX7dtqewtPNuIL0mF3G7ZOpBTvXx4NQ== dependencies: - "@budibase/bbui" "^1.0.79-alpha.5" - "@budibase/frontend-core" "^1.0.79-alpha.5" - "@budibase/string-templates" "^1.0.79-alpha.5" + "@budibase/bbui" "^1.0.79-alpha.7" + "@budibase/frontend-core" "^1.0.79-alpha.7" + "@budibase/string-templates" "^1.0.79-alpha.7" "@spectrum-css/button" "^3.0.3" "@spectrum-css/card" "^3.0.3" "@spectrum-css/divider" "^1.0.3" @@ -1106,12 +1106,12 @@ svelte-flatpickr "^3.1.0" svelte-spa-router "^3.0.5" -"@budibase/frontend-core@^1.0.79-alpha.5": - version "1.0.79-alpha.5" - resolved "https://registry.yarnpkg.com/@budibase/frontend-core/-/frontend-core-1.0.79-alpha.5.tgz#7da5faf83d6cc5a59d8e038c2e9333e27bff35d5" - integrity sha512-5xti0MdKRvNKwYUE5cp4rH8IwLPmuRz39ajck947ut2OWzXV9bt7SXzoKPSSzEGdCBA2DgzJpK3gQWYlqXiJiQ== +"@budibase/frontend-core@^1.0.79-alpha.7": + version "1.0.79-alpha.7" + resolved "https://registry.yarnpkg.com/@budibase/frontend-core/-/frontend-core-1.0.79-alpha.7.tgz#cba8f61932f966dc3f19cc7d5fed45d832ee676e" + integrity sha512-mEspQXLUnjvNcL7QfDN1qIFGRo+AfdcaEq23gKAWXF1R+Byy7VCYDzcowzJY/TT6B4BSq3z6s57z3ILKtqI7zA== dependencies: - "@budibase/bbui" "^1.0.79-alpha.5" + "@budibase/bbui" "^1.0.79-alpha.7" lodash "^4.17.21" svelte "^3.46.2" @@ -1158,10 +1158,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@^1.0.79-alpha.5": - version "1.0.79-alpha.5" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.79-alpha.5.tgz#063f5beca7d3b4a9757df77dcf1bd8a442d7522e" - integrity sha512-Rifn1h1Pn53KYCFX6GHmMq+fD4IEnfRXEWrf4RD7cy4TVCYqCIcI84tnzUwibkyuCbpDw4zh0RR0m4nemf7heg== +"@budibase/string-templates@^1.0.79-alpha.7": + version "1.0.79-alpha.7" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.79-alpha.7.tgz#3e5235e05f13fe406cae62862110f841788d1bc0" + integrity sha512-wdnk0wi9vuSYY7vimIGV1+i0dSONOBg5deZia8v9O8XM9OmJohLUIkJdMNhhv9OCxyeC53gauaxhVdKeop6kmA== dependencies: "@budibase/handlebars-helpers" "^0.11.8" dayjs "^1.10.4" @@ -8707,6 +8707,11 @@ koa-views@^7.0.1: pretty "^2.0.0" resolve-path "^1.4.0" +koa2-ratelimit@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/koa2-ratelimit/-/koa2-ratelimit-1.1.0.tgz#5ab432fdda7b2d63a4fb1b9a0d994c1264396aff" + integrity sha512-AumRCI8YO9TMF9trVP6j68K5qzi21ajZUOCb5VuPWq9pZw+FHXam275S5P1IDAlZjs1cDFBOAAkhwTdTbVCcsg== + koa@2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/koa/-/koa-2.7.0.tgz#7e00843506942b9d82c6cc33749f657c6e5e7adf" From a8a8498a9c31aa3edfa17fd596d74f900d520460 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Mar 2022 12:27:09 +0000 Subject: [PATCH 2/6] Updating rate limiter to make use of Redis. --- packages/server/package.json | 1 + .../server/src/api/routes/public/index.ts | 13 ++++ packages/server/yarn.lock | 66 +++++++++++++++++-- 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 9846d5752c..4986fa84d6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -127,6 +127,7 @@ "pouchdb-all-dbs": "1.0.2", "pouchdb-find": "^7.2.2", "pouchdb-replication-stream": "1.2.9", + "redis": "4", "server-destroy": "1.0.1", "svelte": "^3.38.2", "swagger-parser": "^10.0.3", diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 438ef2c1e9..911977ba1d 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -12,14 +12,27 @@ import env from "../../../environment" // below imports don't have declaration files const Router = require("@koa/router") const RateLimit = require("koa2-ratelimit").RateLimit +const Stores = require("koa2-ratelimit").Stores const { PermissionLevels, PermissionTypes, } = require("@budibase/backend-core/permissions") +const { getRedisOptions } = require("@budibase/backend-core/redis").utils const PREFIX = "/api/public/v1" const DEFAULT_API_LIMITING = 120 +const REDIS_OPTS = getRedisOptions() +RateLimit.defaultOptions({ + store: new Stores.Redis({ + socket: { + host: REDIS_OPTS.host, + port: REDIS_OPTS.port, + }, + password: REDIS_OPTS.opts.password, + database: 1, + }), +}) // rate limiting, allows for 2 requests per second const limiter = RateLimit.middleware({ interval: { min: 1 }, diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index ce4213c322..f5a4db882d 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1885,6 +1885,41 @@ path-to-regexp "^1.1.1" urijs "^1.19.0" +"@node-redis/bloom@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@node-redis/bloom/-/bloom-1.0.1.tgz#144474a0b7dc4a4b91badea2cfa9538ce0a1854e" + integrity sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw== + +"@node-redis/client@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.4.tgz#fe185750df3bcc07524f63fe8dbc8d14d22d6cbb" + integrity sha512-IM/NRAqg7MvNC3bIRQipXGrEarunrdgvrbAzsd3ty93LSHi/M+ybQulOERQi8a3M+P5BL8HenwXjiIoKm6ml2g== + dependencies: + cluster-key-slot "1.1.0" + generic-pool "3.8.2" + redis-parser "3.0.0" + yallist "4.0.0" + +"@node-redis/graph@1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@node-redis/graph/-/graph-1.0.0.tgz#baf8eaac4a400f86ea04d65ec3d65715fd7951ab" + integrity sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g== + +"@node-redis/json@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@node-redis/json/-/json-1.0.2.tgz#8ad2d0f026698dc1a4238cc3d1eb099a3bee5ab8" + integrity sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g== + +"@node-redis/search@1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@node-redis/search/-/search-1.0.3.tgz#7c3d026bf994caf82019fd0c3924cfc09f041a29" + integrity sha512-rsrzkGWI84di/uYtEctS/4qLusWt0DESx/psjfB0TFpORDhe7JfC0h8ary+eHulTksumor244bXLRSqQXbFJmw== + +"@node-redis/time-series@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@node-redis/time-series/-/time-series-1.0.2.tgz#5dd3638374edd85ebe0aa6b0e87addc88fb9df69" + integrity sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -4124,7 +4159,7 @@ cls-hooked@^4.2.2: emitter-listener "^1.0.1" semver "^5.4.1" -cluster-key-slot@^1.1.0: +cluster-key-slot@1.1.0, cluster-key-slot@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== @@ -6103,6 +6138,11 @@ generate-function@^2.3.1: dependencies: is-property "^1.0.2" +generic-pool@3.8.2: + version "3.8.2" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9" + integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -11181,13 +11221,25 @@ redis-info@^3.0.8: dependencies: lodash "^4.17.11" -redis-parser@^3.0.0: +redis-parser@3.0.0, redis-parser@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ= dependencies: redis-errors "^1.0.0" +redis@4: + version "4.0.4" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.4.tgz#b567f82f59086df38433982f7f424b48e924ec7a" + integrity sha512-KaM1OAj/nGrSeybmmOWSMY0LXTGT6FVWgUZZrd2MYzXKJ+VGtqVaciGQeNMfZiQX+kDM8Ke4uttb54m2rm6V0A== + dependencies: + "@node-redis/bloom" "1.0.1" + "@node-redis/client" "1.0.4" + "@node-redis/graph" "1.0.0" + "@node-redis/json" "1.0.2" + "@node-redis/search" "1.0.3" + "@node-redis/time-series" "1.0.2" + regenerate-unicode-properties@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-9.0.0.tgz#54d09c7115e1f53dc2314a974b32c1c344efe326" @@ -13717,16 +13769,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yaml@2.0.0-1: version "2.0.0-1" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.0.0-1.tgz#8c3029b3ee2028306d5bcf396980623115ff8d18" From 32e7f27dfa7093b8eea7e0d5c3480cbc5cddb55c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Mar 2022 12:50:10 +0000 Subject: [PATCH 3/6] Adding check to disable rate limit redis connection in test. --- .../server/src/api/routes/public/index.ts | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 911977ba1d..c4d8247b66 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -22,17 +22,19 @@ const { getRedisOptions } = require("@budibase/backend-core/redis").utils const PREFIX = "/api/public/v1" const DEFAULT_API_LIMITING = 120 -const REDIS_OPTS = getRedisOptions() -RateLimit.defaultOptions({ - store: new Stores.Redis({ - socket: { - host: REDIS_OPTS.host, - port: REDIS_OPTS.port, - }, - password: REDIS_OPTS.opts.password, - database: 1, - }), -}) +if (!env.isTest()) { + const REDIS_OPTS = getRedisOptions() + RateLimit.defaultOptions({ + store: new Stores.Redis({ + socket: { + host: REDIS_OPTS.host, + port: REDIS_OPTS.port, + }, + password: REDIS_OPTS.opts.password, + database: 1, + }), + }) +} // rate limiting, allows for 2 requests per second const limiter = RateLimit.middleware({ interval: { min: 1 }, From 216af35aeb93ebbaa05a8b947db0a65aeab47978 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Mar 2022 12:03:29 +0000 Subject: [PATCH 4/6] Fixing review comments. --- packages/server/package.json | 1 + .../src/api/controllers/row/ExternalRequest.ts | 2 +- packages/server/src/api/routes/public/index.ts | 16 +++++++++++----- packages/server/src/environment.js | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 5d7d2fa6ec..a14eb01430 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -148,6 +148,7 @@ "@types/jest": "^26.0.23", "@types/koa": "^2.13.3", "@types/koa-router": "^7.4.2", + "@types/koa2-ratelimit": "^0.9.2", "@types/node": "^15.12.4", "@types/oracledb": "^5.2.1", "@typescript-eslint/parser": "4.28.0", diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index c1181dd6d5..c109a43afa 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -58,7 +58,7 @@ module External { ) { const primary = table.primary // if passed in array need to copy for shifting etc - let idCopy = cloneDeep(id) + let idCopy: undefined | string | any[] = cloneDeep(id) if (filters) { // need to map over the filters and make sure the _id field isn't present for (let filter of Object.values(filters)) { diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index c4d8247b66..fe10f7b637 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -9,10 +9,9 @@ import { paramResource, paramSubResource } from "../../../middleware/resourceId" import { CtxFn } from "./utils/Endpoint" import mapperMiddleware from "./middleware/mapper" import env from "../../../environment" +import { RateLimit, Stores } from "koa2-ratelimit" // below imports don't have declaration files const Router = require("@koa/router") -const RateLimit = require("koa2-ratelimit").RateLimit -const Stores = require("koa2-ratelimit").Stores const { PermissionLevels, PermissionTypes, @@ -20,7 +19,14 @@ const { const { getRedisOptions } = require("@budibase/backend-core/redis").utils const PREFIX = "/api/public/v1" -const DEFAULT_API_LIMITING = 120 +const DEFAULT_API_REQ_LIMIT_PER_SEC = 10 + +function getApiLimitPerSecond(): number { + if (!env.API_REQ_LIMIT_PER_SEC) { + return DEFAULT_API_REQ_LIMIT_PER_SEC + } + return parseInt(env.API_REQ_LIMIT_PER_SEC) +} if (!env.isTest()) { const REDIS_OPTS = getRedisOptions() @@ -37,9 +43,9 @@ if (!env.isTest()) { } // rate limiting, allows for 2 requests per second const limiter = RateLimit.middleware({ - interval: { min: 1 }, + interval: { sec: 1 }, // per ip, per interval - max: env.API_RATE_LIMITING || DEFAULT_API_LIMITING, + max: getApiLimitPerSecond(), }) const publicRouter = new Router({ diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index e3e321b795..312ecf313d 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -45,7 +45,7 @@ module.exports = { INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, - API_RATE_LIMITING: process.env.API_RATE_LIMITING, + API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC, // environment NODE_ENV: process.env.NODE_ENV, JEST_WORKER_ID: process.env.JEST_WORKER_ID, From 54137ec9fb0f38efec77151f70fbc9c87f15e8cc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Mar 2022 12:19:12 +0000 Subject: [PATCH 5/6] Removing koa rate limit types, was requiring types for every store, types didn't work correctly. --- packages/server/package.json | 2 +- .../server/src/api/routes/public/index.ts | 3 +- packages/server/yarn.lock | 76 ++++++++++++++++++- 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index a14eb01430..4602e4f0d8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -148,9 +148,9 @@ "@types/jest": "^26.0.23", "@types/koa": "^2.13.3", "@types/koa-router": "^7.4.2", - "@types/koa2-ratelimit": "^0.9.2", "@types/node": "^15.12.4", "@types/oracledb": "^5.2.1", + "@types/redis": "^4.0.11", "@typescript-eslint/parser": "4.28.0", "apidoc": "^0.50.2", "babel-jest": "^27.0.2", diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index fe10f7b637..3a89df95d6 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -9,9 +9,9 @@ import { paramResource, paramSubResource } from "../../../middleware/resourceId" import { CtxFn } from "./utils/Endpoint" import mapperMiddleware from "./middleware/mapper" import env from "../../../environment" -import { RateLimit, Stores } from "koa2-ratelimit" // below imports don't have declaration files const Router = require("@koa/router") +const { RateLimit, Stores } = require("koa2-ratelimit") const { PermissionLevels, PermissionTypes, @@ -32,6 +32,7 @@ if (!env.isTest()) { const REDIS_OPTS = getRedisOptions() RateLimit.defaultOptions({ store: new Stores.Redis({ + // @ts-ignore socket: { host: REDIS_OPTS.host, port: REDIS_OPTS.port, diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 19c3578eab..95c1e6736a 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2619,7 +2619,7 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/mongodb@*": +"@types/mongodb@*", "@types/mongodb@^4.0.7": version "4.0.7" resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-4.0.7.tgz#ebaa80c53b684ea52ccfe7721c0f5c9ef7b4f511" integrity sha512-lPUYPpzA43baXqnd36cZ9xxorprybxXDzteVKCPAdp14ppHtFJHnXYvNpmBvtMUTb5fKXVv6sVbzo1LHkWhJlw== @@ -2634,6 +2634,13 @@ "@types/mongodb" "*" "@types/node" "*" +"@types/mongoose@^5.11.97": + version "5.11.97" + resolved "https://registry.yarnpkg.com/@types/mongoose/-/mongoose-5.11.97.tgz#80b0357f3de6807eb597262f52e49c3e13ee14d8" + integrity sha512-cqwOVYT3qXyLiGw7ueU2kX9noE8DPGRY6z8eUxudhXY8NZ7DMKYAxyZkLSevGfhCX3dO/AoX5/SO9lAzfjon0Q== + dependencies: + mongoose "*" + "@types/node@*", "@types/node@>=13.13.4": version "16.11.7" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.7.tgz#36820945061326978c42a01e56b61cd223dfdc42" @@ -2692,6 +2699,13 @@ dependencies: "@types/node" "*" +"@types/redis@^4.0.11": + version "4.0.11" + resolved "https://registry.yarnpkg.com/@types/redis/-/redis-4.0.11.tgz#0bb4c11ac9900a21ad40d2a6768ec6aaf651c0e1" + integrity sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg== + dependencies: + redis "*" + "@types/request@^2.48.7": version "2.48.8" resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.8.tgz#0b90fde3b655ab50976cb8c5ac00faca22f5a82c" @@ -3886,7 +3900,7 @@ bson@^1.1.4: resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a" integrity sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg== -bson@^4.6.1: +bson@^4.2.2, bson@^4.6.1: version "4.6.1" resolved "https://registry.yarnpkg.com/bson/-/bson-4.6.1.tgz#2b5da517539bb0f7f3ffb54ac70a384ca899641c" integrity sha512-I1LQ7Hz5zgwR4QquilLNZwbhPw0Apx7i7X9kGMBTsqPdml/03Q9NBtD9nt/19ahjlphktQImrnderxqpzeVDjw== @@ -4683,6 +4697,13 @@ debug@4, debug@4.3.2, debug@^4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@ dependencies: ms "2.1.2" +debug@4.x: + version "4.3.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.3.tgz#04266e0b70a98d4462e6e288e38259213332b664" + integrity sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q== + dependencies: + ms "2.1.2" + debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -8630,6 +8651,11 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" +kareem@2.3.4: + version "2.3.4" + resolved "https://registry.yarnpkg.com/kareem/-/kareem-2.3.4.tgz#b38c436fb4758775d919b2828b4009db59b52694" + integrity sha512-Vcrt8lcpVl0s8ePx634BxwRqmFo+5DcOhlmNadehxreMTIQi/9hOL/B3hZQQbK5DgMS7Lem3xABXV7/S3jy+7g== + keygrip@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc" @@ -9652,6 +9678,43 @@ mongodb@3.6.3: optionalDependencies: saslprep "^1.0.0" +mongodb@4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.3.1.tgz#e346f76e421ec6f47ddea5c8f5140e6181aaeb94" + integrity sha512-sNa8APSIk+r4x31ZwctKjuPSaeKuvUeNb/fu/3B6dRM02HpEgig7hTHM8A/PJQTlxuC/KFWlDlQjhsk/S43tBg== + dependencies: + bson "^4.6.1" + denque "^2.0.1" + mongodb-connection-string-url "^2.4.1" + socks "^2.6.1" + optionalDependencies: + saslprep "^1.0.3" + +mongoose@*: + version "6.2.4" + resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-6.2.4.tgz#c6ff4f84ab47b6c760e773424b0fd1f392d98563" + integrity sha512-3hA3IGxBzZdlp1+/I9qn53NjEAd01qvKAH2WUCPahjVO8+uAmR0B4m+1bC3x9a4r0ExY8QYQ2ryG3E/v5Tj+jA== + dependencies: + bson "^4.2.2" + kareem "2.3.4" + mongodb "4.3.1" + mpath "0.8.4" + mquery "4.0.2" + ms "2.1.3" + sift "16.0.0" + +mpath@0.8.4: + version "0.8.4" + resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.8.4.tgz#6b566d9581621d9e931dd3b142ed3618e7599313" + integrity sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g== + +mquery@4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/mquery/-/mquery-4.0.2.tgz#a13add5ecd7c2e5a67e0f814b3c7acdfb6772804" + integrity sha512-oAVF0Nil1mT3rxty6Zln4YiD6x6QsUWYz927jZzjMxOK2aqmhEz5JQ7xmrKK7xRFA2dwV+YaOpKU/S+vfNqKxA== + dependencies: + debug "4.x" + mri@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a" @@ -9672,7 +9735,7 @@ ms@2.1.2, ms@^2.1.1: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.1.3: +ms@2.1.3, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -11337,7 +11400,7 @@ redis-parser@3.0.0, redis-parser@^3.0.0: dependencies: redis-errors "^1.0.0" -redis@4: +redis@*, redis@4: version "4.0.4" resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.4.tgz#b567f82f59086df38433982f7f424b48e924ec7a" integrity sha512-KaM1OAj/nGrSeybmmOWSMY0LXTGT6FVWgUZZrd2MYzXKJ+VGtqVaciGQeNMfZiQX+kDM8Ke4uttb54m2rm6V0A== @@ -11923,6 +11986,11 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" +sift@16.0.0: + version "16.0.0" + resolved "https://registry.yarnpkg.com/sift/-/sift-16.0.0.tgz#447991577db61f1a8fab727a8a98a6db57a23eb8" + integrity sha512-ILTjdP2Mv9V1kIxWMXeMTIRbOBrqKc4JAXmFMnFq3fKeyQ2Qwa3Dw1ubcye3vR+Y6ofA0b9gNDr/y2t6eUeIzQ== + sigmund@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590" From 86f649c5db039bde45afff6d528386ee469f8a2a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Mar 2022 12:31:56 +0000 Subject: [PATCH 6/6] Upping test rate limit level. --- packages/server/src/api/routes/public/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 3a89df95d6..800eae6101 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -19,7 +19,8 @@ const { const { getRedisOptions } = require("@budibase/backend-core/redis").utils const PREFIX = "/api/public/v1" -const DEFAULT_API_REQ_LIMIT_PER_SEC = 10 +// allow a lot more requests when in test +const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 function getApiLimitPerSecond(): number { if (!env.API_REQ_LIMIT_PER_SEC) {