From b07382fa2e0efb0009bf98f14f3140ebd07da591 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Sat, 4 Nov 2023 11:04:09 +0100 Subject: [PATCH 01/13] WIP - Migrating worker --- packages/types/src/sdk/licensing/plan.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 1604dfb8af..5ac8b1c9f6 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -7,7 +7,9 @@ export enum PlanType { /** @deprecated */ PREMIUM = "premium", PREMIUM_PLUS = "premium_plus", + /** @deprecated */ BUSINESS = "business", + ENTERPRISE_BASIC = "enterprise_basic", ENTERPRISE = "enterprise", } From 5088caf0f944c82cad3b292df3c06896a0583893 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 6 Nov 2023 08:37:53 +0100 Subject: [PATCH 02/13] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 3820c0c93a..34cce46d9c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 +Subproject commit 34cce46d9c67f03aa66272e079e1d460a731225e From 47292b8ab4fd95a7d1ffaada5baf28bd394b977d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 8 Nov 2023 14:37:19 +0000 Subject: [PATCH 03/13] Make DB name non-optional. --- packages/backend-core/src/context/mainContext.ts | 3 +++ packages/backend-core/src/db/couch/DatabaseImpl.ts | 5 +---- packages/backend-core/src/db/db.ts | 12 +----------- packages/server/src/api/controllers/datasource.ts | 2 +- packages/server/src/db/utils.ts | 12 +++++++++--- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 609c18abb5..52ab17d31e 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database { */ export function getAppDB(opts?: any): Database { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve app DB - no app ID.") + } return getDB(appId, opts) } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 330b15e680..1ace60ed5b 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -48,10 +48,7 @@ export class DatabaseImpl implements Database { private readonly couchInfo = getCouchInfo() - constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { - if (dbName == null) { - throw new Error("Database name cannot be undefined.") - } + constructor(dbName: string, opts?: DatabaseOpts, connection?: string) { this.name = dbName this.pouchOpts = opts || {} if (connection) { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 9aae64b892..54c4d3ae0f 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,10 +1,7 @@ -import env from "../environment" import { directCouchQuery, DatabaseImpl } from "./couch" import { CouchFindOptions, Database } from "@budibase/types" -const dbList = new Set() - -export function getDB(dbName?: string, opts?: any): Database { +export function getDB(dbName: string, opts?: any): Database { return new DatabaseImpl(dbName, opts) } @@ -22,13 +19,6 @@ export async function doWithDB( return await cb(db) } -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - return [...dbList] -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 5d024d51b6..39bc612b32 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -337,7 +337,7 @@ export async function destroy(ctx: UserCtx) { if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { await destroyInternalTablesBySourceId(datasourceId) } else { - const queries = await db.allDocs(getQueryParams(datasourceId, null)) + const queries = await db.allDocs(getQueryParams(datasourceId)) await db.bulkDocs( queries.rows.map((row: any) => ({ _id: row.id, diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index d532d8a8b2..715db552c9 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -6,6 +6,7 @@ import { RelationshipFieldMetadata, VirtualDocumentType, INTERNAL_TABLE_SOURCE_ID, + DatabaseQueryOpts, } from "@budibase/types" import { FieldTypes } from "../constants" export { DocumentType, VirtualDocumentType } from "@budibase/types" @@ -229,7 +230,10 @@ export function getAutomationMetadataParams(otherProps: any = {}) { /** * Gets parameters for retrieving a query, this is a utility function for the getDocParams function. */ -export function getQueryParams(datasourceId?: Optional, otherProps: any = {}) { +export function getQueryParams( + datasourceId?: Optional, + otherProps: Partial = {} +) { if (datasourceId == null) { return getDocParams(DocumentType.QUERY, null, otherProps) } @@ -256,7 +260,7 @@ export function generateMetadataID(type: string, entityId: string) { export function getMetadataParams( type: string, entityId?: Optional, - otherProps: any = {} + otherProps: Partial = {} ) { let docId = `${type}${SEPARATOR}` if (entityId != null) { @@ -269,7 +273,9 @@ export function generateMemoryViewID(viewName: string) { return `${DocumentType.MEM_VIEW}${SEPARATOR}${viewName}` } -export function getMemoryViewParams(otherProps: any = {}) { +export function getMemoryViewParams( + otherProps: Partial = {} +) { return getDocParams(DocumentType.MEM_VIEW, null, otherProps) } From e3a4c34f8d2b7094965b1ff10f23fb3a8cd8144f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 8 Nov 2023 16:17:24 +0000 Subject: [PATCH 04/13] Pass types through context callbacks. --- .../backend-core/src/cache/appMetadata.ts | 2 +- packages/backend-core/src/context/Context.ts | 2 +- .../backend-core/src/context/mainContext.ts | 20 +++++++++---------- packages/backend-core/src/db/db.ts | 6 +++--- packages/server/src/automations/triggers.ts | 2 +- .../src/tests/utilities/TestConfiguration.ts | 11 +++++----- packages/server/src/threads/automation.ts | 2 +- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index bd3efc20db..d442511fb8 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -19,7 +19,7 @@ async function populateFromDB(appId: string) { return doWithDB( appId, (db: Database) => { - return db.get(DocumentType.APP_METADATA) + return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index d29b6935a8..a59f5c6503 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -4,7 +4,7 @@ import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() - static run(context: ContextMap, func: any) { + static run(context: ContextMap, func: () => T) { return Context.storage.run(context, () => func()) } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 52ab17d31e..d2259cfcab 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap { return context } -async function newContext(updates: ContextMap, task: any) { +async function newContext(updates: ContextMap, task: () => T) { // see if there already is a context setup let context: ContextMap = updateContext(updates) return Context.run(context, task) } -export async function doInAutomationContext(params: { +export async function doInAutomationContext(params: { appId: string automationId: string - task: any -}): Promise { + task: () => T +}): Promise { const tenantId = getTenantIDFromAppID(params.appId) return newContext( { @@ -144,10 +144,10 @@ export async function doInTenant( return newContext(updates, task) } -export async function doInAppContext( +export async function doInAppContext( appId: string | null, - task: any -): Promise { + task: () => T +): Promise { if (!appId && !env.isTest()) { throw new Error("appId is required") } @@ -165,10 +165,10 @@ export async function doInAppContext( return newContext(updates, task) } -export async function doInIdentityContext( +export async function doInIdentityContext( identity: IdentityContext, - task: any -): Promise { + task: () => T +): Promise { if (!identity) { throw new Error("identity is required") } diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 54c4d3ae0f..3e69d49f0e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,7 +1,7 @@ import { directCouchQuery, DatabaseImpl } from "./couch" -import { CouchFindOptions, Database } from "@budibase/types" +import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" -export function getDB(dbName: string, opts?: any): Database { +export function getDB(dbName: string, opts?: DatabaseOpts): Database { return new DatabaseImpl(dbName, opts) } @@ -11,7 +11,7 @@ export function getDB(dbName: string, opts?: any): Database { export async function doWithDB( dbName: string, cb: (db: Database) => Promise, - opts = {} + opts?: DatabaseOpts ) { const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index f0eca759f5..ac977bbefb 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -94,7 +94,7 @@ export async function externalTrigger( automation: Automation, params: { fields: Record; timeout?: number }, { getResponses }: { getResponses?: boolean } = {} -) { +): Promise { if ( automation.definition != null && automation.definition.trigger != null && diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 6877561fcb..3a14a87d2a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -510,13 +510,14 @@ class TestConfiguration { // create dev app // clear any old app this.appId = null - await context.doInAppContext(null, async () => { - this.app = await this._req( + this.app = await context.doInAppContext(null, async () => { + const app = await this._req( { name: appName }, null, controllers.app.create ) - this.appId = this.app?.appId! + this.appId = app.appId! + return app }) return await context.doInAppContext(this.appId, async () => { // create production app @@ -525,7 +526,7 @@ class TestConfiguration { this.allApps.push(this.prodApp) this.allApps.push(this.app) - return this.app + return this.app! }) } @@ -537,7 +538,7 @@ class TestConfiguration { return context.doInAppContext(prodAppId, async () => { const db = context.getProdAppDB() - return await db.get(dbCore.DocumentType.APP_METADATA) + return await db.get(dbCore.DocumentType.APP_METADATA) }) } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9241289e86..d1fcc2be72 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -241,7 +241,7 @@ class Orchestrator { }) } - async execute() { + async execute(): Promise { // this will retrieve from context created at start of thread this._context.env = await sdkUtils.getEnvironmentVariables() let automation = this._automation From b03719ff1c72509e7edd75f7a1eb82695c658c80 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Wed, 8 Nov 2023 17:34:28 +0100 Subject: [PATCH 05/13] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 34cce46d9c..cbd3c7e333 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 34cce46d9c67f03aa66272e079e1d460a731225e +Subproject commit cbd3c7e33389c976dc73934667beb65d60c605f3 From e4bc6a5fc78448b1c447874a321bf2dd021ec647 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 8 Nov 2023 18:41:15 +0000 Subject: [PATCH 06/13] Attempting to fix QA-core inconsistencies in CI - disabling rate limiting during test runs for public API. --- .../server/src/api/routes/public/index.ts | 72 ++++++++++--------- packages/server/src/environment.ts | 1 + qa-core/package.json | 2 +- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index b37ed931fc..f27f3f8857 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -27,51 +27,59 @@ interface KoaRateLimitOptions { } const PREFIX = "/api/public/v1" -// allow a lot more requests when in test -const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 -function getApiLimitPerSecond(): number { - if (!env.API_REQ_LIMIT_PER_SEC) { - return DEFAULT_API_REQ_LIMIT_PER_SEC - } - return parseInt(env.API_REQ_LIMIT_PER_SEC) -} +// type can't be known - untyped libraries +let limiter: any, rateLimitStore: any +if (!env.DISABLE_RATE_LIMITING) { + // allow a lot more requests when in test + const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 -let rateLimitStore: any = null -if (!env.isTest()) { - const { password, host, port } = redis.utils.getRedisConnectionDetails() - let options: KoaRateLimitOptions = { - socket: { - host: host, - port: port, - }, + function getApiLimitPerSecond(): number { + if (!env.API_REQ_LIMIT_PER_SEC) { + return DEFAULT_API_REQ_LIMIT_PER_SEC + } + return parseInt(env.API_REQ_LIMIT_PER_SEC) } - if (password) { - options.password = password - } + if (!env.isTest()) { + const { password, host, port } = redis.utils.getRedisConnectionDetails() + let options: KoaRateLimitOptions = { + socket: { + host: host, + port: port, + }, + } - if (!env.REDIS_CLUSTERED) { - // Can't set direct redis db in clustered env - options.database = SelectableDatabase.RATE_LIMITING + if (password) { + options.password = password + } + + if (!env.REDIS_CLUSTERED) { + // Can't set direct redis db in clustered env + options.database = SelectableDatabase.RATE_LIMITING + } + rateLimitStore = new Stores.Redis(options) + RateLimit.defaultOptions({ + store: rateLimitStore, + }) } - rateLimitStore = new Stores.Redis(options) - RateLimit.defaultOptions({ - store: rateLimitStore, + // rate limiting, allows for 2 requests per second + limiter = RateLimit.middleware({ + interval: { sec: 1 }, + // per ip, per interval + max: getApiLimitPerSecond(), }) +} else { + console.log("**** PUBLIC API RATE LIMITING DISABLED ****") } -// rate limiting, allows for 2 requests per second -const limiter = RateLimit.middleware({ - interval: { sec: 1 }, - // per ip, per interval - max: getApiLimitPerSecond(), -}) const publicRouter = new Router({ prefix: PREFIX, }) -publicRouter.use(limiter) +if (limiter) { + publicRouter.use(limiter) +} function addMiddleware( endpoints: any, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 91424113ac..c126a61c22 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -61,6 +61,7 @@ const environment = { ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS, + DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING, MULTI_TENANCY: process.env.MULTI_TENANCY, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, diff --git a/qa-core/package.json b/qa-core/package.json index cfccd5e650..87172d2ed9 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -20,7 +20,7 @@ "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.", "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci", "serve": "start-server-and-test dev:built http://localhost:4001/health", - "dev:built": "cd ../ && yarn dev:built" + "dev:built": "cd ../ && DISABLE_RATE_LIMITING=1 yarn dev:built" }, "devDependencies": { "@budibase/types": "^2.3.17", From 8436a8d22055d47cf5f331cc4e65259ae5d87e5d Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 09:54:05 +0100 Subject: [PATCH 07/13] Day passes are optional --- packages/types/src/sdk/licensing/quota.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 85700f167b..2d33c14182 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,7 +61,7 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined } export type MonthlyQuotas = { [MonthlyQuotaName.QUERIES]: Quota [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota + [MonthlyQuotaName.DAY_PASSES]?: Quota } export type StaticQuotas = { From e1ca1d67322779eae0857028ea715a899933da14 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 09:55:09 +0100 Subject: [PATCH 08/13] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index cbd3c7e333..ab17824872 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit cbd3c7e33389c976dc73934667beb65d60c605f3 +Subproject commit ab17824872a08010886abdc5893b14d3430efe00 From 311ad6b6a5010435fd56778151f85356e4ee01aa Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 10:10:04 +0100 Subject: [PATCH 09/13] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index ab17824872..18f558efb0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ab17824872a08010886abdc5893b14d3430efe00 +Subproject commit 18f558efb0f2d4a021a46d6236d12d93f6ce474f From 7afcfbac93fc172e3b0a754e101f06de90d600bc Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 9 Nov 2023 09:27:06 +0000 Subject: [PATCH 10/13] Bump version to 2.13.6 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5605642877..ce3128165f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.5", + "version": "2.13.6", "npmClient": "yarn", "packages": [ "packages/*" From 1f85b56a99400b3cda5735379a98a359279180a5 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 11:44:36 +0100 Subject: [PATCH 11/13] Revert "Day passes are optional" This reverts commit 8436a8d22055d47cf5f331cc4e65259ae5d87e5d. --- packages/types/src/sdk/licensing/quota.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 2d33c14182..85700f167b 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,7 +61,7 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined } export type MonthlyQuotas = { [MonthlyQuotaName.QUERIES]: Quota [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]?: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota } export type StaticQuotas = { From 5ddc3faf22a7100cf83a286ae372967b83a268bb Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 11:45:38 +0100 Subject: [PATCH 12/13] Upgrade pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 18f558efb0..837bbc7b33 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 18f558efb0f2d4a021a46d6236d12d93f6ce474f +Subproject commit 837bbc7b3317c5681fa22f2b92a8d1a7d39c3c95 From 14c6e1bc4a4cf9c3f68cda4b300e8e511e15e73d Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 12:00:21 +0100 Subject: [PATCH 13/13] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 837bbc7b33..e202f415d9 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 837bbc7b3317c5681fa22f2b92a8d1a7d39c3c95 +Subproject commit e202f415d9fa540d08cc2ba6e27394fbc22f357b