From cbf0cf76d33d90b73fd3f3ec7a55631010155c52 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Nov 2024 17:09:32 +0100 Subject: [PATCH 01/24] Use timestamp from runtime --- packages/server/src/automations/utils.ts | 10 +++++++--- packages/types/src/sdk/automations/index.ts | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 62125ea589..365dc36b68 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -70,6 +70,10 @@ export async function processEvent(job: AutomationJob) { const task = async () => { try { + if (isCronTrigger(job.data.automation)) { + // Requires the timestamp at run time + job.data.event.timestamp = Date.now() + } // need to actually await these so that an error can be captured properly console.log("automation running", ...loggingArgs(job)) @@ -210,15 +214,15 @@ export async function enableCronTrigger(appId: any, automation: Automation) { } // make a job id rather than letting Bull decide, makes it easier to handle on way out const jobId = `${appId}_cron_${utils.newid()}` - const job: any = await automationQueue.add( + const job = await automationQueue.add( { automation, - event: { appId, timestamp: Date.now() }, + event: { appId }, }, { repeat: { cron: cronExp }, jobId } ) // Assign cron job ID from bull so we can remove it later if the cron trigger is removed - trigger.cronJobId = job.id + trigger.cronJobId = job.id.toString() // can't use getAppDB here as this is likely to be called from dev app, // but this call could be for dev app or prod app, need to just use what // was passed in diff --git a/packages/types/src/sdk/automations/index.ts b/packages/types/src/sdk/automations/index.ts index 9ceded03ee..ba79d47021 100644 --- a/packages/types/src/sdk/automations/index.ts +++ b/packages/types/src/sdk/automations/index.ts @@ -14,6 +14,7 @@ export interface AutomationDataEvent { row?: Row oldRow?: Row user?: UserBindings + timestamp?: number } export interface AutomationData { From 0667dd9d30cd3038b61fd006d18c907f5a2cf349 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 12:38:58 +0100 Subject: [PATCH 02/24] Add test --- .../backend-core/src/queue/inMemoryQueue.ts | 2 +- .../tests/cron-automations.spec.ts | 49 +++++++++++++++++++ .../server/src/tests/utilities/structures.ts | 32 ++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/automations/tests/cron-automations.spec.ts diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 62b971f9f5..1d8544828d 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -141,7 +141,7 @@ class InMemoryQueue implements Partial { } else { pushMessage() } - return {} as any + return { id: jobId } as any } /** diff --git a/packages/server/src/automations/tests/cron-automations.spec.ts b/packages/server/src/automations/tests/cron-automations.spec.ts new file mode 100644 index 0000000000..62c8ccd612 --- /dev/null +++ b/packages/server/src/automations/tests/cron-automations.spec.ts @@ -0,0 +1,49 @@ +import tk from "timekeeper" +import "../../environment" +import * as automations from "../index" +import * as setup from "./utilities" +import { basicCronAutomation } from "../../tests/utilities/structures" + +const initialTime = Date.now() +tk.freeze(initialTime) + +const oneMinuteInMs = 60 * 1000 + +describe("cron automations", () => { + let config = setup.getConfig() + + beforeAll(async () => { + await automations.init() + await config.init() + }) + + afterAll(async () => { + await automations.shutdown() + setup.afterAll() + }) + + beforeEach(() => { + tk.freeze(initialTime) + }) + + async function travel(ms: number) { + tk.travel(Date.now() + ms) + } + + it("should initialise the automation timestamp", async () => { + const automation = basicCronAutomation(config.appId!, "* * * * *") + await config.api.automation.post(automation) + await travel(oneMinuteInMs) + await config.publish() + + const automationLogs = await config.getAutomationLogs() + expect(automationLogs.data).toHaveLength(1) + expect(automationLogs.data).toEqual([ + expect.objectContaining({ + trigger: expect.objectContaining({ + outputs: { timestamp: initialTime + oneMinuteInMs }, + }), + }), + ]) + }) +}) diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 49046d9eda..beb03ecf08 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -245,6 +245,38 @@ export function basicAutomation(appId?: string): Automation { } } +export function basicCronAutomation(appId: string, cron: string): Automation { + const automation: Automation = { + name: `Automation ${generator.guid()}`, + definition: { + trigger: { + stepId: AutomationTriggerStepId.CRON, + name: "test", + tagline: "test", + icon: "test", + description: "test", + type: AutomationStepType.TRIGGER, + id: "test", + inputs: { + cron, + }, + schema: { + inputs: { + properties: {}, + }, + outputs: { + properties: {}, + }, + }, + }, + steps: [], + }, + type: "automation", + appId, + } + return automation +} + export function serverLogAutomation(appId?: string): Automation { return { name: "My Automation", From 252c23f8bd1b813134050d3260cc87d2ac03cfda Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 13:55:47 +0100 Subject: [PATCH 03/24] Fix tests --- .../src/cache/tests/docWritethrough.spec.ts | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index 47b3f0672f..ab50a59274 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -10,18 +10,24 @@ import { init, } from "../docWritethrough" -import InMemoryQueue from "../../queue/inMemoryQueue" - const initialTime = Date.now() async function waitForQueueCompletion() { - const queue: InMemoryQueue = DocWritethroughProcessor.queue as never - await queue.waitForCompletion() + await DocWritethroughProcessor.queue.resume() + do { + await DocWritethroughProcessor.queue.whenCurrentJobsFinished() + } while (await DocWritethroughProcessor.queue.count()) + + await DocWritethroughProcessor.queue.whenCurrentJobsFinished() + + await DocWritethroughProcessor.queue.pause() } describe("docWritethrough", () => { - beforeAll(() => { + beforeAll(async () => { init() + await DocWritethroughProcessor.queue.isReady() + await DocWritethroughProcessor.queue.pause() }) const config = new DBTestConfiguration() @@ -67,7 +73,7 @@ describe("docWritethrough", () => { const patch3 = generatePatchObject(3) await docWritethrough.patch(patch3) - expect(await db.get(documentId)).toEqual({ + expect(await db.tryGet(documentId)).toEqual({ _id: documentId, ...patch1, ...patch2, @@ -92,7 +98,7 @@ describe("docWritethrough", () => { await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining({ _id: documentId, ...patch1, @@ -117,7 +123,7 @@ describe("docWritethrough", () => { await waitForQueueCompletion() expect(date1).not.toEqual(date2) - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining({ createdAt: date1.toISOString(), updatedAt: date2.toISOString(), @@ -135,7 +141,7 @@ describe("docWritethrough", () => { await docWritethrough.patch(patch2) const keyToOverride = _.sample(Object.keys(patch1))! - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining({ [keyToOverride]: patch1[keyToOverride], }) @@ -150,7 +156,7 @@ describe("docWritethrough", () => { await docWritethrough.patch(patch3) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining({ ...patch1, ...patch2, @@ -180,14 +186,14 @@ describe("docWritethrough", () => { await secondDocWritethrough.patch(doc2Patch2) await waitForQueueCompletion() - expect(await db.get(docWritethrough.docId)).toEqual( + expect(await db.tryGet(docWritethrough.docId)).toEqual( expect.objectContaining({ ...doc1Patch, ...doc1Patch2, }) ) - expect(await db.get(secondDocWritethrough.docId)).toEqual( + expect(await db.tryGet(secondDocWritethrough.docId)).toEqual( expect.objectContaining({ ...doc2Patch, ...doc2Patch2, @@ -203,7 +209,7 @@ describe("docWritethrough", () => { await docWritethrough.patch(initialPatch) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining(initialPatch) ) @@ -214,10 +220,10 @@ describe("docWritethrough", () => { await docWritethrough.patch(extraPatch) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining(extraPatch) ) - expect(await db.get(documentId)).not.toEqual( + expect(await db.tryGet(documentId)).not.toEqual( expect.objectContaining(initialPatch) ) }) @@ -242,7 +248,7 @@ describe("docWritethrough", () => { expect(queueMessageSpy).toHaveBeenCalledTimes(5) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining(patches) ) @@ -250,7 +256,7 @@ describe("docWritethrough", () => { expect(queueMessageSpy).toHaveBeenCalledTimes(45) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining(patches) ) @@ -258,7 +264,7 @@ describe("docWritethrough", () => { expect(queueMessageSpy).toHaveBeenCalledTimes(55) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining(patches) ) }) @@ -279,13 +285,13 @@ describe("docWritethrough", () => { await incrementalPatches(5) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining({ [keyToOverride]: 5 }) ) await incrementalPatches(40) await waitForQueueCompletion() - expect(await db.get(documentId)).toEqual( + expect(await db.tryGet(documentId)).toEqual( expect.objectContaining({ [keyToOverride]: 45 }) ) }) From a952cd7d087c87374895ecacff7f383fb2d8e912 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 13:57:06 +0100 Subject: [PATCH 04/24] Enable disabled test --- .../backend-core/src/cache/tests/docWritethrough.spec.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index ab50a59274..d13f3c9aa9 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -270,14 +270,12 @@ describe("docWritethrough", () => { }) }) - // This is not yet supported - // eslint-disable-next-line jest/no-disabled-tests - it.skip("patches will execute in order", async () => { + it("patches will execute in order", async () => { let incrementalValue = 0 const keyToOverride = generator.word() async function incrementalPatches(count: number) { for (let i = 0; i < count; i++) { - await docWritethrough.patch({ [keyToOverride]: incrementalValue++ }) + await docWritethrough.patch({ [keyToOverride]: ++incrementalValue }) } } From d0aab5fa87ef3458d3ee02e4606d928e6b9d81a7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 14:17:48 +0100 Subject: [PATCH 05/24] Fix pro --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 04bee88597..21e9cb4107 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 04bee88597edb1edb88ed299d0597b587f0362ec +Subproject commit 21e9cb41076c7064b6d5e16e0434705149440023 From 8c88d650035b9ec06840ecd855f316044b99929f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 14:36:18 +0100 Subject: [PATCH 06/24] Simplify api --- .../backend-core/src/cache/tests/docWritethrough.spec.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index d13f3c9aa9..cb8e2ff3fe 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -13,21 +13,16 @@ import { const initialTime = Date.now() async function waitForQueueCompletion() { - await DocWritethroughProcessor.queue.resume() do { await DocWritethroughProcessor.queue.whenCurrentJobsFinished() } while (await DocWritethroughProcessor.queue.count()) await DocWritethroughProcessor.queue.whenCurrentJobsFinished() - - await DocWritethroughProcessor.queue.pause() } describe("docWritethrough", () => { beforeAll(async () => { init() - await DocWritethroughProcessor.queue.isReady() - await DocWritethroughProcessor.queue.pause() }) const config = new DBTestConfiguration() From 358f91177c5c29fc7f7354caf06398e48d2f9d2d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 16:43:35 +0100 Subject: [PATCH 07/24] Fix test --- .../api/routes/global/tests/auditLogs.spec.ts | 165 ++++++++++-------- 1 file changed, 88 insertions(+), 77 deletions(-) diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index b540836583..55664f9893 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -12,105 +12,116 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" -describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { +describe("/api/global/auditlogs (%s)", () => { const config = new TestConfiguration() - let envCleanup: (() => void) | undefined beforeAll(async () => { - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: method === "sql", - }) await config.beforeAll() }) afterAll(async () => { - envCleanup?.() await config.afterAll() }) - describe("POST /api/global/auditlogs/search", () => { - it("should be able to fire some events (create audit logs)", async () => { - await context.doInTenant(config.tenantId, async () => { - const userId = config.user!._id! - const identity = { - ...BASE_IDENTITY, - _id: userId, - tenantId: config.tenantId, - } - await context.doInIdentityContext(identity, async () => { - for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { - await events.user.created(structures.users.user()) + describe.each(["lucene", "sql"])( + "POST /api/global/auditlogs/search", + method => { + let envCleanup: (() => void) | undefined + + beforeAll(async () => { + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: method === "sql", + }) + await config.useNewTenant() + }) + + afterAll(() => { + envCleanup?.() + }) + + it("should be able to fire some events (create audit logs)", async () => { + await context.doInTenant(config.tenantId, async () => { + const userId = config.user!._id! + const identity = { + ...BASE_IDENTITY, + _id: userId, + tenantId: config.tenantId, } - await context.doInAppContext(APP_ID, async () => { - await events.app.created(structures.apps.app(APP_ID)) + await context.doInIdentityContext(identity, async () => { + for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { + await events.user.created(structures.users.user()) + } + await context.doInAppContext(APP_ID, async () => { + await events.app.created(structures.apps.app(APP_ID)) + }) + // fetch the user created events + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data).toBeDefined() + // there will be an initial event which comes from the default user creation + expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) }) - // fetch the user created events - const response = await config.api.auditLogs.search({ - events: [Event.USER_CREATED], - }) - expect(response.data).toBeDefined() - // there will be an initial event which comes from the default user creation - expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) }) }) - }) - it("should be able to search by event", async () => { - const response = await config.api.auditLogs.search({ - events: [Event.USER_CREATED], + it("should be able to search by event", async () => { + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.event).toBe(Event.USER_CREATED) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.event).toBe(Event.USER_CREATED) - } - }) - it("should be able to search by time range (frozen)", async () => { - // this is frozen, only need to add 1 and minus 1 - const now = new Date() - const start = new Date() - start.setSeconds(now.getSeconds() - 1) - const end = new Date() - end.setSeconds(now.getSeconds() + 1) - const response = await config.api.auditLogs.search({ - startDate: start.toISOString(), - endDate: end.toISOString(), + it("should be able to search by time range (frozen)", async () => { + // this is frozen, only need to add 1 and minus 1 + const now = new Date() + const start = new Date() + start.setSeconds(now.getSeconds() - 1) + const end = new Date() + end.setSeconds(now.getSeconds() + 1) + const response = await config.api.auditLogs.search({ + startDate: start.toISOString(), + endDate: end.toISOString(), + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.timestamp).toBe(now.toISOString()) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.timestamp).toBe(now.toISOString()) - } - }) - it("should be able to search by user ID", async () => { - const userId = config.user!._id! - const response = await config.api.auditLogs.search({ - userIds: [userId], + it("should be able to search by user ID", async () => { + const userId = config.user!._id! + const response = await config.api.auditLogs.search({ + userIds: [userId], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.user._id).toBe(userId) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.user._id).toBe(userId) - } - }) - it("should be able to search by app ID", async () => { - const response = await config.api.auditLogs.search({ - appIds: [APP_ID], + it("should be able to search by app ID", async () => { + const response = await config.api.auditLogs.search({ + appIds: [APP_ID], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.app?._id).toBe(APP_ID) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.app?._id).toBe(APP_ID) - } - }) - it("should be able to search by full string", async () => { - const response = await config.api.auditLogs.search({ - fullSearch: "User", + it("should be able to search by full string", async () => { + const response = await config.api.auditLogs.search({ + fullSearch: "User", + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.name.includes("User")).toBe(true) + } }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.name.includes("User")).toBe(true) - } - }) - }) + } + ) }) From 4f6a09fb023b9b017863e9da96277f58bde9a977 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 17:14:03 +0100 Subject: [PATCH 08/24] Add utils --- .../src/events/processors/AuditLogsProcessor.ts | 4 ++++ packages/backend-core/src/events/processors/index.ts | 4 ++-- packages/backend-core/tests/core/utilities/index.ts | 1 + packages/backend-core/tests/core/utilities/queue.ts | 9 +++++++++ .../worker/src/api/routes/global/tests/auditLogs.spec.ts | 7 ++++++- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 packages/backend-core/tests/core/utilities/queue.ts diff --git a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts index 3dd2ab9d10..8dadf7b7a5 100644 --- a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts +++ b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts @@ -17,6 +17,10 @@ export default class AuditLogsProcessor implements EventProcessor { static auditLogsEnabled = false static auditLogQueue: BullQueue.Queue + get queue() { + return AuditLogsProcessor.auditLogQueue + } + // can't use constructor as need to return promise static init(fn: AuditLogFn) { AuditLogsProcessor.auditLogsEnabled = true diff --git a/packages/backend-core/src/events/processors/index.ts b/packages/backend-core/src/events/processors/index.ts index 6646764e47..f88cf43c97 100644 --- a/packages/backend-core/src/events/processors/index.ts +++ b/packages/backend-core/src/events/processors/index.ts @@ -5,8 +5,8 @@ import Processors from "./Processors" import { AuditLogFn } from "@budibase/types" export const analyticsProcessor = new AnalyticsProcessor() -const loggingProcessor = new LoggingProcessor() -const auditLogsProcessor = new AuditLogsProcessor() +export const loggingProcessor = new LoggingProcessor() +export const auditLogsProcessor = new AuditLogsProcessor() export function init(auditingFn: AuditLogFn) { return AuditLogsProcessor.init(auditingFn) diff --git a/packages/backend-core/tests/core/utilities/index.ts b/packages/backend-core/tests/core/utilities/index.ts index 787d69be2c..c3d81784c8 100644 --- a/packages/backend-core/tests/core/utilities/index.ts +++ b/packages/backend-core/tests/core/utilities/index.ts @@ -4,3 +4,4 @@ export { generator } from "./structures" export * as testContainerUtils from "./testContainerUtils" export * as utils from "./utils" export * from "./jestUtils" +export * as queue from "./queue" diff --git a/packages/backend-core/tests/core/utilities/queue.ts b/packages/backend-core/tests/core/utilities/queue.ts new file mode 100644 index 0000000000..49dd33ca29 --- /dev/null +++ b/packages/backend-core/tests/core/utilities/queue.ts @@ -0,0 +1,9 @@ +import { Queue } from "bull" + +export async function processMessages(queue: Queue) { + do { + await queue.whenCurrentJobsFinished() + } while (await queue.count()) + + await queue.whenCurrentJobsFinished() +} diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 55664f9893..601c3ffe62 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,4 +1,4 @@ -import { mocks, structures } from "@budibase/backend-core/tests" +import { mocks, structures, queue } from "@budibase/backend-core/tests" import { context, events, features } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" import { TestConfiguration } from "../../../../tests" @@ -54,6 +54,11 @@ describe("/api/global/auditlogs (%s)", () => { await context.doInAppContext(APP_ID, async () => { await events.app.created(structures.apps.app(APP_ID)) }) + + await queue.processMessages( + events.processors.auditLogsProcessor.queue + ) + // fetch the user created events const response = await config.api.auditLogs.search({ events: [Event.USER_CREATED], From b2488af6dac8d076726a7fb2a4ed2ca4006f35a4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 17:16:49 +0100 Subject: [PATCH 09/24] Use utils --- .../src/cache/tests/docWritethrough.spec.ts | 13 +++++++------ packages/backend-core/tests/core/utilities/index.ts | 1 - .../tests/core/utilities/utils/index.ts | 1 + .../tests/core/utilities/{ => utils}/queue.ts | 0 packages/pro | 2 +- .../src/api/routes/global/tests/auditLogs.spec.ts | 4 ++-- 6 files changed, 11 insertions(+), 10 deletions(-) rename packages/backend-core/tests/core/utilities/{ => utils}/queue.ts (100%) diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index cb8e2ff3fe..7e7e28bcb6 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -1,7 +1,12 @@ import tk from "timekeeper" import _ from "lodash" -import { DBTestConfiguration, generator, structures } from "../../../tests" +import { + DBTestConfiguration, + generator, + structures, + utils, +} from "../../../tests" import { getDB } from "../../db" import { @@ -13,11 +18,7 @@ import { const initialTime = Date.now() async function waitForQueueCompletion() { - do { - await DocWritethroughProcessor.queue.whenCurrentJobsFinished() - } while (await DocWritethroughProcessor.queue.count()) - - await DocWritethroughProcessor.queue.whenCurrentJobsFinished() + await utils.queue.processMessages(DocWritethroughProcessor.queue) } describe("docWritethrough", () => { diff --git a/packages/backend-core/tests/core/utilities/index.ts b/packages/backend-core/tests/core/utilities/index.ts index c3d81784c8..787d69be2c 100644 --- a/packages/backend-core/tests/core/utilities/index.ts +++ b/packages/backend-core/tests/core/utilities/index.ts @@ -4,4 +4,3 @@ export { generator } from "./structures" export * as testContainerUtils from "./testContainerUtils" export * as utils from "./utils" export * from "./jestUtils" -export * as queue from "./queue" diff --git a/packages/backend-core/tests/core/utilities/utils/index.ts b/packages/backend-core/tests/core/utilities/utils/index.ts index 41a249c7e6..3d28189c53 100644 --- a/packages/backend-core/tests/core/utilities/utils/index.ts +++ b/packages/backend-core/tests/core/utilities/utils/index.ts @@ -1 +1,2 @@ export * as time from "./time" +export * as queue from "./queue" diff --git a/packages/backend-core/tests/core/utilities/queue.ts b/packages/backend-core/tests/core/utilities/utils/queue.ts similarity index 100% rename from packages/backend-core/tests/core/utilities/queue.ts rename to packages/backend-core/tests/core/utilities/utils/queue.ts diff --git a/packages/pro b/packages/pro index 21e9cb4107..a35c797b1f 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 21e9cb41076c7064b6d5e16e0434705149440023 +Subproject commit a35c797b1f2cc067b6edd78d498bb2e239d677e1 diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 601c3ffe62..b3717439e7 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,4 +1,4 @@ -import { mocks, structures, queue } from "@budibase/backend-core/tests" +import { mocks, structures, utils } from "@budibase/backend-core/tests" import { context, events, features } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" import { TestConfiguration } from "../../../../tests" @@ -55,7 +55,7 @@ describe("/api/global/auditlogs (%s)", () => { await events.app.created(structures.apps.app(APP_ID)) }) - await queue.processMessages( + await utils.queue.processMessages( events.processors.auditLogsProcessor.queue ) From 02ebe9cb151123e131c8f7e4cb42f5b5942a02dc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 18:18:34 +0100 Subject: [PATCH 10/24] Do not close apps --- .../server/src/tests/utilities/TestConfiguration.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 5ed60a59b6..e37ed70ffb 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -237,11 +237,12 @@ export default class TestConfiguration { if (!this) { return } - if (this.server) { - this.server.close() - } else { - require("../../app").getServer().close() - } + + // if (this.server) { + // this.server.close() + // } else { + // require("../../app").getServer().close() + // } if (this.allApps) { cleanup(this.allApps.map(app => app.appId)) } From f7f056ef760a352fc8c91dcb2bd107d1d9f123bd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 18:23:57 +0100 Subject: [PATCH 11/24] Clean --- packages/backend-core/src/cache/tests/docWritethrough.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index 7e7e28bcb6..cc993c9a05 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -22,7 +22,7 @@ async function waitForQueueCompletion() { } describe("docWritethrough", () => { - beforeAll(async () => { + beforeAll(() => { init() }) From 505146f4b3c42a729ed080aa9b5f7881f108fc06 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 18:33:06 +0100 Subject: [PATCH 12/24] Add utils --- .../tests/core/utilities/index.ts | 1 + .../tests/core/utilities/queue.ts | 9 + .../api/routes/global/tests/auditLogs.spec.ts | 192 +++++++++--------- .../worker/src/tests/TestConfiguration.ts | 10 +- 4 files changed, 106 insertions(+), 106 deletions(-) create mode 100644 packages/backend-core/tests/core/utilities/queue.ts diff --git a/packages/backend-core/tests/core/utilities/index.ts b/packages/backend-core/tests/core/utilities/index.ts index 787d69be2c..c3d81784c8 100644 --- a/packages/backend-core/tests/core/utilities/index.ts +++ b/packages/backend-core/tests/core/utilities/index.ts @@ -4,3 +4,4 @@ export { generator } from "./structures" export * as testContainerUtils from "./testContainerUtils" export * as utils from "./utils" export * from "./jestUtils" +export * as queue from "./queue" diff --git a/packages/backend-core/tests/core/utilities/queue.ts b/packages/backend-core/tests/core/utilities/queue.ts new file mode 100644 index 0000000000..49dd33ca29 --- /dev/null +++ b/packages/backend-core/tests/core/utilities/queue.ts @@ -0,0 +1,9 @@ +import { Queue } from "bull" + +export async function processMessages(queue: Queue) { + do { + await queue.whenCurrentJobsFinished() + } while (await queue.count()) + + await queue.whenCurrentJobsFinished() +} diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index b3717439e7..d4d4fed120 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -12,121 +12,111 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" -describe("/api/global/auditlogs (%s)", () => { +describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { const config = new TestConfiguration() + let envCleanup: (() => void) | undefined beforeAll(async () => { + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: method === "sql", + }) await config.beforeAll() + await config.useNewTenant() }) afterAll(async () => { + envCleanup?.() await config.afterAll() }) - describe.each(["lucene", "sql"])( - "POST /api/global/auditlogs/search", - method => { - let envCleanup: (() => void) | undefined - - beforeAll(async () => { - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: method === "sql", - }) - await config.useNewTenant() - }) - - afterAll(() => { - envCleanup?.() - }) - - it("should be able to fire some events (create audit logs)", async () => { - await context.doInTenant(config.tenantId, async () => { - const userId = config.user!._id! - const identity = { - ...BASE_IDENTITY, - _id: userId, - tenantId: config.tenantId, - } - await context.doInIdentityContext(identity, async () => { - for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { - await events.user.created(structures.users.user()) - } - await context.doInAppContext(APP_ID, async () => { - await events.app.created(structures.apps.app(APP_ID)) - }) - - await utils.queue.processMessages( - events.processors.auditLogsProcessor.queue - ) - - // fetch the user created events - const response = await config.api.auditLogs.search({ - events: [Event.USER_CREATED], - }) - expect(response.data).toBeDefined() - // there will be an initial event which comes from the default user creation - expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) - }) - }) - }) - - it("should be able to search by event", async () => { - const response = await config.api.auditLogs.search({ - events: [Event.USER_CREATED], - }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.event).toBe(Event.USER_CREATED) - } - }) - - it("should be able to search by time range (frozen)", async () => { - // this is frozen, only need to add 1 and minus 1 - const now = new Date() - const start = new Date() - start.setSeconds(now.getSeconds() - 1) - const end = new Date() - end.setSeconds(now.getSeconds() + 1) - const response = await config.api.auditLogs.search({ - startDate: start.toISOString(), - endDate: end.toISOString(), - }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.timestamp).toBe(now.toISOString()) - } - }) - - it("should be able to search by user ID", async () => { + describe("POST /api/global/auditlogs/search", () => { + it("should be able to fire some events (create audit logs)", async () => { + await context.doInTenant(config.tenantId, async () => { const userId = config.user!._id! - const response = await config.api.auditLogs.search({ - userIds: [userId], - }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.user._id).toBe(userId) + const identity = { + ...BASE_IDENTITY, + _id: userId, + tenantId: config.tenantId, } - }) + await context.doInIdentityContext(identity, async () => { + for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { + await events.user.created(structures.users.user()) + } + await context.doInAppContext(APP_ID, async () => { + await events.app.created(structures.apps.app(APP_ID)) + }) - it("should be able to search by app ID", async () => { - const response = await config.api.auditLogs.search({ - appIds: [APP_ID], - }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.app?._id).toBe(APP_ID) - } - }) + await utils.queue.processMessages( + events.processors.auditLogsProcessor.queue + ) - it("should be able to search by full string", async () => { - const response = await config.api.auditLogs.search({ - fullSearch: "User", + // fetch the user created events + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data).toBeDefined() + // there will be an initial event which comes from the default user creation + expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) }) - expect(response.data.length).toBeGreaterThan(0) - for (let log of response.data) { - expect(log.name.includes("User")).toBe(true) - } }) - } - ) + }) + + it("should be able to search by event", async () => { + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.event).toBe(Event.USER_CREATED) + } + }) + + it("should be able to search by time range (frozen)", async () => { + // this is frozen, only need to add 1 and minus 1 + const now = new Date() + const start = new Date() + start.setSeconds(now.getSeconds() - 1) + const end = new Date() + end.setSeconds(now.getSeconds() + 1) + const response = await config.api.auditLogs.search({ + startDate: start.toISOString(), + endDate: end.toISOString(), + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.timestamp).toBe(now.toISOString()) + } + }) + + it("should be able to search by user ID", async () => { + const userId = config.user!._id! + const response = await config.api.auditLogs.search({ + userIds: [userId], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.user._id).toBe(userId) + } + }) + + it("should be able to search by app ID", async () => { + const response = await config.api.auditLogs.search({ + appIds: [APP_ID], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.app?._id).toBe(APP_ID) + } + }) + + it("should be able to search by full string", async () => { + const response = await config.api.auditLogs.search({ + fullSearch: "User", + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.name.includes("User")).toBe(true) + } + }) + }) }) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index c2cf005308..f93de73916 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -123,11 +123,11 @@ class TestConfiguration { } async afterAll() { - if (this.server) { - await this.server.close() - } else { - await require("../index").default.close() - } + // if (this.server) { + // await this.server.close() + // } else { + // await require("../index").default.close() + // } } // TENANCY From c5da1a06c60f54a1a0bafb6ea81b445888e1fda7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Nov 2024 23:08:37 +0100 Subject: [PATCH 13/24] Close server back --- .../src/tests/utilities/TestConfiguration.ts | 10 +++++----- .../src/api/routes/global/tests/auditLogs.spec.ts | 15 +++++++++++---- packages/worker/src/tests/TestConfiguration.ts | 12 ++++++------ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index e37ed70ffb..abecf6df44 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -238,11 +238,11 @@ export default class TestConfiguration { return } - // if (this.server) { - // this.server.close() - // } else { - // require("../../app").getServer().close() - // } + if (this.server) { + this.server.close() + } else { + require("../../app").getServer().close() + } if (this.allApps) { cleanup(this.allApps.map(app => app.appId)) } diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index d4d4fed120..03abd9e77e 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -12,21 +12,28 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" +const config = new TestConfiguration() + +beforeAll(async () => { + await config.beforeAll() +}) +afterAll(async () => { + await config.afterAll() +}) + describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { - const config = new TestConfiguration() let envCleanup: (() => void) | undefined beforeAll(async () => { envCleanup = features.testutils.setFeatureFlags("*", { SQS: method === "sql", }) - await config.beforeAll() + await config.useNewTenant() }) - afterAll(async () => { + afterAll(() => { envCleanup?.() - await config.afterAll() }) describe("POST /api/global/auditlogs/search", () => { diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index f93de73916..440d6dc776 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -12,7 +12,7 @@ dbConfig.init() import env from "../environment" import * as controllers from "./controllers" -const supertest = require("supertest") +import supertest from "supertest" import { Config } from "../constants" import { @@ -123,11 +123,11 @@ class TestConfiguration { } async afterAll() { - // if (this.server) { - // await this.server.close() - // } else { - // await require("../index").default.close() - // } + if (this.server) { + await this.server.close() + } else { + await require("../index").default.close() + } } // TENANCY From a7bb89a710cb347f17331b5cfd7d54d2ab5992a8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 10:28:24 +0100 Subject: [PATCH 14/24] Add useRealQueues helpers --- .../tests/core/utilities/utils/queue.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/backend-core/tests/core/utilities/utils/queue.ts b/packages/backend-core/tests/core/utilities/utils/queue.ts index 49dd33ca29..b5ffdd79b1 100644 --- a/packages/backend-core/tests/core/utilities/utils/queue.ts +++ b/packages/backend-core/tests/core/utilities/utils/queue.ts @@ -1,4 +1,26 @@ import { Queue } from "bull" +import { GenericContainer, Wait } from "testcontainers" + +export async function useRealQueues() { + const redis = await new GenericContainer("redis") + .withExposedPorts(6379) + .withEnvironment({}) + .withLabels({ "com.budibase": "true" }) + .withReuse() + .withWaitStrategy( + Wait.forSuccessfulCommand( + `until redis-cli ping | grep -q PONG; do + echo "Waiting for Redis to be ready..." + sleep 1 + done + echo "Redis is ready!"` + ).withStartupTimeout(10000) + ) + .start() + + const port = redis.getMappedPort(6379) + process.env.BULL_TEST_REDIS = `http://127.0.0.1:${port}` +} export async function processMessages(queue: Queue) { do { From 866bef81454cc03f47559385ddb4c81095f4c721 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 10:38:34 +0100 Subject: [PATCH 15/24] Make use of helpers --- .../src/cache/tests/docWritethrough.spec.ts | 2 ++ packages/backend-core/src/queue/queue.ts | 13 ++++++++++--- .../tests/core/utilities/utils/queue.ts | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts index cc993c9a05..b72651e21f 100644 --- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -21,6 +21,8 @@ async function waitForQueueCompletion() { await utils.queue.processMessages(DocWritethroughProcessor.queue) } +beforeAll(() => utils.queue.useRealQueues()) + describe("docWritethrough", () => { beforeAll(() => { init() diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index f633d0885e..f5d710f02d 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -15,7 +15,7 @@ const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() // cleanup the queue every 60 seconds const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() -let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] +let QUEUES: BullQueue.Queue[] = [] let cleanupInterval: NodeJS.Timeout async function cleanup() { @@ -45,11 +45,18 @@ export function createQueue( if (opts.jobOptions) { queueConfig.defaultJobOptions = opts.jobOptions } - let queue: any + let queue: BullQueue.Queue if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) + } else if ( + process.env.BULL_TEST_REDIS_PORT && + !isNaN(+process.env.BULL_TEST_REDIS_PORT) + ) { + queue = new BullQueue(jobQueue, { + redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT }, + }) } else { - queue = new InMemoryQueue(jobQueue, queueConfig) + queue = new InMemoryQueue(jobQueue, queueConfig) as any } addListeners(queue, jobQueue, opts?.removeStalledCb) QUEUES.push(queue) diff --git a/packages/backend-core/tests/core/utilities/utils/queue.ts b/packages/backend-core/tests/core/utilities/utils/queue.ts index b5ffdd79b1..57ad6dcf27 100644 --- a/packages/backend-core/tests/core/utilities/utils/queue.ts +++ b/packages/backend-core/tests/core/utilities/utils/queue.ts @@ -19,7 +19,7 @@ export async function useRealQueues() { .start() const port = redis.getMappedPort(6379) - process.env.BULL_TEST_REDIS = `http://127.0.0.1:${port}` + process.env.BULL_TEST_REDIS_PORT = port.toString() } export async function processMessages(queue: Queue) { From 9f215d82c61700cbcf5673fbc156228c35775081 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 10:45:00 +0100 Subject: [PATCH 16/24] Fix pro test --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index a35c797b1f..80770215c6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit a35c797b1f2cc067b6edd78d498bb2e239d677e1 +Subproject commit 80770215c6159e4d47f3529fd02e74bc8ad07543 From db48e913e9cf49f2da97b8462404b33768e2a58b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 10:50:17 +0100 Subject: [PATCH 17/24] Undo auditlog.spec test changes --- .../api/routes/global/tests/auditLogs.spec.ts | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 03abd9e77e..b540836583 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,4 +1,4 @@ -import { mocks, structures, utils } from "@budibase/backend-core/tests" +import { mocks, structures } from "@budibase/backend-core/tests" import { context, events, features } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" import { TestConfiguration } from "../../../../tests" @@ -12,28 +12,20 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" -const config = new TestConfiguration() - -beforeAll(async () => { - await config.beforeAll() -}) -afterAll(async () => { - await config.afterAll() -}) - describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { + const config = new TestConfiguration() let envCleanup: (() => void) | undefined beforeAll(async () => { envCleanup = features.testutils.setFeatureFlags("*", { SQS: method === "sql", }) - - await config.useNewTenant() + await config.beforeAll() }) - afterAll(() => { + afterAll(async () => { envCleanup?.() + await config.afterAll() }) describe("POST /api/global/auditlogs/search", () => { @@ -52,11 +44,6 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { await context.doInAppContext(APP_ID, async () => { await events.app.created(structures.apps.app(APP_ID)) }) - - await utils.queue.processMessages( - events.processors.auditLogsProcessor.queue - ) - // fetch the user created events const response = await config.api.auditLogs.search({ events: [Event.USER_CREATED], From 7126534bd2c430823f1d4a0894d9e1ca858ccae8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 10:53:02 +0100 Subject: [PATCH 18/24] Undo changes --- .../backend-core/src/events/processors/AuditLogsProcessor.ts | 4 ---- packages/backend-core/src/events/processors/index.ts | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts index 8dadf7b7a5..3dd2ab9d10 100644 --- a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts +++ b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts @@ -17,10 +17,6 @@ export default class AuditLogsProcessor implements EventProcessor { static auditLogsEnabled = false static auditLogQueue: BullQueue.Queue - get queue() { - return AuditLogsProcessor.auditLogQueue - } - // can't use constructor as need to return promise static init(fn: AuditLogFn) { AuditLogsProcessor.auditLogsEnabled = true diff --git a/packages/backend-core/src/events/processors/index.ts b/packages/backend-core/src/events/processors/index.ts index f88cf43c97..6646764e47 100644 --- a/packages/backend-core/src/events/processors/index.ts +++ b/packages/backend-core/src/events/processors/index.ts @@ -5,8 +5,8 @@ import Processors from "./Processors" import { AuditLogFn } from "@budibase/types" export const analyticsProcessor = new AnalyticsProcessor() -export const loggingProcessor = new LoggingProcessor() -export const auditLogsProcessor = new AuditLogsProcessor() +const loggingProcessor = new LoggingProcessor() +const auditLogsProcessor = new AuditLogsProcessor() export function init(auditingFn: AuditLogFn) { return AuditLogsProcessor.init(auditingFn) From 1ad8e0d4584d40dfd7695051c10cba9aa0af1627 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 12:12:23 +0100 Subject: [PATCH 19/24] PR comments --- .../backend-core/tests/core/utilities/utils/queue.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/utils/queue.ts b/packages/backend-core/tests/core/utilities/utils/queue.ts index 57ad6dcf27..ab887e6267 100644 --- a/packages/backend-core/tests/core/utilities/utils/queue.ts +++ b/packages/backend-core/tests/core/utilities/utils/queue.ts @@ -4,17 +4,8 @@ import { GenericContainer, Wait } from "testcontainers" export async function useRealQueues() { const redis = await new GenericContainer("redis") .withExposedPorts(6379) - .withEnvironment({}) - .withLabels({ "com.budibase": "true" }) - .withReuse() .withWaitStrategy( - Wait.forSuccessfulCommand( - `until redis-cli ping | grep -q PONG; do - echo "Waiting for Redis to be ready..." - sleep 1 - done - echo "Redis is ready!"` - ).withStartupTimeout(10000) + Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000) ) .start() From 518191b882e5e20effd9226af4bda3064f8e91af Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 12:15:45 +0100 Subject: [PATCH 20/24] Clean --- packages/backend-core/src/queue/inMemoryQueue.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 1d8544828d..dd8d3daa37 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -1,5 +1,5 @@ import events from "events" -import { newid, timeout } from "../utils" +import { newid } from "../utils" import { Queue, QueueOptions, JobOptions } from "./queue" interface JobMessage { @@ -184,16 +184,6 @@ class InMemoryQueue implements Partial { // do nothing return this as any } - - async waitForCompletion() { - do { - await timeout(50) - } while (this.hasRunningJobs()) - } - - hasRunningJobs() { - return this._addCount > this._runCount - } } export default InMemoryQueue From 28724602a236cfec00f37bcf7e886ae759af4dc8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 7 Nov 2024 13:12:03 +0100 Subject: [PATCH 21/24] use startContainer utils --- .../core/utilities/testContainerUtils.ts | 57 ++++++++++++++++++ .../tests/core/utilities/utils/queue.ts | 19 +++--- .../src/integrations/tests/utils/index.ts | 59 +------------------ 3 files changed, 71 insertions(+), 64 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 1a25bb28f4..71d7fa32db 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -1,4 +1,6 @@ import { execSync } from "child_process" +import { cloneDeep } from "lodash" +import { GenericContainer, StartedTestContainer } from "testcontainers" const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g") @@ -106,3 +108,58 @@ export function setupEnv(...envs: any[]) { } } } + +export async function startContainer(container: GenericContainer) { + const imageName = (container as any).imageName.string as string + let key: string = imageName + if (imageName.includes("@sha256")) { + key = imageName.split("@")[0] + } + key = key.replace(/\//g, "-").replace(/:/g, "-") + + container = container + .withReuse() + .withLabels({ "com.budibase": "true" }) + .withName(`${key}_testcontainer`) + + let startedContainer: StartedTestContainer | undefined = undefined + let lastError = undefined + for (let i = 0; i < 10; i++) { + try { + // container.start() is not an idempotent operation, calling `start` + // modifies the internal state of a GenericContainer instance such that + // the hash it uses to determine reuse changes. We need to clone the + // container before calling start to ensure that we're using the same + // reuse hash every time. + const containerCopy = cloneDeep(container) + startedContainer = await containerCopy.start() + lastError = undefined + break + } catch (e: any) { + lastError = e + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } + + if (!startedContainer) { + if (lastError) { + throw lastError + } + throw new Error(`failed to start container: ${imageName}`) + } + + const info = getContainerById(startedContainer.getId()) + if (!info) { + throw new Error("Container not found") + } + + // Some Docker runtimes, when you expose a port, will bind it to both + // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 + // addresses are not shared, and testcontainers will sometimes give you back + // the ipv6 port. There's no way to know that this has happened, and if you + // try to then connect to `localhost:port` you may attempt to bind to the v4 + // address which could be unbound or even an entirely different container. For + // that reason, we don't use testcontainers' `getExposedPort` function, + // preferring instead our own method that guaranteed v4 ports. + return getExposedV4Ports(info) +} diff --git a/packages/backend-core/tests/core/utilities/utils/queue.ts b/packages/backend-core/tests/core/utilities/utils/queue.ts index ab887e6267..3ad7d6b4b4 100644 --- a/packages/backend-core/tests/core/utilities/utils/queue.ts +++ b/packages/backend-core/tests/core/utilities/utils/queue.ts @@ -1,15 +1,20 @@ import { Queue } from "bull" import { GenericContainer, Wait } from "testcontainers" +import { startContainer } from "../testContainerUtils" export async function useRealQueues() { - const redis = await new GenericContainer("redis") - .withExposedPorts(6379) - .withWaitStrategy( - Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000) - ) - .start() + const ports = await startContainer( + new GenericContainer("redis") + .withExposedPorts(6379) + .withWaitStrategy( + Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000) + ) + ) - const port = redis.getMappedPort(6379) + const port = ports.find(x => x.container === 6379)?.host + if (!port) { + throw new Error("Redis port not found") + } process.env.BULL_TEST_REDIS_PORT = port.toString() } diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index b6f8b5b92a..6313556df7 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -6,12 +6,12 @@ import * as mysql from "./mysql" import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" -import { GenericContainer, StartedTestContainer } from "testcontainers" import { testContainerUtils } from "@budibase/backend-core/tests" -import cloneDeep from "lodash/cloneDeep" export type DatasourceProvider = () => Promise +export const { startContainer } = testContainerUtils + export enum DatabaseName { POSTGRES = "postgres", MONGODB = "mongodb", @@ -71,58 +71,3 @@ export async function knexClient(ds: Datasource) { } } } - -export async function startContainer(container: GenericContainer) { - const imageName = (container as any).imageName.string as string - let key: string = imageName - if (imageName.includes("@sha256")) { - key = imageName.split("@")[0] - } - key = key.replaceAll("/", "-").replaceAll(":", "-") - - container = container - .withReuse() - .withLabels({ "com.budibase": "true" }) - .withName(`${key}_testcontainer`) - - let startedContainer: StartedTestContainer | undefined = undefined - let lastError = undefined - for (let i = 0; i < 10; i++) { - try { - // container.start() is not an idempotent operation, calling `start` - // modifies the internal state of a GenericContainer instance such that - // the hash it uses to determine reuse changes. We need to clone the - // container before calling start to ensure that we're using the same - // reuse hash every time. - const containerCopy = cloneDeep(container) - startedContainer = await containerCopy.start() - lastError = undefined - break - } catch (e: any) { - lastError = e - await new Promise(resolve => setTimeout(resolve, 1000)) - } - } - - if (!startedContainer) { - if (lastError) { - throw lastError - } - throw new Error(`failed to start container: ${imageName}`) - } - - const info = testContainerUtils.getContainerById(startedContainer.getId()) - if (!info) { - throw new Error("Container not found") - } - - // Some Docker runtimes, when you expose a port, will bind it to both - // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6 - // addresses are not shared, and testcontainers will sometimes give you back - // the ipv6 port. There's no way to know that this has happened, and if you - // try to then connect to `localhost:port` you may attempt to bind to the v4 - // address which could be unbound or even an entirely different container. For - // that reason, we don't use testcontainers' `getExposedPort` function, - // preferring instead our own method that guaranteed v4 ports. - return testContainerUtils.getExposedV4Ports(info) -} From 33656a4ae5756fc6894a031dc27ec544618ef9f4 Mon Sep 17 00:00:00 2001 From: andz-bb Date: Fri, 8 Nov 2024 14:04:51 +0000 Subject: [PATCH 22/24] prevent attachment column updates when importing budibase templates --- packages/server/src/api/controllers/application.ts | 6 +++++- packages/server/src/sdk/app/backups/imports.ts | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index e7d0ed7ba7..101257c321 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -153,7 +153,11 @@ async function createInstance(appId: string, template: AppTemplate) { await createAllSearchIndex() if (template && template.useTemplate) { - await sdk.backups.importApp(appId, db, template) + const opts = { + importObjStoreContents: true, + updateAttachmentColumns: !template.key, // preserve attachments when using Budibase templates + } + await sdk.backups.importApp(appId, db, template, opts) } else { // create the users table await db.put(USERS_TABLE_SCHEMA) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index f69fb9f5c8..edc8d9ea6f 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -170,7 +170,10 @@ export async function importApp( appId: string, db: Database, template: TemplateType, - opts: { importObjStoreContents: boolean } = { importObjStoreContents: true } + opts: { + importObjStoreContents: boolean + updateAttachmentColumns?: boolean + } = { importObjStoreContents: true, updateAttachmentColumns: true } ) { let prodAppId = dbCore.getProdAppID(appId) let dbStream: any @@ -219,7 +222,9 @@ export async function importApp( if (!ok) { throw "Error loading database dump from template." } - await updateAttachmentColumns(prodAppId, db) + if (opts.updateAttachmentColumns) { + await updateAttachmentColumns(prodAppId, db) + } await updateAutomations(prodAppId, db) // clear up afterward if (tmpPath) { From 932b119a66c246aad0746fc986d2d12524183148 Mon Sep 17 00:00:00 2001 From: andz-bb Date: Fri, 8 Nov 2024 14:24:31 +0000 Subject: [PATCH 23/24] make opt not optional --- packages/server/src/sdk/app/applications/import.ts | 1 + packages/server/src/sdk/app/backups/imports.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts index f712548fcb..c63708f42e 100644 --- a/packages/server/src/sdk/app/applications/import.ts +++ b/packages/server/src/sdk/app/applications/import.ts @@ -123,6 +123,7 @@ export async function updateWithExport( // don't need obj store, the existing app already has everything we need await backups.importApp(devId, tempDb, template, { importObjStoreContents: false, + updateAttachmentColumns: true, }) const newMetadata = await getNewAppMetadata(tempDb, appDb) // get the documents to copy diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index edc8d9ea6f..3ec0e8833b 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -172,7 +172,7 @@ export async function importApp( template: TemplateType, opts: { importObjStoreContents: boolean - updateAttachmentColumns?: boolean + updateAttachmentColumns: boolean } = { importObjStoreContents: true, updateAttachmentColumns: true } ) { let prodAppId = dbCore.getProdAppID(appId) From 1c2f009f6e7198ab1db996c2fdc46800e8d517ec Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 8 Nov 2024 14:49:03 +0000 Subject: [PATCH 24/24] Bump version to 3.2.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 19b603b1cd..8d45ff71ac 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.0", + "version": "3.2.1", "npmClient": "yarn", "packages": [ "packages/*",