From 41d99f6a582500a500b2246968d624411e697929 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Oct 2023 17:41:08 +0000 Subject: [PATCH 1/3] Updating app backup exports to not include automation logs as these bloat the backups. --- packages/backend-core/__mocks__/aws-sdk.ts | 1 + .../tests/core/utilities/mocks/date.ts | 5 ++ packages/pro | 2 +- packages/server/__mocks__/aws-sdk.ts | 7 +++ .../src/api/routes/tests/backup.spec.ts | 37 ++++++++++---- .../server/src/sdk/app/backups/exports.ts | 6 ++- .../src/tests/utilities/TestConfiguration.ts | 5 +- .../server/src/tests/utilities/api/backup.ts | 49 +++++++++++++++++++ .../server/src/tests/utilities/api/index.ts | 3 ++ packages/types/src/api/web/app/backup.ts | 5 ++ 10 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 packages/server/src/tests/utilities/api/backup.ts diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts index b8d91dbaa9..e3be511d08 100644 --- a/packages/backend-core/__mocks__/aws-sdk.ts +++ b/packages/backend-core/__mocks__/aws-sdk.ts @@ -3,6 +3,7 @@ const mockS3 = { deleteObject: jest.fn().mockReturnThis(), deleteObjects: jest.fn().mockReturnThis(), createBucket: jest.fn().mockReturnThis(), + getObject: jest.fn().mockReturnThis(), listObject: jest.fn().mockReturnThis(), getSignedUrl: jest.fn((operation: string, params: any) => { return `http://s3.example.com/${params.Bucket}/${params.Key}` diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index f580b68349..9f33058d29 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,2 +1,7 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") + +export const MOCK_DATE_PLUS = (ms: number) => { + return new Date(Date.now() + ms) +} + export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/pro b/packages/pro index 3820c0c93a..132b080643 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 +Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 8a66f0e213..fa6d099f56 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -70,6 +70,13 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.getObject = jest.fn( + response({ + Body: "", + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 92e0176060..0c62b32155 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -5,6 +5,8 @@ import sdk from "../../../sdk" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import { mocks } from "@budibase/backend-core/tests" +mocks.licenses.useBackups() + describe("/backups", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -15,13 +17,13 @@ describe("/backups", () => { await config.init() }) - describe("exportAppDump", () => { + describe("/api/backups/export", () => { it("should be able to export app", async () => { - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) - .expect(200) - expect(res.headers["content-type"]).toEqual("application/gzip") + const { body, headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) + expect(body instanceof Buffer).toBe(true) + expect(headers["content-type"]).toEqual("application/gzip") expect(events.app.exported).toBeCalledTimes(1) }) @@ -36,11 +38,11 @@ describe("/backups", () => { it("should infer the app name from the app", async () => { tk.freeze(mocks.date.MOCK_DATE) - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) + const { headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) - expect(res.headers["content-disposition"]).toEqual( + expect(headers["content-disposition"]).toEqual( `attachment; filename="${ config.getApp()!.name }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"` @@ -48,6 +50,21 @@ describe("/backups", () => { }) }) + describe("/api/backups/import", () => { + it("should be able to import an app", async () => { + const appId = config.getAppId()! + const automation = await config.createAutomation() + await config.createAutomationLog(automation, appId) + await config.createScreen() + const exportRes = await config.api.backup.createBackup(appId) + expect(exportRes.backupId).toBeDefined() + const importRes = await config.api.backup.importBackup( + appId, + exportRes.backupId + ) + }) + }) + describe("calculateBackupStats", () => { it("should be able to calculate the backup statistics", async () => { await config.createAutomation() diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index d5ea31cdf5..6432bd2e0d 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -84,7 +84,11 @@ export async function exportDB( } function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] + const ids = [ + USER_METDATA_PREFIX, + LINK_USER_METADATA_PREFIX, + AUTOMATION_LOG_PREFIX, + ] if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cec8c8aa12..869a42012a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -774,8 +774,9 @@ class TestConfiguration { // AUTOMATION LOG - async createAutomationLog(automation: Automation) { - return await context.doInAppContext(this.getProdAppId(), async () => { + async createAutomationLog(automation: Automation, appId?: string) { + appId = appId || this.getProdAppId() + return await context.doInAppContext(appId!, async () => { return await pro.sdk.automations.logs.storeLog( automation, basicAutomationResults(automation._id!) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts new file mode 100644 index 0000000000..4cee19fdc6 --- /dev/null +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -0,0 +1,49 @@ +import { + CreateAppBackupResponse, + ImportAppBackupResponse, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" +import tk from "timekeeper" +import { mocks } from "@budibase/backend-core/tests" + +export class BackupAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + exportBasicBackup = async (appId: string) => { + const result = await this.request + .post(`/api/backups/export?appId=${appId}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /application\/gzip/) + .expect(200) + return { + body: result.body as Buffer, + headers: result.headers, + } + } + + createBackup = async (appId: string) => { + tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) + const result = await this.request + .post(`/api/apps/${appId}/backups`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as CreateAppBackupResponse + } + + importBackup = async ( + appId: string, + backupId: string + ): Promise => { + tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) + const result = await this.request + .post(`/api/apps/${appId}/backups/${backupId}/import`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as ImportAppBackupResponse + } +} diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fce8237760..c344068295 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource" import { LegacyViewAPI } from "./legacyView" import { ScreenAPI } from "./screen" import { ApplicationAPI } from "./application" +import { BackupAPI } from "./backup" export default class API { table: TableAPI @@ -17,6 +18,7 @@ export default class API { datasource: DatasourceAPI screen: ScreenAPI application: ApplicationAPI + backup: BackupAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -27,5 +29,6 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) + this.backup = new BackupAPI(config) } } diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index c9a8d07f5e..f77707e9c6 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -20,3 +20,8 @@ export interface CreateAppBackupResponse { export interface UpdateAppBackupRequest { name: string } + +export interface ImportAppBackupResponse { + restoreId: string + message: string +} From 15b1f3efe63fb8d4eceb3dc66d4c3ed3f90fd6ee Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 10:51:46 +0000 Subject: [PATCH 2/3] Removing duplicate filtering of automation logs. --- packages/server/src/sdk/app/backups/exports.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 6432bd2e0d..c349dcb927 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -26,7 +26,6 @@ export interface DBDumpOpts { export interface ExportOpts extends DBDumpOpts { tar?: boolean excludeRows?: boolean - excludeLogs?: boolean encryptPassword?: string } @@ -83,7 +82,7 @@ export async function exportDB( }) } -function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { +function defineFilter(excludeRows?: boolean) { const ids = [ USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, @@ -92,9 +91,6 @@ function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } - if (excludeLogs) { - ids.push(AUTOMATION_LOG_PREFIX) - } return (doc: any) => !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) } @@ -122,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { fs.writeFileSync(join(tmpPath, path), contents) } } - // get all of the files + // get all the files else { tmpPath = await objectStore.retrieveDirectory( ObjectStoreBuckets.APPS, @@ -145,7 +141,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { // enforce an export of app DB to the tmp path const dbPath = join(tmpPath, DB_EXPORT_FILE) await exportDB(appId, { - filter: defineFilter(config?.excludeRows, config?.excludeLogs), + filter: defineFilter(config?.excludeRows), exportPath: dbPath, }) @@ -195,7 +191,6 @@ export async function streamExportApp({ }) { const tmpPath = await exportApp(appId, { excludeRows, - excludeLogs: true, tar: true, encryptPassword, }) From 88cc8a19eef23af0ace19b1fab3014d70cc44ba5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 11:19:53 +0000 Subject: [PATCH 3/3] Resetting timekeeper and resetting pro back to normal. --- packages/backend-core/tests/core/utilities/mocks/date.ts | 4 ---- packages/pro | 2 +- packages/server/src/api/routes/tests/backup.spec.ts | 1 + packages/server/src/tests/utilities/api/backup.ts | 4 ---- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index 9f33058d29..1e6d105d93 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,7 +1,3 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") -export const MOCK_DATE_PLUS = (ms: number) => { - return new Date(Date.now() + ms) -} - export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/pro b/packages/pro index 132b080643..3820c0c93a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f +Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 0c62b32155..d12b5e1507 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -14,6 +14,7 @@ describe("/backups", () => { afterAll(setup.afterAll) beforeEach(async () => { + tk.reset() await config.init() }) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts index 4cee19fdc6..f9cbc7086e 100644 --- a/packages/server/src/tests/utilities/api/backup.ts +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -4,8 +4,6 @@ import { } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" -import tk from "timekeeper" -import { mocks } from "@budibase/backend-core/tests" export class BackupAPI extends TestAPI { constructor(config: TestConfiguration) { @@ -25,7 +23,6 @@ export class BackupAPI extends TestAPI { } createBackup = async (appId: string) => { - tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) const result = await this.request .post(`/api/apps/${appId}/backups`) .set(this.config.defaultHeaders()) @@ -38,7 +35,6 @@ export class BackupAPI extends TestAPI { appId: string, backupId: string ): Promise => { - tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) const result = await this.request .post(`/api/apps/${appId}/backups/${backupId}/import`) .set(this.config.defaultHeaders())