From d9f52295ee88dced451edabc7d32bdd07df63815 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 27 Oct 2023 11:36:45 +0100 Subject: [PATCH 1/7] Moved the description block out of the header to ensure it's visible for the view mode --- .../components/app/blocks/form/InnerFormBlock.svelte | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte index e65d2cf90b..f7e9a0d2ed 100644 --- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte @@ -220,15 +220,11 @@ {/if} - {#if description} - - {/if} {/if} + {#if description} + + {/if} {#key fields} {#each fields as field, idx} From 41d99f6a582500a500b2246968d624411e697929 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Oct 2023 17:41:08 +0000 Subject: [PATCH 2/7] Updating app backup exports to not include automation logs as these bloat the backups. --- packages/backend-core/__mocks__/aws-sdk.ts | 1 + .../tests/core/utilities/mocks/date.ts | 5 ++ packages/pro | 2 +- packages/server/__mocks__/aws-sdk.ts | 7 +++ .../src/api/routes/tests/backup.spec.ts | 37 ++++++++++---- .../server/src/sdk/app/backups/exports.ts | 6 ++- .../src/tests/utilities/TestConfiguration.ts | 5 +- .../server/src/tests/utilities/api/backup.ts | 49 +++++++++++++++++++ .../server/src/tests/utilities/api/index.ts | 3 ++ packages/types/src/api/web/app/backup.ts | 5 ++ 10 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 packages/server/src/tests/utilities/api/backup.ts diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts index b8d91dbaa9..e3be511d08 100644 --- a/packages/backend-core/__mocks__/aws-sdk.ts +++ b/packages/backend-core/__mocks__/aws-sdk.ts @@ -3,6 +3,7 @@ const mockS3 = { deleteObject: jest.fn().mockReturnThis(), deleteObjects: jest.fn().mockReturnThis(), createBucket: jest.fn().mockReturnThis(), + getObject: jest.fn().mockReturnThis(), listObject: jest.fn().mockReturnThis(), getSignedUrl: jest.fn((operation: string, params: any) => { return `http://s3.example.com/${params.Bucket}/${params.Key}` diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index f580b68349..9f33058d29 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,2 +1,7 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") + +export const MOCK_DATE_PLUS = (ms: number) => { + return new Date(Date.now() + ms) +} + export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/pro b/packages/pro index 3820c0c93a..132b080643 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 +Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 8a66f0e213..fa6d099f56 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -70,6 +70,13 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.getObject = jest.fn( + response({ + Body: "", + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 92e0176060..0c62b32155 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -5,6 +5,8 @@ import sdk from "../../../sdk" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import { mocks } from "@budibase/backend-core/tests" +mocks.licenses.useBackups() + describe("/backups", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -15,13 +17,13 @@ describe("/backups", () => { await config.init() }) - describe("exportAppDump", () => { + describe("/api/backups/export", () => { it("should be able to export app", async () => { - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) - .expect(200) - expect(res.headers["content-type"]).toEqual("application/gzip") + const { body, headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) + expect(body instanceof Buffer).toBe(true) + expect(headers["content-type"]).toEqual("application/gzip") expect(events.app.exported).toBeCalledTimes(1) }) @@ -36,11 +38,11 @@ describe("/backups", () => { it("should infer the app name from the app", async () => { tk.freeze(mocks.date.MOCK_DATE) - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) + const { headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) - expect(res.headers["content-disposition"]).toEqual( + expect(headers["content-disposition"]).toEqual( `attachment; filename="${ config.getApp()!.name }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"` @@ -48,6 +50,21 @@ describe("/backups", () => { }) }) + describe("/api/backups/import", () => { + it("should be able to import an app", async () => { + const appId = config.getAppId()! + const automation = await config.createAutomation() + await config.createAutomationLog(automation, appId) + await config.createScreen() + const exportRes = await config.api.backup.createBackup(appId) + expect(exportRes.backupId).toBeDefined() + const importRes = await config.api.backup.importBackup( + appId, + exportRes.backupId + ) + }) + }) + describe("calculateBackupStats", () => { it("should be able to calculate the backup statistics", async () => { await config.createAutomation() diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index d5ea31cdf5..6432bd2e0d 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -84,7 +84,11 @@ export async function exportDB( } function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] + const ids = [ + USER_METDATA_PREFIX, + LINK_USER_METADATA_PREFIX, + AUTOMATION_LOG_PREFIX, + ] if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cec8c8aa12..869a42012a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -774,8 +774,9 @@ class TestConfiguration { // AUTOMATION LOG - async createAutomationLog(automation: Automation) { - return await context.doInAppContext(this.getProdAppId(), async () => { + async createAutomationLog(automation: Automation, appId?: string) { + appId = appId || this.getProdAppId() + return await context.doInAppContext(appId!, async () => { return await pro.sdk.automations.logs.storeLog( automation, basicAutomationResults(automation._id!) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts new file mode 100644 index 0000000000..4cee19fdc6 --- /dev/null +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -0,0 +1,49 @@ +import { + CreateAppBackupResponse, + ImportAppBackupResponse, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" +import tk from "timekeeper" +import { mocks } from "@budibase/backend-core/tests" + +export class BackupAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + exportBasicBackup = async (appId: string) => { + const result = await this.request + .post(`/api/backups/export?appId=${appId}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /application\/gzip/) + .expect(200) + return { + body: result.body as Buffer, + headers: result.headers, + } + } + + createBackup = async (appId: string) => { + tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) + const result = await this.request + .post(`/api/apps/${appId}/backups`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as CreateAppBackupResponse + } + + importBackup = async ( + appId: string, + backupId: string + ): Promise => { + tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) + const result = await this.request + .post(`/api/apps/${appId}/backups/${backupId}/import`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as ImportAppBackupResponse + } +} diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fce8237760..c344068295 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource" import { LegacyViewAPI } from "./legacyView" import { ScreenAPI } from "./screen" import { ApplicationAPI } from "./application" +import { BackupAPI } from "./backup" export default class API { table: TableAPI @@ -17,6 +18,7 @@ export default class API { datasource: DatasourceAPI screen: ScreenAPI application: ApplicationAPI + backup: BackupAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -27,5 +29,6 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) + this.backup = new BackupAPI(config) } } diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index c9a8d07f5e..f77707e9c6 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -20,3 +20,8 @@ export interface CreateAppBackupResponse { export interface UpdateAppBackupRequest { name: string } + +export interface ImportAppBackupResponse { + restoreId: string + message: string +} From 15b1f3efe63fb8d4eceb3dc66d4c3ed3f90fd6ee Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 10:51:46 +0000 Subject: [PATCH 3/7] Removing duplicate filtering of automation logs. --- packages/server/src/sdk/app/backups/exports.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 6432bd2e0d..c349dcb927 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -26,7 +26,6 @@ export interface DBDumpOpts { export interface ExportOpts extends DBDumpOpts { tar?: boolean excludeRows?: boolean - excludeLogs?: boolean encryptPassword?: string } @@ -83,7 +82,7 @@ export async function exportDB( }) } -function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { +function defineFilter(excludeRows?: boolean) { const ids = [ USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, @@ -92,9 +91,6 @@ function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } - if (excludeLogs) { - ids.push(AUTOMATION_LOG_PREFIX) - } return (doc: any) => !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) } @@ -122,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { fs.writeFileSync(join(tmpPath, path), contents) } } - // get all of the files + // get all the files else { tmpPath = await objectStore.retrieveDirectory( ObjectStoreBuckets.APPS, @@ -145,7 +141,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { // enforce an export of app DB to the tmp path const dbPath = join(tmpPath, DB_EXPORT_FILE) await exportDB(appId, { - filter: defineFilter(config?.excludeRows, config?.excludeLogs), + filter: defineFilter(config?.excludeRows), exportPath: dbPath, }) @@ -195,7 +191,6 @@ export async function streamExportApp({ }) { const tmpPath = await exportApp(appId, { excludeRows, - excludeLogs: true, tar: true, encryptPassword, }) From 88cc8a19eef23af0ace19b1fab3014d70cc44ba5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 11:19:53 +0000 Subject: [PATCH 4/7] Resetting timekeeper and resetting pro back to normal. --- packages/backend-core/tests/core/utilities/mocks/date.ts | 4 ---- packages/pro | 2 +- packages/server/src/api/routes/tests/backup.spec.ts | 1 + packages/server/src/tests/utilities/api/backup.ts | 4 ---- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index 9f33058d29..1e6d105d93 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,7 +1,3 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") -export const MOCK_DATE_PLUS = (ms: number) => { - return new Date(Date.now() + ms) -} - export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/pro b/packages/pro index 132b080643..3820c0c93a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f +Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 0c62b32155..d12b5e1507 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -14,6 +14,7 @@ describe("/backups", () => { afterAll(setup.afterAll) beforeEach(async () => { + tk.reset() await config.init() }) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts index 4cee19fdc6..f9cbc7086e 100644 --- a/packages/server/src/tests/utilities/api/backup.ts +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -4,8 +4,6 @@ import { } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" -import tk from "timekeeper" -import { mocks } from "@budibase/backend-core/tests" export class BackupAPI extends TestAPI { constructor(config: TestConfiguration) { @@ -25,7 +23,6 @@ export class BackupAPI extends TestAPI { } createBackup = async (appId: string) => { - tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) const result = await this.request .post(`/api/apps/${appId}/backups`) .set(this.config.defaultHeaders()) @@ -38,7 +35,6 @@ export class BackupAPI extends TestAPI { appId: string, backupId: string ): Promise => { - tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) const result = await this.request .post(`/api/apps/${appId}/backups/${backupId}/import`) .set(this.config.defaultHeaders()) From 7e33aacbb1a77b0fa989d8647d9bf680c9b9ddfe Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 31 Oct 2023 14:48:23 +0000 Subject: [PATCH 5/7] Stop the sample data being identified as an external source. --- packages/server/src/integrations/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b4fff0737a..33895e4fe1 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -11,6 +11,7 @@ import { InvalidColumns, NoEmptyFilterStrings } from "../constants" import { helpers } from "@budibase/shared-core" import * as external from "../api/controllers/table/external" import * as internal from "../api/controllers/table/internal" +import { DEFAULT_BB_DATASOURCE_ID } from "../db/defaultData/datasource_bb_default" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g @@ -96,7 +97,8 @@ export function isInternalTableID(tableId: string) { export function isExternalTable(table: Table) { if ( table?.sourceId && - table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) + table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && + table?.sourceId !== DEFAULT_BB_DATASOURCE_ID ) { return true } else if (table?.sourceType === TableSourceType.EXTERNAL) { From 17319a6981a79ef41abe8fe4f7a26a6b02d4f9cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 17:52:39 +0000 Subject: [PATCH 6/7] Moving things around so that DEFAULT_BB_DATASOURCE_ID can be imported without cyclics occurring. --- .../server/src/api/controllers/application.ts | 7 ++----- packages/server/src/constants/index.ts | 5 +++++ .../src/db/defaultData/datasource_bb_default.ts | 16 +++++++++------- packages/server/src/integrations/utils.ts | 9 +++++---- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 4afd7b23f9..4e4c66858e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -32,11 +32,8 @@ import { tenancy, users, } from "@budibase/backend-core" -import { USERS_TABLE_SCHEMA } from "../../constants" -import { - buildDefaultDocs, - DEFAULT_BB_DATASOURCE_ID, -} from "../../db/defaultData/datasource_bb_default" +import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants" +import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream } from "../../utilities" import { doesUserHaveLock } from "../../utilities/redis" diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 1104fb9b29..fb5c42e7b8 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -172,3 +172,8 @@ export enum AutomationErrors { export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber" +export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" +export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" +export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" +export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index 584b76f879..b430f9ffb6 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -1,4 +1,12 @@ -import { AutoFieldSubTypes, FieldTypes } from "../../constants" +import { + AutoFieldSubTypes, + FieldTypes, + DEFAULT_BB_DATASOURCE_ID, + DEFAULT_INVENTORY_TABLE_ID, + DEFAULT_EMPLOYEE_TABLE_ID, + DEFAULT_EXPENSES_TABLE_ID, + DEFAULT_JOBS_TABLE_ID, +} from "../../constants" import { importToRows } from "../../api/controllers/table/utils" import { cloneDeep } from "lodash/fp" import LinkDocument from "../linkedRows/LinkDocument" @@ -16,12 +24,6 @@ import { TableSourceType, } from "@budibase/types" -export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" -export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" -export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" -export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" - const defaultDatasource = { _id: DEFAULT_BB_DATASOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 33895e4fe1..fe8a9055b0 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -7,11 +7,12 @@ import { TableSourceType, } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" -import { InvalidColumns, NoEmptyFilterStrings } from "../constants" +import { + InvalidColumns, + NoEmptyFilterStrings, + DEFAULT_BB_DATASOURCE_ID, +} from "../constants" import { helpers } from "@budibase/shared-core" -import * as external from "../api/controllers/table/external" -import * as internal from "../api/controllers/table/internal" -import { DEFAULT_BB_DATASOURCE_ID } from "../db/defaultData/datasource_bb_default" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g From 234cdbbf2441c4f3921a7226e14b920ba4506bab Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 1 Nov 2023 10:51:34 +0000 Subject: [PATCH 7/7] Bump version to 2.12.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 6df4a4c4cd..cb92b3ba0d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.1", + "version": "2.12.2", "npmClient": "yarn", "packages": [ "packages/*"