From 0ca72f99d1fad4f8c4892a44d24700a6bbf98a7d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Oct 2022 16:48:32 +0100 Subject: [PATCH 1/3] Adding statistics output for backups. --- .../src/events/publishers/backup.ts | 4 +- packages/server/src/sdk/app/backups/backup.ts | 22 ++++-- .../server/src/sdk/app/statistics/index.ts | 77 +++++++++++++++++++ packages/types/src/documents/app/backup.ts | 26 +++---- 4 files changed, 105 insertions(+), 24 deletions(-) create mode 100644 packages/server/src/sdk/app/statistics/index.ts diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts index 00b4f8db69..bd346cad64 100644 --- a/packages/backend-core/src/events/publishers/backup.ts +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -4,8 +4,8 @@ import { publishEvent } from "../events" export async function appBackupRestored(backup: AppBackup) { const properties: AppBackupRevertEvent = { appId: backup.appId, - backupName: backup.name, - backupCreatedAt: backup.createdAt, + backupName: backup.name!, + backupCreatedAt: backup.timestamp, } await publishEvent(Event.APP_BACKUP_RESTORED, properties) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 0f2a1aa760..6f7906d1cc 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -3,6 +3,7 @@ import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core" import { AppBackupQueueData } from "@budibase/types" import { exportApp } from "./exports" import { importApp } from "./imports" +import { calculateBackupStats } from "../statistics" import { Job } from "bull" import fs from "fs" import env from "../../../environment" @@ -52,26 +53,35 @@ async function exportProcessor(job: Job) { name = data.export!.name || `${trigger} - backup` const tenantId = tenancy.getTenantIDFromAppID(appId) await tenancy.doInTenant(tenantId, async () => { - const createdAt = new Date().toISOString() - const tarPath = await exportApp(appId, { tar: true }) - let filename = `${appId}/backup-${createdAt}.tar.gz` + const devAppId = dbCore.getDevAppID(appId), + prodAppId = dbCore.getProdAppID(appId) + const timestamp = new Date().toISOString() + const tarPath = await exportApp(devAppId, { tar: true }) + const contents = await calculateBackupStats(devAppId) + let filename = `${prodAppId}/backup-${timestamp}.tar.gz` // add the tenant to the bucket path if backing up within a multi-tenant environment if (env.MULTI_TENANCY) { filename = `${tenantId}/${filename}` } const bucket = objectStore.ObjectStoreBuckets.BACKUPS const metadata = { - appId, - createdAt, + appId: prodAppId, + timestamp, trigger, name, + contents, } await objectStore.upload({ path: tarPath, type: "application/gzip", bucket, filename, - metadata, + metadata: { + name, + trigger, + timestamp, + appId: prodAppId, + }, }) await backups.storeAppBackupMetadata(filename, metadata) // clear up the tarball after uploading it diff --git a/packages/server/src/sdk/app/statistics/index.ts b/packages/server/src/sdk/app/statistics/index.ts new file mode 100644 index 0000000000..3f03158264 --- /dev/null +++ b/packages/server/src/sdk/app/statistics/index.ts @@ -0,0 +1,77 @@ +import { context, db as dbCore } from "@budibase/backend-core" +import { + getDatasourceParams, + getTableParams, + getAutomationParams, + getScreenParams, +} from "../../../db/utils" + +async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { + if (db) { + return cb(db) + } else { + const devAppId = dbCore.getDevAppID(appId) + return context.doInAppContext(devAppId, () => { + const db = context.getAppDB() + return cb(db) + }) + } +} + +export async function calculateDatasourceCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const datasourceList = await db.allDocs(getDatasourceParams()) + const tableList = await db.allDocs(getTableParams()) + return datasourceList.rows.length + tableList.rows.length + }, + db + ) +} + +export async function calculateAutomationCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const automationList = await db.allDocs(getAutomationParams()) + return automationList.rows.length + }, + db + ) +} + +export async function calculateScreenCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const screenList = await db.allDocs(getScreenParams()) + return screenList.rows.length + }, + db + ) +} + +export async function calculateBackupStats(appId: string) { + return runInContext(appId, async (db: PouchDB.Database) => { + const promises = [] + promises.push(calculateDatasourceCount(appId, db)) + promises.push(calculateAutomationCount(appId, db)) + promises.push(calculateScreenCount(appId, db)) + const responses = await Promise.all(promises) + return { + datasources: responses[0], + automations: responses[1], + screens: responses[2], + } + }) +} diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index 28e927c772..a62b603a96 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -11,21 +11,23 @@ export enum AppBackupEventType { IMPORT = "import", } -export interface AppBackup extends Document { - trigger: AppBackupTrigger - name: string - createdAt: string - createdBy?: string - filename: string +export interface AppBackupMetadata { appId: string - userId?: string - contents?: { + trigger: AppBackupTrigger + name?: string + createdBy?: string + timestamp: string + contents: { datasources: string[] screens: string[] automations: string[] } } +export interface AppBackup extends Document, AppBackupMetadata { + filename: string +} + export type AppBackupFetchOpts = { trigger?: AppBackupTrigger limit?: number @@ -47,11 +49,3 @@ export interface AppBackupQueueData { backupId: string } } - -export interface AppBackupMetadata { - appId: string - trigger: AppBackupTrigger - name?: string - createdBy?: string - createdAt: string -} From e5ebd97ecc2cf8218c13721b8d44de26067875b4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Oct 2022 16:57:14 +0100 Subject: [PATCH 2/3] Fixing issue with using templates. --- packages/server/src/sdk/app/backups/imports.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index b29a9eede5..aaab85ec92 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -6,6 +6,7 @@ import { uploadDirectory, upload, } from "../../../utilities/fileSystem/utilities" +import { downloadTemplate } from "../../../utilities/fileSystem" import { ObjectStoreBuckets, FieldTypes } from "../../../constants" import { join } from "path" import fs from "fs" @@ -89,7 +90,7 @@ async function getTemplateStream(template: TemplateType) { return fs.createReadStream(template.file.path) } else if (template.key) { const [type, name] = template.key.split("/") - const tmpPath = await exports.downloadTemplate(type, name) + const tmpPath = await downloadTemplate(type, name) return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) } } From 6112d097af5197dfd1fdc115696b6627cc716ee4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Oct 2022 19:19:36 +0100 Subject: [PATCH 3/3] Some type updates and processor handling for backup document being created before backup/restore occurs. --- packages/server/src/sdk/app/backups/backup.ts | 24 +++++++------ packages/types/src/documents/app/backup.ts | 34 +++++++++++++------ 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 6f7906d1cc..97fe12e63d 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -1,6 +1,6 @@ import { backups } from "@budibase/pro" -import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core" -import { AppBackupQueueData } from "@budibase/types" +import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core" +import { AppBackupQueueData, AppBackupStatus } from "@budibase/types" import { exportApp } from "./exports" import { importApp } from "./imports" import { calculateBackupStats } from "../statistics" @@ -42,6 +42,11 @@ async function importProcessor(job: Job) { await removeExistingApp(devAppId) await performImport(backupTarPath) } + await backups.updateRestoreStatus( + data.docId, + data.docRev, + AppBackupStatus.COMPLETE + ) fs.rmSync(backupTarPath) }) } @@ -64,13 +69,6 @@ async function exportProcessor(job: Job) { filename = `${tenantId}/${filename}` } const bucket = objectStore.ObjectStoreBuckets.BACKUPS - const metadata = { - appId: prodAppId, - timestamp, - trigger, - name, - contents, - } await objectStore.upload({ path: tarPath, type: "application/gzip", @@ -83,7 +81,13 @@ async function exportProcessor(job: Job) { appId: prodAppId, }, }) - await backups.storeAppBackupMetadata(filename, metadata) + await backups.updateBackupStatus( + data.docId, + data.docRev, + AppBackupStatus.COMPLETE, + contents, + filename + ) // clear up the tarball after uploading it fs.rmSync(tarPath) }) diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index a62b603a96..a6d75efcff 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -1,31 +1,41 @@ import { Document } from "../document" +export enum AppBackupType { + BACKUP = "backup", + RESTORE = "restore", +} + +export enum AppBackupStatus { + STARTED = "started", + COMPLETE = "complete", + FAILED = "failed", +} + export enum AppBackupTrigger { PUBLISH = "publish", MANUAL = "manual", SCHEDULED = "scheduled", } -export enum AppBackupEventType { - EXPORT = "export", - IMPORT = "import", +export interface AppBackupContents { + datasources: string[] + screens: string[] + automations: string[] } export interface AppBackupMetadata { appId: string - trigger: AppBackupTrigger + trigger?: AppBackupTrigger + type: AppBackupType + status: AppBackupStatus name?: string createdBy?: string timestamp: string - contents: { - datasources: string[] - screens: string[] - automations: string[] - } + contents?: AppBackupContents } export interface AppBackup extends Document, AppBackupMetadata { - filename: string + filename?: string } export type AppBackupFetchOpts = { @@ -38,8 +48,9 @@ export type AppBackupFetchOpts = { } export interface AppBackupQueueData { - eventType: AppBackupEventType appId: string + docId: string + docRev: string export?: { trigger: AppBackupTrigger name?: string @@ -47,5 +58,6 @@ export interface AppBackupQueueData { } import?: { backupId: string + createdBy?: string } }