diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts index 00b4f8db69..bd346cad64 100644 --- a/packages/backend-core/src/events/publishers/backup.ts +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -4,8 +4,8 @@ import { publishEvent } from "../events" export async function appBackupRestored(backup: AppBackup) { const properties: AppBackupRevertEvent = { appId: backup.appId, - backupName: backup.name, - backupCreatedAt: backup.createdAt, + backupName: backup.name!, + backupCreatedAt: backup.timestamp, } await publishEvent(Event.APP_BACKUP_RESTORED, properties) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 0f2a1aa760..97fe12e63d 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -1,8 +1,9 @@ import { backups } from "@budibase/pro" -import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core" -import { AppBackupQueueData } from "@budibase/types" +import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core" +import { AppBackupQueueData, AppBackupStatus } from "@budibase/types" import { exportApp } from "./exports" import { importApp } from "./imports" +import { calculateBackupStats } from "../statistics" import { Job } from "bull" import fs from "fs" import env from "../../../environment" @@ -41,6 +42,11 @@ async function importProcessor(job: Job) { await removeExistingApp(devAppId) await performImport(backupTarPath) } + await backups.updateRestoreStatus( + data.docId, + data.docRev, + AppBackupStatus.COMPLETE + ) fs.rmSync(backupTarPath) }) } @@ -52,28 +58,36 @@ async function exportProcessor(job: Job) { name = data.export!.name || `${trigger} - backup` const tenantId = tenancy.getTenantIDFromAppID(appId) await tenancy.doInTenant(tenantId, async () => { - const createdAt = new Date().toISOString() - const tarPath = await exportApp(appId, { tar: true }) - let filename = `${appId}/backup-${createdAt}.tar.gz` + const devAppId = dbCore.getDevAppID(appId), + prodAppId = dbCore.getProdAppID(appId) + const timestamp = new Date().toISOString() + const tarPath = await exportApp(devAppId, { tar: true }) + const contents = await calculateBackupStats(devAppId) + let filename = `${prodAppId}/backup-${timestamp}.tar.gz` // add the tenant to the bucket path if backing up within a multi-tenant environment if (env.MULTI_TENANCY) { filename = `${tenantId}/${filename}` } const bucket = objectStore.ObjectStoreBuckets.BACKUPS - const metadata = { - appId, - createdAt, - trigger, - name, - } await objectStore.upload({ path: tarPath, type: "application/gzip", bucket, filename, - metadata, + metadata: { + name, + trigger, + timestamp, + appId: prodAppId, + }, }) - await backups.storeAppBackupMetadata(filename, metadata) + await backups.updateBackupStatus( + data.docId, + data.docRev, + AppBackupStatus.COMPLETE, + contents, + filename + ) // clear up the tarball after uploading it fs.rmSync(tarPath) }) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index b29a9eede5..aaab85ec92 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -6,6 +6,7 @@ import { uploadDirectory, upload, } from "../../../utilities/fileSystem/utilities" +import { downloadTemplate } from "../../../utilities/fileSystem" import { ObjectStoreBuckets, FieldTypes } from "../../../constants" import { join } from "path" import fs from "fs" @@ -89,7 +90,7 @@ async function getTemplateStream(template: TemplateType) { return fs.createReadStream(template.file.path) } else if (template.key) { const [type, name] = template.key.split("/") - const tmpPath = await exports.downloadTemplate(type, name) + const tmpPath = await downloadTemplate(type, name) return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) } } diff --git a/packages/server/src/sdk/app/statistics/index.ts b/packages/server/src/sdk/app/statistics/index.ts new file mode 100644 index 0000000000..3f03158264 --- /dev/null +++ b/packages/server/src/sdk/app/statistics/index.ts @@ -0,0 +1,77 @@ +import { context, db as dbCore } from "@budibase/backend-core" +import { + getDatasourceParams, + getTableParams, + getAutomationParams, + getScreenParams, +} from "../../../db/utils" + +async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { + if (db) { + return cb(db) + } else { + const devAppId = dbCore.getDevAppID(appId) + return context.doInAppContext(devAppId, () => { + const db = context.getAppDB() + return cb(db) + }) + } +} + +export async function calculateDatasourceCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const datasourceList = await db.allDocs(getDatasourceParams()) + const tableList = await db.allDocs(getTableParams()) + return datasourceList.rows.length + tableList.rows.length + }, + db + ) +} + +export async function calculateAutomationCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const automationList = await db.allDocs(getAutomationParams()) + return automationList.rows.length + }, + db + ) +} + +export async function calculateScreenCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const screenList = await db.allDocs(getScreenParams()) + return screenList.rows.length + }, + db + ) +} + +export async function calculateBackupStats(appId: string) { + return runInContext(appId, async (db: PouchDB.Database) => { + const promises = [] + promises.push(calculateDatasourceCount(appId, db)) + promises.push(calculateAutomationCount(appId, db)) + promises.push(calculateScreenCount(appId, db)) + const responses = await Promise.all(promises) + return { + datasources: responses[0], + automations: responses[1], + screens: responses[2], + } + }) +} diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index 28e927c772..a6d75efcff 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -1,29 +1,41 @@ import { Document } from "../document" +export enum AppBackupType { + BACKUP = "backup", + RESTORE = "restore", +} + +export enum AppBackupStatus { + STARTED = "started", + COMPLETE = "complete", + FAILED = "failed", +} + export enum AppBackupTrigger { PUBLISH = "publish", MANUAL = "manual", SCHEDULED = "scheduled", } -export enum AppBackupEventType { - EXPORT = "export", - IMPORT = "import", +export interface AppBackupContents { + datasources: string[] + screens: string[] + automations: string[] } -export interface AppBackup extends Document { - trigger: AppBackupTrigger - name: string - createdAt: string - createdBy?: string - filename: string +export interface AppBackupMetadata { appId: string - userId?: string - contents?: { - datasources: string[] - screens: string[] - automations: string[] - } + trigger?: AppBackupTrigger + type: AppBackupType + status: AppBackupStatus + name?: string + createdBy?: string + timestamp: string + contents?: AppBackupContents +} + +export interface AppBackup extends Document, AppBackupMetadata { + filename?: string } export type AppBackupFetchOpts = { @@ -36,8 +48,9 @@ export type AppBackupFetchOpts = { } export interface AppBackupQueueData { - eventType: AppBackupEventType appId: string + docId: string + docRev: string export?: { trigger: AppBackupTrigger name?: string @@ -45,13 +58,6 @@ export interface AppBackupQueueData { } import?: { backupId: string + createdBy?: string } } - -export interface AppBackupMetadata { - appId: string - trigger: AppBackupTrigger - name?: string - createdBy?: string - createdAt: string -}