Merge branch 'feature/app-backups' of github.com:Budibase/budibase into feature/backups-ui

packages/backend-core/src/events/publishers/backup.ts

@@ -4,8 +4,8 @@ import { publishEvent } from "../events"
 export async function appBackupRestored(backup: AppBackup) {
   const properties: AppBackupRevertEvent = {
     appId: backup.appId,
-    backupName: backup.name,
-    backupCreatedAt: backup.createdAt,
+    backupName: backup.name!,
+    backupCreatedAt: backup.timestamp,
   }
 
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)

packages/server/src/sdk/app/backups/backup.ts

@@ -1,8 +1,9 @@
 import { backups } from "@budibase/pro"
-import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core"
-import { AppBackupQueueData } from "@budibase/types"
+import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core"
+import { AppBackupQueueData, AppBackupStatus } from "@budibase/types"
 import { exportApp } from "./exports"
 import { importApp } from "./imports"
+import { calculateBackupStats } from "../statistics"
 import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
@@ -41,6 +42,11 @@ async function importProcessor(job: Job) {
       await removeExistingApp(devAppId)
       await performImport(backupTarPath)
     }
+    await backups.updateRestoreStatus(
+      data.docId,
+      data.docRev,
+      AppBackupStatus.COMPLETE
+    )
     fs.rmSync(backupTarPath)
   })
 }
@@ -52,28 +58,36 @@ async function exportProcessor(job: Job) {
     name = data.export!.name || `${trigger} - backup`
   const tenantId = tenancy.getTenantIDFromAppID(appId)
   await tenancy.doInTenant(tenantId, async () => {
-    const createdAt = new Date().toISOString()
-    const tarPath = await exportApp(appId, { tar: true })
-    let filename = `${appId}/backup-${createdAt}.tar.gz`
+    const devAppId = dbCore.getDevAppID(appId),
+      prodAppId = dbCore.getProdAppID(appId)
+    const timestamp = new Date().toISOString()
+    const tarPath = await exportApp(devAppId, { tar: true })
+    const contents = await calculateBackupStats(devAppId)
+    let filename = `${prodAppId}/backup-${timestamp}.tar.gz`
     // add the tenant to the bucket path if backing up within a multi-tenant environment
     if (env.MULTI_TENANCY) {
       filename = `${tenantId}/${filename}`
     }
     const bucket = objectStore.ObjectStoreBuckets.BACKUPS
-    const metadata = {
-      appId,
-      createdAt,
-      trigger,
-      name,
-    }
     await objectStore.upload({
       path: tarPath,
       type: "application/gzip",
       bucket,
       filename,
-      metadata,
+      metadata: {
+        name,
+        trigger,
+        timestamp,
+        appId: prodAppId,
+      },
     })
-    await backups.storeAppBackupMetadata(filename, metadata)
+    await backups.updateBackupStatus(
+      data.docId,
+      data.docRev,
+      AppBackupStatus.COMPLETE,
+      contents,
+      filename
+    )
     // clear up the tarball after uploading it
     fs.rmSync(tarPath)
   })

packages/server/src/sdk/app/backups/imports.ts

@@ -6,6 +6,7 @@ import {
   uploadDirectory,
   upload,
 } from "../../../utilities/fileSystem/utilities"
+import { downloadTemplate } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
@@ -89,7 +90,7 @@ async function getTemplateStream(template: TemplateType) {
     return fs.createReadStream(template.file.path)
   } else if (template.key) {
     const [type, name] = template.key.split("/")
-    const tmpPath = await exports.downloadTemplate(type, name)
+    const tmpPath = await downloadTemplate(type, name)
     return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
   }
 }

packages/server/src/sdk/app/statistics/index.ts

@@ -0,0 +1,77 @@
+import { context, db as dbCore } from "@budibase/backend-core"
+import {
+  getDatasourceParams,
+  getTableParams,
+  getAutomationParams,
+  getScreenParams,
+} from "../../../db/utils"
+
+async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
+  if (db) {
+    return cb(db)
+  } else {
+    const devAppId = dbCore.getDevAppID(appId)
+    return context.doInAppContext(devAppId, () => {
+      const db = context.getAppDB()
+      return cb(db)
+    })
+  }
+}
+
+export async function calculateDatasourceCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const datasourceList = await db.allDocs(getDatasourceParams())
+      const tableList = await db.allDocs(getTableParams())
+      return datasourceList.rows.length + tableList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateAutomationCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const automationList = await db.allDocs(getAutomationParams())
+      return automationList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateScreenCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const screenList = await db.allDocs(getScreenParams())
+      return screenList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateBackupStats(appId: string) {
+  return runInContext(appId, async (db: PouchDB.Database) => {
+    const promises = []
+    promises.push(calculateDatasourceCount(appId, db))
+    promises.push(calculateAutomationCount(appId, db))
+    promises.push(calculateScreenCount(appId, db))
+    const responses = await Promise.all(promises)
+    return {
+      datasources: responses[0],
+      automations: responses[1],
+      screens: responses[2],
+    }
+  })
+}

packages/types/src/documents/app/backup.ts

@@ -1,29 +1,41 @@
 import { Document } from "../document"
 
+export enum AppBackupType {
+  BACKUP = "backup",
+  RESTORE = "restore",
+}
+
+export enum AppBackupStatus {
+  STARTED = "started",
+  COMPLETE = "complete",
+  FAILED = "failed",
+}
+
 export enum AppBackupTrigger {
   PUBLISH = "publish",
   MANUAL = "manual",
   SCHEDULED = "scheduled",
 }
 
-export enum AppBackupEventType {
-  EXPORT = "export",
-  IMPORT = "import",
+export interface AppBackupContents {
+  datasources: string[]
+  screens: string[]
+  automations: string[]
 }
 
-export interface AppBackup extends Document {
-  trigger: AppBackupTrigger
-  name: string
-  createdAt: string
-  createdBy?: string
-  filename: string
+export interface AppBackupMetadata {
   appId: string
-  userId?: string
-  contents?: {
-    datasources: string[]
-    screens: string[]
-    automations: string[]
-  }
+  trigger?: AppBackupTrigger
+  type: AppBackupType
+  status: AppBackupStatus
+  name?: string
+  createdBy?: string
+  timestamp: string
+  contents?: AppBackupContents
+}
+
+export interface AppBackup extends Document, AppBackupMetadata {
+  filename?: string
 }
 
 export type AppBackupFetchOpts = {
@@ -36,8 +48,9 @@ export type AppBackupFetchOpts = {
 }
 
 export interface AppBackupQueueData {
-  eventType: AppBackupEventType
   appId: string
+  docId: string
+  docRev: string
   export?: {
     trigger: AppBackupTrigger
     name?: string
@@ -45,13 +58,6 @@ export interface AppBackupQueueData {
   }
   import?: {
     backupId: string
+    createdBy?: string
   }
 }
-
-export interface AppBackupMetadata {
-  appId: string
-  trigger: AppBackupTrigger
-  name?: string
-  createdBy?: string
-  createdAt: string
-}