From 9c9d2de4a555ce2087fb1bfe3338ff9ff3a5589d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 3 Oct 2022 11:56:31 +0100
Subject: [PATCH 01/46] Type updates for app backups work in pro + linting.

---
 packages/types/src/documents/global/index.ts     | 1 +
 packages/types/src/documents/global/schedules.ts | 9 +++++++++
 packages/types/src/sdk/licensing/feature.ts      | 1 +
 3 files changed, 11 insertions(+)
 create mode 100644 packages/types/src/documents/global/schedules.ts

diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts
index 84684df369..edce416a3f 100644
--- a/packages/types/src/documents/global/index.ts
+++ b/packages/types/src/documents/global/index.ts
@@ -3,3 +3,4 @@ export * from "./user"
 export * from "./userGroup"
 export * from "./plugin"
 export * from "./quotas"
+export * from "./schedules"
diff --git a/packages/types/src/documents/global/schedules.ts b/packages/types/src/documents/global/schedules.ts
new file mode 100644
index 0000000000..357a841a9a
--- /dev/null
+++ b/packages/types/src/documents/global/schedules.ts
@@ -0,0 +1,9 @@
+export enum ScheduleType {
+  APP_BACKUP = "app_backup",
+}
+
+export enum ScheduleRepeatPeriod {
+  DAILY = "daily",
+  WEEKLY = "weekly",
+  MONTHLY = "monthly",
+}
diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts
index cbd1f4a50c..f06a8d1382 100644
--- a/packages/types/src/sdk/licensing/feature.ts
+++ b/packages/types/src/sdk/licensing/feature.ts
@@ -1,3 +1,4 @@
 export enum Feature {
   USER_GROUPS = "userGroups",
+  APP_BACKUPS = "appBackups",
 }

From 29659813efa90cb5d78b03249a69a5ae08b1a99a Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 3 Oct 2022 14:02:58 +0100
Subject: [PATCH 02/46] Add document and api types

---
 packages/types/src/api/web/app/backup.ts      | 15 +++++++++
 packages/types/src/api/web/app/index.ts       |  1 +
 packages/types/src/api/web/index.ts           |  2 ++
 packages/types/src/api/web/schedule.ts        | 15 +++++++++
 packages/types/src/api/web/user.ts            |  7 ++++
 packages/types/src/documents/app/backup.ts    | 21 ++++++++++++
 packages/types/src/documents/app/index.ts     |  1 +
 packages/types/src/documents/global/index.ts  |  2 +-
 .../types/src/documents/global/schedule.ts    | 32 +++++++++++++++++++
 .../types/src/documents/global/schedules.ts   |  9 ------
 .../src/api/controllers/global/users.ts       |  4 ++-
 packages/worker/src/sdk/users/users.ts        |  7 +++-
 12 files changed, 104 insertions(+), 12 deletions(-)
 create mode 100644 packages/types/src/api/web/app/backup.ts
 create mode 100644 packages/types/src/api/web/app/index.ts
 create mode 100644 packages/types/src/api/web/schedule.ts
 create mode 100644 packages/types/src/documents/app/backup.ts
 create mode 100644 packages/types/src/documents/global/schedule.ts
 delete mode 100644 packages/types/src/documents/global/schedules.ts

diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts
new file mode 100644
index 0000000000..57ffba0d70
--- /dev/null
+++ b/packages/types/src/api/web/app/backup.ts
@@ -0,0 +1,15 @@
+import { AppBackupTrigger } from "../../../documents"
+
+export interface SearchAppBackupsRequest {
+  trigger: AppBackupTrigger
+  startDate: string
+  endDate: string
+}
+
+export interface CreateAppBackupRequest {
+  name: string
+}
+
+export interface UpdateAppBackupRequest {
+  name: string
+}
diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts
new file mode 100644
index 0000000000..1d73755cb6
--- /dev/null
+++ b/packages/types/src/api/web/app/index.ts
@@ -0,0 +1 @@
+export * from "./backup"
diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts
index 0129fb38d9..1dbe22aa46 100644
--- a/packages/types/src/api/web/index.ts
+++ b/packages/types/src/api/web/index.ts
@@ -1,3 +1,5 @@
 export * from "./analytics"
 export * from "./user"
 export * from "./errors"
+export * from "./schedule"
+export * from "./app"
diff --git a/packages/types/src/api/web/schedule.ts b/packages/types/src/api/web/schedule.ts
new file mode 100644
index 0000000000..bf762b1603
--- /dev/null
+++ b/packages/types/src/api/web/schedule.ts
@@ -0,0 +1,15 @@
+import {
+  ScheduleMetadata,
+  ScheduleRepeatPeriod,
+  ScheduleType,
+} from "../../documents"
+
+export interface CreateScheduleRequest {
+  type: ScheduleType
+  name: string
+  startDate: string
+  repeat: ScheduleRepeatPeriod
+  metadata: ScheduleMetadata
+}
+
+export interface UpdateScheduleRequest extends CreateScheduleRequest {}
diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index c66d3203e8..98ffcdf360 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -44,3 +44,10 @@ export interface InviteUsersResponse {
   successful: { email: string }[]
   unsuccessful: { email: string; reason: string }[]
 }
+
+export interface SearchUsersRequest {
+  page?: string
+  email?: string
+  appId?: string
+  userIds?: string[]
+}
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
new file mode 100644
index 0000000000..a935ed5ba1
--- /dev/null
+++ b/packages/types/src/documents/app/backup.ts
@@ -0,0 +1,21 @@
+import { Document } from "../document"
+
+export enum AppBackupTrigger {
+  PUBLISH = "publish",
+  MANUAL = "manual",
+  SCHEDULED = "scheduled",
+}
+
+export interface AppBackupContents {
+  datasources: string[]
+  screens: string[]
+  automations: string[]
+}
+
+export interface AppBackup extends Document {
+  trigger: AppBackupTrigger
+  name: string
+  date: string
+  userId: string
+  contents: AppBackupContents
+}
diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts
index e8b29257fc..dad594b804 100644
--- a/packages/types/src/documents/app/index.ts
+++ b/packages/types/src/documents/app/index.ts
@@ -10,3 +10,4 @@ export * from "./view"
 export * from "../document"
 export * from "./row"
 export * from "./user"
+export * from "./backup"
diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts
index edce416a3f..9f779d4937 100644
--- a/packages/types/src/documents/global/index.ts
+++ b/packages/types/src/documents/global/index.ts
@@ -3,4 +3,4 @@ export * from "./user"
 export * from "./userGroup"
 export * from "./plugin"
 export * from "./quotas"
-export * from "./schedules"
+export * from "./schedule"
diff --git a/packages/types/src/documents/global/schedule.ts b/packages/types/src/documents/global/schedule.ts
new file mode 100644
index 0000000000..63bb9284fe
--- /dev/null
+++ b/packages/types/src/documents/global/schedule.ts
@@ -0,0 +1,32 @@
+import { Document } from "../document"
+
+export enum ScheduleType {
+  APP_BACKUP = "app_backup",
+}
+
+export enum ScheduleRepeatPeriod {
+  DAILY = "daily",
+  WEEKLY = "weekly",
+  MONTHLY = "monthly",
+}
+
+export interface Schedule extends Document {
+  type: ScheduleType
+  name: string
+  startDate: string
+  repeat: ScheduleRepeatPeriod
+  metadata: ScheduleMetadata
+}
+
+export type ScheduleMetadata = AppBackupScheduleMetadata
+
+export const isAppBackupMetadata = (
+  type: ScheduleType,
+  metadata: ScheduleMetadata
+): metadata is AppBackupScheduleMetadata => {
+  return type === ScheduleType.APP_BACKUP
+}
+
+export interface AppBackupScheduleMetadata {
+  apps: string[]
+}
diff --git a/packages/types/src/documents/global/schedules.ts b/packages/types/src/documents/global/schedules.ts
deleted file mode 100644
index 357a841a9a..0000000000
--- a/packages/types/src/documents/global/schedules.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-export enum ScheduleType {
-  APP_BACKUP = "app_backup",
-}
-
-export enum ScheduleRepeatPeriod {
-  DAILY = "daily",
-  WEEKLY = "weekly",
-  MONTHLY = "monthly",
-}
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 8894330f67..ea1df5b45a 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -7,6 +7,7 @@ import {
   CloudAccount,
   InviteUserRequest,
   InviteUsersRequest,
+  SearchUsersRequest,
   User,
 } from "@budibase/types"
 import {
@@ -144,7 +145,8 @@ export const destroy = async (ctx: any) => {
 }
 
 export const search = async (ctx: any) => {
-  const paginated = await sdk.users.paginatedUsers(ctx.request.body)
+  const body = ctx.request.body as SearchUsersRequest
+  const paginated = await sdk.users.paginatedUsers(body)
   // user hashed password shouldn't ever be returned
   for (let user of paginated.data) {
     if (user) {
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index 775514ea5e..3b98c8ef52 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -27,6 +27,7 @@ import {
   MigrationType,
   PlatformUserByEmail,
   RowResponse,
+  SearchUsersRequest,
   User,
 } from "@budibase/types"
 import { sendEmail } from "../../utilities/email"
@@ -56,7 +57,8 @@ export const paginatedUsers = async ({
   page,
   email,
   appId,
-}: { page?: string; email?: string; appId?: string } = {}) => {
+  userIds,
+}: SearchUsersRequest = {}) => {
   const db = tenancy.getGlobalDB()
   // get one extra document, to have the next page
   const opts: any = {
@@ -77,6 +79,9 @@ export const paginatedUsers = async ({
   } else if (email) {
     userList = await usersCore.searchGlobalUsersByEmail(email, opts)
     property = "email"
+  }
+  if (userIds) {
+    // TODO: search users by userIds
   } else {
     // no search, query allDocs
     const response = await db.allDocs(dbUtils.getGlobalUserParams(null, opts))

From e78ed801b249fbf85ca7b1e886082c9a6e7640f5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 3 Oct 2022 19:36:33 +0100
Subject: [PATCH 03/46] Linting.

---
 .../TestConfiguration/InternalAPIClient.ts          |  6 ++----
 .../internal-api/TestConfiguration/applications.ts  |  8 ++------
 .../config/internal-api/TestConfiguration/auth.ts   |  6 +++---
 .../config/internal-api/fixtures/applications.ts    |  9 ++++-----
 .../tests/internal-api/applications/create.spec.ts  | 13 +++++++------
 5 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts
index bfcbb9f4e2..dafc2b1ff2 100644
--- a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts
+++ b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts
@@ -16,9 +16,7 @@ class InternalAPIClient {
 
   constructor(appId?: string) {
     if (!env.BUDIBASE_SERVER_URL) {
-      throw new Error(
-        "Must set BUDIBASE_SERVER_URL env var"
-      )
+      throw new Error("Must set BUDIBASE_SERVER_URL env var")
     }
     this.host = `${env.BUDIBASE_SERVER_URL}/api`
     this.appId = appId
@@ -55,4 +53,4 @@ class InternalAPIClient {
   put = this.apiCall("PUT")
 }
 
-export default InternalAPIClient
\ No newline at end of file
+export default InternalAPIClient
diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts
index 10e4a6657b..0c51487122 100644
--- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts
+++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts
@@ -1,6 +1,4 @@
-import {
-  Application,
-} from "@budibase/server/api/controllers/public/mapping/types"
+import { Application } from "@budibase/server/api/controllers/public/mapping/types"
 import { App } from "@budibase/types"
 import { Response } from "node-fetch"
 import InternalAPIClient from "./InternalAPIClient"
@@ -37,9 +35,7 @@ export default class AppApi {
     return [response, json]
   }
 
-  async create(
-    body: any
-  ): Promise<[Response, Partial<App>]> {
+  async create(body: any): Promise<[Response, Partial<App>]> {
     const response = await this.api.post(`/applications`, { body })
     const json = await response.json()
     return [response, json]
diff --git a/qa-core/src/config/internal-api/TestConfiguration/auth.ts b/qa-core/src/config/internal-api/TestConfiguration/auth.ts
index 6ac53f24b6..d83c859ab3 100644
--- a/qa-core/src/config/internal-api/TestConfiguration/auth.ts
+++ b/qa-core/src/config/internal-api/TestConfiguration/auth.ts
@@ -9,11 +9,11 @@ export default class AuthApi {
   }
 
   async login(): Promise<[Response, any]> {
-    const response = await this.api.post(`/global/auth/default/login`, { 
+    const response = await this.api.post(`/global/auth/default/login`, {
       body: {
         username: process.env.BB_ADMIN_USER_EMAIL,
-        password: process.env.BB_ADMIN_USER_PASSWORD
-      }
+        password: process.env.BB_ADMIN_USER_PASSWORD,
+      },
     })
     const cookie = response.headers.get("set-cookie")
     this.api.cookie = cookie as any
diff --git a/qa-core/src/config/internal-api/fixtures/applications.ts b/qa-core/src/config/internal-api/fixtures/applications.ts
index dfad7e0b46..9076a05e1b 100644
--- a/qa-core/src/config/internal-api/fixtures/applications.ts
+++ b/qa-core/src/config/internal-api/fixtures/applications.ts
@@ -1,10 +1,9 @@
 import generator from "../../generator"
-import {
-  Application,
-} from "@budibase/server/api/controllers/public/mapping/types"
+import { Application } from "@budibase/server/api/controllers/public/mapping/types"
 
-
-const generate = (overrides: Partial<Application> = {}): Partial<Application> => ({
+const generate = (
+  overrides: Partial<Application> = {}
+): Partial<Application> => ({
   name: generator.word(),
   url: `/${generator.word()}`,
   ...overrides,
diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts
index 81d43d9c91..2c934e0bd7 100644
--- a/qa-core/src/tests/internal-api/applications/create.spec.ts
+++ b/qa-core/src/tests/internal-api/applications/create.spec.ts
@@ -24,14 +24,14 @@ describe("Internal API - /applications endpoints", () => {
       useTemplate: "true",
       templateName: "Near Miss Register",
       templateKey: "app/near-miss-register",
-      templateFile: undefined
+      templateFile: undefined,
     })
   }
 
   it("GET - fetch applications", async () => {
     await config.applications.create({
       ...generateApp(),
-      useTemplate: false
+      useTemplate: false,
     })
     const [response, apps] = await config.applications.fetch()
     expect(response).toHaveStatusCode(200)
@@ -57,7 +57,7 @@ describe("Internal API - /applications endpoints", () => {
     expect(publish).toEqual({
       _id: expect.any(String),
       appUrl: app.url,
-      status: "SUCCESS"
+      status: "SUCCESS",
     })
   })
 
@@ -70,7 +70,8 @@ describe("Internal API - /applications endpoints", () => {
     config.applications.api.appId = app.appId
 
     // check preview renders
-    const [previewResponse, previewRenders] = await config.applications.canRender()
+    const [previewResponse, previewRenders] =
+      await config.applications.canRender()
     expect(previewResponse).toHaveStatusCode(200)
     expect(previewRenders).toBe(true)
 
@@ -79,8 +80,8 @@ describe("Internal API - /applications endpoints", () => {
 
     // check published app renders
     config.applications.api.appId = db.getProdAppID(app.appId)
-    const [publishedAppResponse, publishedAppRenders] = await config.applications.canRender()
+    const [publishedAppResponse, publishedAppRenders] =
+      await config.applications.canRender()
     expect(publishedAppRenders).toBe(true)
   })
-
 })

From ef52bde670d5b73e5a742216de6573014d9d0af5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Oct 2022 19:10:45 +0100
Subject: [PATCH 04/46] Building out initial SDK work - converting some
 existing exporting work to typescript.

---
 .../api/controllers/{backup.js => backup.ts}  |  12 +-
 packages/server/src/api/controllers/cloud.js  |  11 +-
 packages/server/src/api/routes/backup.js      |  10 --
 packages/server/src/api/routes/backup.ts      |  10 ++
 packages/server/src/sdk/app/export.ts         | 113 ++++++++++++++++++
 packages/server/src/sdk/app/index.ts          |   1 +
 packages/server/src/sdk/index.ts              |   5 +
 .../server/src/utilities/fileSystem/index.js  | 106 ----------------
 8 files changed, 139 insertions(+), 129 deletions(-)
 rename packages/server/src/api/controllers/{backup.js => backup.ts} (56%)
 delete mode 100644 packages/server/src/api/routes/backup.js
 create mode 100644 packages/server/src/api/routes/backup.ts
 create mode 100644 packages/server/src/sdk/app/export.ts
 create mode 100644 packages/server/src/sdk/app/index.ts
 create mode 100644 packages/server/src/sdk/index.ts

diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.ts
similarity index 56%
rename from packages/server/src/api/controllers/backup.js
rename to packages/server/src/api/controllers/backup.ts
index 5e08c823ed..cd2ea415a4 100644
--- a/packages/server/src/api/controllers/backup.js
+++ b/packages/server/src/api/controllers/backup.ts
@@ -1,15 +1,15 @@
-const { streamBackup } = require("../../utilities/fileSystem")
-const { events, context } = require("@budibase/backend-core")
-const { DocumentType } = require("../../db/utils")
-const { isQsTrue } = require("../../utilities")
+import sdk from "../../sdk"
+import { events, context } from "@budibase/backend-core"
+import { DocumentType } from "../../db/utils"
+import { isQsTrue } from "../../utilities"
 
-exports.exportAppDump = async function (ctx) {
+export async function exportAppDump(ctx: any) {
   let { appId, excludeRows } = ctx.query
   const appName = decodeURI(ctx.query.appname)
   excludeRows = isQsTrue(excludeRows)
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
-  ctx.body = await streamBackup(appId, excludeRows)
+  ctx.body = await sdk.apps.exports.streamBackup(appId, excludeRows)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()
diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index 1e6abb1d3b..55aa6bb548 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -1,14 +1,11 @@
 const env = require("../../environment")
 const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db")
-const {
-  exportDB,
-  sendTempFile,
-  readFileSync,
-} = require("../../utilities/fileSystem")
+const { sendTempFile, readFileSync } = require("../../utilities/fileSystem")
 const { stringToReadStream } = require("../../utilities")
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { create } = require("./application")
 const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils")
+const sdk = require("../../sdk")
 
 async function createApp(appName, appImport) {
   const ctx = {
@@ -27,7 +24,7 @@ exports.exportApps = async ctx => {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
   const apps = await getAllApps({ all: true })
-  const globalDBString = await exportDB(getGlobalDBName(), {
+  const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentType.USER),
   })
   let allDBs = {
@@ -38,7 +35,7 @@ exports.exportApps = async ctx => {
     // only export the dev apps as they will be the latest, the user can republish the apps
     // in their self hosted environment
     if (isDevAppID(appId)) {
-      allDBs[app.name] = await exportDB(appId)
+      allDBs[app.name] = await sdk.apps.exports.exportDB(appId)
     }
   }
   const filename = `cloud-export-${new Date().getTime()}.txt`
diff --git a/packages/server/src/api/routes/backup.js b/packages/server/src/api/routes/backup.js
deleted file mode 100644
index 9f3b27e95a..0000000000
--- a/packages/server/src/api/routes/backup.js
+++ /dev/null
@@ -1,10 +0,0 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/backup")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/backend-core/permissions")
-
-const router = new Router()
-
-router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump)
-
-module.exports = router
diff --git a/packages/server/src/api/routes/backup.ts b/packages/server/src/api/routes/backup.ts
new file mode 100644
index 0000000000..2473fa9f67
--- /dev/null
+++ b/packages/server/src/api/routes/backup.ts
@@ -0,0 +1,10 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/backup"
+import authorized from "../../middleware/authorized"
+import { BUILDER } from "@budibase/backend-core/permissions"
+
+const router = new Router()
+
+router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump)
+
+export default router
diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts
new file mode 100644
index 0000000000..70c4446273
--- /dev/null
+++ b/packages/server/src/sdk/app/export.ts
@@ -0,0 +1,113 @@
+import { closeDB, dangerousGetDB, doWithDB } from "@budibase/backend-core/db"
+import { budibaseTempDir } from "../../utilities/budibaseDir"
+import { streamUpload } from "../../utilities/fileSystem/utilities"
+import { ObjectStoreBuckets } from "../../constants"
+import {
+  LINK_USER_METADATA_PREFIX,
+  TABLE_ROW_PREFIX,
+  USER_METDATA_PREFIX,
+} from "../../db/utils"
+import fs from "fs"
+import env from "../../environment"
+import { join } from "path"
+const MemoryStream = require("memorystream")
+
+/**
+ * Exports a DB to either file or a variable (memory).
+ * @param {string} dbName the DB which is to be exported.
+ * @param {object} opts various options for the export, e.g. whether to stream,
+ * a filter function or the name of the export.
+ * @return {*} either a readable stream or a string
+ */
+export async function exportDB(
+  dbName: string,
+  opts: { stream?: boolean; filter?: any; exportName?: string } = {}
+) {
+  // streaming a DB dump is a bit more complicated, can't close DB
+  if (opts?.stream) {
+    const db = dangerousGetDB(dbName)
+    const memStream = new MemoryStream()
+    memStream.on("end", async () => {
+      await closeDB(db)
+    })
+    db.dump(memStream, { filter: opts?.filter })
+    return memStream
+  }
+
+  return doWithDB(dbName, async (db: any) => {
+    // Write the dump to file if required
+    if (opts?.exportName) {
+      const path = join(budibaseTempDir(), opts?.exportName)
+      const writeStream = fs.createWriteStream(path)
+      await db.dump(writeStream, { filter: opts?.filter })
+
+      // Upload the dump to the object store if self-hosted
+      if (env.SELF_HOSTED) {
+        await streamUpload(
+          ObjectStoreBuckets.BACKUPS,
+          join(dbName, opts?.exportName),
+          fs.createReadStream(path)
+        )
+      }
+
+      return fs.createReadStream(path)
+    }
+
+    // Stringify the dump in memory if required
+    const memStream = new MemoryStream()
+    let appString = ""
+    memStream.on("data", (chunk: any) => {
+      appString += chunk.toString()
+    })
+    await db.dump(memStream, { filter: opts?.filter })
+    return appString
+  })
+}
+
+function defineFilter(excludeRows?: boolean) {
+  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
+  if (excludeRows) {
+    ids.push(TABLE_ROW_PREFIX)
+  }
+  return (doc: any) =>
+    !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
+}
+
+/**
+ * Local utility to back up the database state for an app, excluding global user
+ * data or user relationships.
+ * @param {string} appId The app to back up
+ * @param {object} config Config to send to export DB
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
+ * @returns {*} either a string or a stream of the backup
+ */
+async function backupAppData(
+  appId: string,
+  config: any,
+  excludeRows?: boolean
+) {
+  return await exportDB(appId, {
+    ...config,
+    filter: defineFilter(excludeRows),
+  })
+}
+
+/**
+ * Streams a backup of the database state for an app
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
+ * @returns {*} a readable stream of the backup which is written in real time
+ */
+export async function streamBackup(appId: string, excludeRows: boolean) {
+  return await backupAppData(appId, { stream: true }, excludeRows)
+}
+
+/**
+ * Takes a copy of the database state for an app to the object store.
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {string} backupName The name of the backup located in the object store.
+ * @return {*} a readable stream to the completed backup file
+ */
+export async function performBackup(appId: string, backupName: string) {
+  return await backupAppData(appId, { exportName: backupName })
+}
diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts
new file mode 100644
index 0000000000..3927539bc8
--- /dev/null
+++ b/packages/server/src/sdk/app/index.ts
@@ -0,0 +1 @@
+export * as exports from "./export"
diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts
new file mode 100644
index 0000000000..9199c1cc72
--- /dev/null
+++ b/packages/server/src/sdk/index.ts
@@ -0,0 +1,5 @@
+import * as apps from "./app"
+
+export default {
+  apps,
+}
diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js
index 4e9b13cca0..656ba4816c 100644
--- a/packages/server/src/utilities/fileSystem/index.js
+++ b/packages/server/src/utilities/fileSystem/index.js
@@ -2,17 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const {
-  doWithDB,
-  dangerousGetDB,
-  closeDB,
-} = require("@budibase/backend-core/db")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   upload,
   retrieve,
   retrieveToTmp,
-  streamUpload,
   deleteFolder,
   downloadTarball,
   downloadTarballDirect,
@@ -21,12 +15,6 @@ const {
 const { updateClientLibrary } = require("./clientLibrary")
 const { checkSlashesInUrl } = require("../")
 const env = require("../../environment")
-const {
-  USER_METDATA_PREFIX,
-  LINK_USER_METADATA_PREFIX,
-  TABLE_ROW_PREFIX,
-} = require("../../db/utils")
-const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
 const fetch = require("node-fetch")
@@ -124,100 +112,6 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
-exports.defineFilter = excludeRows => {
-  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
-  if (excludeRows) {
-    ids.push(TABLE_ROW_PREFIX)
-  }
-  return doc =>
-    !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
-}
-
-/**
- * Local utility to back up the database state for an app, excluding global user
- * data or user relationships.
- * @param {string} appId The app to backup
- * @param {object} config Config to send to export DB
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} either a string or a stream of the backup
- */
-const backupAppData = async (appId, config, excludeRows) => {
-  return await exports.exportDB(appId, {
-    ...config,
-    filter: exports.defineFilter(excludeRows),
-  })
-}
-
-/**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
- */
-exports.performBackup = async (appId, backupName) => {
-  return await backupAppData(appId, { exportName: backupName })
-}
-
-/**
- * Streams a backup of the database state for an app
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} a readable stream of the backup which is written in real time
- */
-exports.streamBackup = async (appId, excludeRows) => {
-  return await backupAppData(appId, { stream: true }, excludeRows)
-}
-
-/**
- * Exports a DB to either file or a variable (memory).
- * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - provide a filename to write the backup to a file
- * @param {boolean} stream optional - whether to perform a full backup
- * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return {*} either a readable stream or a string
- */
-exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
-  // streaming a DB dump is a bit more complicated, can't close DB
-  if (stream) {
-    const db = dangerousGetDB(dbName)
-    const memStream = new MemoryStream()
-    memStream.on("end", async () => {
-      await closeDB(db)
-    })
-    db.dump(memStream, { filter })
-    return memStream
-  }
-
-  return doWithDB(dbName, async db => {
-    // Write the dump to file if required
-    if (exportName) {
-      const path = join(budibaseTempDir(), exportName)
-      const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter })
-
-      // Upload the dump to the object store if self hosted
-      if (env.SELF_HOSTED) {
-        await streamUpload(
-          ObjectStoreBuckets.BACKUPS,
-          join(dbName, exportName),
-          fs.createReadStream(path)
-        )
-      }
-
-      return fs.createReadStream(path)
-    }
-
-    // Stringify the dump in memory if required
-    const memStream = new MemoryStream()
-    let appString = ""
-    memStream.on("data", chunk => {
-      appString += chunk.toString()
-    })
-    await db.dump(memStream, { filter })
-    return appString
-  })
-}
-
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.
  * @param {string} fileContents contents which will be written to a temp file.

From 9efb8f98bc4a105e00680eae97ed6d0916ada783 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 7 Oct 2022 21:08:20 +0100
Subject: [PATCH 05/46] Updating koa versions to align with pro - types were
 inaccurate and couldn't be imported correctly.

---
 packages/server/package.json            |   8 +-
 packages/server/src/api/routes/index.ts |   8 +-
 packages/server/yarn.lock               | 118 +++++-------------------
 packages/worker/package.json            |   1 -
 4 files changed, 36 insertions(+), 99 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 2f2409850e..dba6bfbe79 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -86,7 +86,7 @@
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "5.0.2",
-    "@koa/router": "8.0.0",
+    "@koa/router": "8.0.8",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
     "airtable": "0.10.1",
@@ -112,7 +112,7 @@
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
     "knex": "0.95.15",
-    "koa": "2.7.0",
+    "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-connect": "2.1.0",
@@ -163,8 +163,8 @@
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/jest": "27.5.1",
-    "@types/koa": "2.13.4",
-    "@types/koa__router": "8.0.0",
+    "@types/koa": "2.13.5",
+    "@types/koa__router": "8.0.11",
     "@types/lodash": "4.14.180",
     "@types/mongodb": "3.6.3",
     "@types/node": "14.18.20",
diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts
index 64f5f9cb89..1cf34d4a68 100644
--- a/packages/server/src/api/routes/index.ts
+++ b/packages/server/src/api/routes/index.ts
@@ -25,11 +25,15 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
+import Router from "@koa/router"
+import { api } from "@budibase/pro"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
 
-export const mainRoutes = [
+const appBackupRoutes = api.appBackups
+const scheduleRoutes = api.schedules
+export const mainRoutes: Router[] = [
   authRoutes,
   deployRoutes,
   layoutRoutes,
@@ -59,4 +63,6 @@ export const mainRoutes = [
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
+  appBackupRoutes,
+  scheduleRoutes,
 ]
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index 18ab07c17f..bf95495815 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -2003,18 +2003,6 @@
   resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796"
   integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==
 
-"@koa/router@8.0.0":
-  version "8.0.0"
-  resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.0.tgz#fd4ffa6f03d8293a04c023cb4a22b612401fbe70"
-  integrity sha512-P70CGOGs6JPu/mnrd9lt6ESzlBXLHT/uTK8+5U4M7Oapt8la/tiZv2c7X9jq0ksFsM59RH3AwJYzKOuavDcjIw==
-  dependencies:
-    debug "^3.1.0"
-    http-errors "^1.3.1"
-    koa-compose "^3.0.0"
-    methods "^1.0.1"
-    path-to-regexp "^1.1.1"
-    urijs "^1.19.0"
-
 "@koa/router@8.0.8":
   version "8.0.8"
   resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.8.tgz#95f32d11373d03d89dcb63fabe9ac6f471095236"
@@ -2851,7 +2839,7 @@
   dependencies:
     "@types/koa" "*"
 
-"@types/koa@*", "@types/koa@2.13.4":
+"@types/koa@*":
   version "2.13.4"
   resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b"
   integrity sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw==
@@ -2865,10 +2853,24 @@
     "@types/koa-compose" "*"
     "@types/node" "*"
 
-"@types/koa__router@8.0.0":
-  version "8.0.0"
-  resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.0.tgz#057a7254a25df5bc93b42a1acacb2d99cd02d297"
-  integrity sha512-XaGqudqJyFOmByN+f9BrEIZEgLfBnvVtZlm/beuTxWpbWpMHiA+ZmA+mB5dsrbGemko61wUA+WG0jhUzMSq+JA==
+"@types/koa@2.13.5":
+  version "2.13.5"
+  resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.5.tgz#64b3ca4d54e08c0062e89ec666c9f45443b21a61"
+  integrity sha512-HSUOdzKz3by4fnqagwthW/1w/yJspTgppyyalPVbgZf8jQWvdIXcVW5h2DGtw4zYntOaeRGx49r1hxoPWrD4aA==
+  dependencies:
+    "@types/accepts" "*"
+    "@types/content-disposition" "*"
+    "@types/cookies" "*"
+    "@types/http-assert" "*"
+    "@types/http-errors" "*"
+    "@types/keygrip" "*"
+    "@types/koa-compose" "*"
+    "@types/node" "*"
+
+"@types/koa__router@8.0.11":
+  version "8.0.11"
+  resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.11.tgz#d7b37e6db934fc072ea1baa2ab92bc8ac4564f3e"
+  integrity sha512-WXgKWpBsbS14kzmzD9LeFapOIa678h7zvUHxDwXwSx4ETKXhXLVUAToX6jZ/U7EihM7qwyD9W/BZvB0MRu7MTQ==
   dependencies:
     "@types/koa" "*"
 
@@ -3523,7 +3525,7 @@ any-base@^1.1.0:
   resolved "https://registry.yarnpkg.com/any-base/-/any-base-1.1.0.tgz#ae101a62bc08a597b4c9ab5b7089d456630549fe"
   integrity sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==
 
-any-promise@^1.0.0, any-promise@^1.1.0:
+any-promise@^1.0.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
   integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==
@@ -4907,14 +4909,6 @@ cookiejar@^2.1.0:
   resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.3.tgz#fc7a6216e408e74414b90230050842dacda75acc"
   integrity sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ==
 
-cookies@~0.7.1:
-  version "0.7.3"
-  resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.7.3.tgz#7912ce21fbf2e8c2da70cf1c3f351aecf59dadfa"
-  integrity sha512-+gixgxYSgQLTaTIilDHAdlNPZDENDQernEMiIcZpYYP14zgHsCt4Ce1FEjFtcp6GefhozebB6orvhAAWx/IS0A==
-  dependencies:
-    depd "~1.1.2"
-    keygrip "~1.0.3"
-
 cookies@~0.8.0:
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.8.0.tgz#1293ce4b391740a8406e3c9870e828c4b54f3f90"
@@ -5138,13 +5132,6 @@ debug@^3.1.0, debug@^3.2.6, debug@^3.2.7:
   dependencies:
     ms "^2.1.1"
 
-debug@~3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
-  integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
-  dependencies:
-    ms "2.0.0"
-
 debuglog@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
@@ -5710,11 +5697,6 @@ error-ex@^1.3.1:
   dependencies:
     is-arrayish "^0.2.1"
 
-error-inject@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37"
-  integrity sha512-JM8N6PytDbmIYm1IhPWlo8vr3NtfjhDY/1MhD/a5b/aad/USE8a0+NsqE9d5n+GVGmuNkPQWm4bFQWv18d8tMg==
-
 error-stack-parser@^2.0.6:
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/error-stack-parser/-/error-stack-parser-2.1.4.tgz#229cb01cdbfa84440bfa91876285b94680188286"
@@ -7396,7 +7378,7 @@ http-errors@2.0.0:
     statuses "2.0.1"
     toidentifier "1.0.1"
 
-http-errors@^1.3.1, http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0:
+http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0:
   version "1.8.1"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c"
   integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==
@@ -9265,11 +9247,6 @@ jws@^4.0.0:
     jwa "^2.0.0"
     safe-buffer "^5.0.1"
 
-keygrip@~1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc"
-  integrity sha512-/PpesirAIfaklxUzp4Yb7xBper9MwP6hNRA6BGGUFCgbJ+BM5CKBtsoxinNXkLHAr+GXS1/lSlF2rP7cv5Fl+g==
-
 keygrip@~1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.1.0.tgz#871b1681d5e159c62a445b0c74b615e0917e7226"
@@ -9355,13 +9332,6 @@ koa-body@4.2.0:
     co-body "^5.1.1"
     formidable "^1.1.1"
 
-koa-compose@^3.0.0:
-  version "3.2.1"
-  resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-3.2.1.tgz#a85ccb40b7d986d8e5a345b3a1ace8eabcf54de7"
-  integrity sha512-8gen2cvKHIZ35eDEik5WOo8zbVp9t4cP8p4hW4uE55waxolLRexKKrqfCpwhGVppnB40jWeF8bZeTVg99eZgPw==
-  dependencies:
-    any-promise "^1.1.0"
-
 koa-compose@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-4.1.0.tgz#507306b9371901db41121c812e923d0d67d3e877"
@@ -9383,14 +9353,6 @@ koa-connect@2.1.0:
   resolved "https://registry.yarnpkg.com/koa-connect/-/koa-connect-2.1.0.tgz#16bce0a917c4cb24233aaac83fbc5b83804b4a1c"
   integrity sha512-O9pcFafHk0oQsBevlbTBlB9co+2RUQJ4zCzu3qJPmGlGoeEZkne+7gWDkecqDPSbCtED6LmhlQladxs6NjOnMQ==
 
-koa-convert@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-1.2.0.tgz#da40875df49de0539098d1700b50820cebcd21d0"
-  integrity sha512-K9XqjmEDStGX09v3oxR7t5uPRy0jqJdvodHa6wxWTHrTfDq0WUNnYTOOUZN6g8OM8oZQXprQASbiIXG2Ez8ehA==
-  dependencies:
-    co "^4.6.0"
-    koa-compose "^3.0.0"
-
 koa-convert@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-2.0.0.tgz#86a0c44d81d40551bae22fee6709904573eea4f5"
@@ -9492,37 +9454,7 @@ koa2-ratelimit@1.1.1:
   resolved "https://registry.yarnpkg.com/koa2-ratelimit/-/koa2-ratelimit-1.1.1.tgz#9c1d8257770e4a0a08063ba2ddcaf690fd457d23"
   integrity sha512-IpxGMdZqEhMykW0yYKGVB4vDEacPvSBH4hNpDL38ABj3W2KHNLujAljGEDg7eEjXvrRbXRSWXzANhV3c9v7nyg==
 
-koa@2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/koa/-/koa-2.7.0.tgz#7e00843506942b9d82c6cc33749f657c6e5e7adf"
-  integrity sha512-7ojD05s2Q+hFudF8tDLZ1CpCdVZw8JQELWSkcfG9bdtoTDzMmkRF6BQBU7JzIzCCOY3xd3tftiy/loHBUYaY2Q==
-  dependencies:
-    accepts "^1.3.5"
-    cache-content-type "^1.0.0"
-    content-disposition "~0.5.2"
-    content-type "^1.0.4"
-    cookies "~0.7.1"
-    debug "~3.1.0"
-    delegates "^1.0.0"
-    depd "^1.1.2"
-    destroy "^1.0.4"
-    error-inject "^1.0.0"
-    escape-html "^1.0.3"
-    fresh "~0.5.2"
-    http-assert "^1.3.0"
-    http-errors "^1.6.3"
-    is-generator-function "^1.0.7"
-    koa-compose "^4.1.0"
-    koa-convert "^1.2.0"
-    koa-is-json "^1.0.0"
-    on-finished "^2.3.0"
-    only "~0.0.2"
-    parseurl "^1.3.2"
-    statuses "^1.5.0"
-    type-is "^1.6.16"
-    vary "^1.1.2"
-
-koa@^2.13.1, koa@^2.13.4:
+koa@2.13.4, koa@^2.13.1, koa@^2.13.4:
   version "2.13.4"
   resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e"
   integrity sha512-43zkIKubNbnrULWlHdN5h1g3SEKXOEzoAlRsHOTFpnlDu8JlAOZSMJBLULusuXRequboiwJcj5vtYXKB3k7+2g==
@@ -10116,7 +10048,7 @@ merge2@^1.3.0, merge2@^1.4.1:
   resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
   integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
 
-methods@^1.0.1, methods@^1.1.1, methods@^1.1.2:
+methods@^1.1.1, methods@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==
@@ -11202,7 +11134,7 @@ path-parser@^6.1.0:
     search-params "3.0.0"
     tslib "^1.10.0"
 
-path-to-regexp@1.x, path-to-regexp@^1.1.1:
+path-to-regexp@1.x:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a"
   integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==
@@ -14203,7 +14135,7 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
-urijs@^1.19.0, urijs@^1.19.2:
+urijs@^1.19.2:
   version "1.19.11"
   resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.11.tgz#204b0d6b605ae80bea54bea39280cdb7c9f923cc"
   integrity sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==
diff --git a/packages/worker/package.json b/packages/worker/package.json
index 46927f8725..3ce04ac0f2 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -72,7 +72,6 @@
   "devDependencies": {
     "@types/jest": "26.0.23",
     "@types/koa": "2.13.4",
-    "@types/koa-router": "7.4.4",
     "@types/koa__router": "8.0.11",
     "@types/node": "14.18.20",
     "@types/uuid": "8.3.4",

From 1f36eec89a7faa297b6a589536b0110c6327639c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 10 Oct 2022 20:08:59 +0100
Subject: [PATCH 06/46] Some updates towards supporting attachments in app
 exports.

---
 .../backend-core/src/objectStore/index.ts     | 64 +++++++++++++++++--
 packages/builder/src/stores/portal/apps.js    | 13 ++++
 packages/server/src/api/controllers/backup.ts |  2 +-
 packages/server/src/api/controllers/cloud.js  |  2 +-
 .../src/api/controllers/static/index.ts       |  4 +-
 packages/server/src/constants/index.js        |  6 +-
 packages/server/src/sdk/app/export.ts         | 59 ++++++++---------
 .../src/utilities/fileSystem/utilities.js     |  2 +
 packages/worker/src/sdk/users/users.ts        | 11 +---
 9 files changed, 110 insertions(+), 53 deletions(-)

diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts
index 17e002cc49..903ba28ed1 100644
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@@ -18,6 +18,10 @@ const STATE = {
   bucketCreationPromises: {},
 }
 
+type ListParams = {
+  ContinuationToken?: string
+}
+
 const CONTENT_TYPE_MAP: any = {
   html: "text/html",
   css: "text/css",
@@ -93,7 +97,7 @@ export const ObjectStore = (bucket: any) => {
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export const makeSureBucketExists = async (client: any, bucketName: any) => {
+export const makeSureBucketExists = async (client: any, bucketName: string) => {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -168,8 +172,8 @@ export const upload = async ({
  * through to the object store.
  */
 export const streamUpload = async (
-  bucketName: any,
-  filename: any,
+  bucketName: string,
+  filename: string,
   stream: any,
   extra = {}
 ) => {
@@ -202,7 +206,7 @@ export const streamUpload = async (
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export const retrieve = async (bucketName: any, filepath: any) => {
+export const retrieve = async (bucketName: string, filepath: string) => {
   const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -217,10 +221,38 @@ export const retrieve = async (bucketName: any, filepath: any) => {
   }
 }
 
+export const listAllObjects = async (bucketName: string, path: string) => {
+  const objectStore = ObjectStore(bucketName)
+  const list = (params: ListParams = {}) => {
+    return objectStore
+      .listObjectsV2({
+        ...params,
+        Bucket: sanitizeBucket(bucketName),
+        Prefix: sanitizeKey(path),
+      })
+      .promise()
+  }
+  let isTruncated = false,
+    token,
+    objects: AWS.S3.Types.Object[] = []
+  do {
+    let params: ListParams = {}
+    if (token) {
+      params.ContinuationToken = token
+    }
+    const response = await list(params)
+    if (response.Contents) {
+      objects = objects.concat(response.Contents)
+    }
+    isTruncated = !!response.IsTruncated
+  } while (isTruncated)
+  return objects
+}
+
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-export const retrieveToTmp = async (bucketName: any, filepath: any) => {
+export const retrieveToTmp = async (bucketName: string, filepath: string) => {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
@@ -229,10 +261,30 @@ export const retrieveToTmp = async (bucketName: any, filepath: any) => {
   return outputPath
 }
 
+export const retrieveDirectory = async (bucketName: string, path: string) => {
+  let writePath = join(budibaseTempDir(), v4())
+  const objects = await listAllObjects(bucketName, path)
+  let fullObjects = await Promise.all(
+    objects.map(obj => retrieve(bucketName, obj.Key!))
+  )
+  let count = 0
+  for (let obj of objects) {
+    const filename = obj.Key!
+    const data = fullObjects[count++]
+    const possiblePath = filename.split("/")
+    if (possiblePath.length > 1) {
+      const dirs = possiblePath.slice(0, possiblePath.length - 1)
+      fs.mkdirSync(join(writePath, ...dirs), { recursive: true })
+    }
+    fs.writeFileSync(join(writePath, ...possiblePath), data)
+  }
+  return writePath
+}
+
 /**
  * Delete a single file.
  */
-export const deleteFile = async (bucketName: any, filepath: any) => {
+export const deleteFile = async (bucketName: string, filepath: string) => {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
diff --git a/packages/builder/src/stores/portal/apps.js b/packages/builder/src/stores/portal/apps.js
index a83e35e941..f84e0c973c 100644
--- a/packages/builder/src/stores/portal/apps.js
+++ b/packages/builder/src/stores/portal/apps.js
@@ -2,6 +2,9 @@ import { writable } from "svelte/store"
 import { AppStatus } from "../../constants"
 import { API } from "api"
 
+// properties that should always come from the dev app, not the deployed
+const DEV_PROPS = ["updatedBy", "updatedAt"]
+
 const extractAppId = id => {
   const split = id?.split("_") || []
   return split.length ? split[split.length - 1] : null
@@ -57,9 +60,19 @@ export function createAppStore() {
           return
         }
 
+        let devProps = {}
+        if (appMap[id]) {
+          const entries = Object.entries(appMap[id]).filter(
+            ([key]) => DEV_PROPS.indexOf(key) !== -1
+          )
+          entries.forEach(entry => {
+            devProps[entry[0]] = entry[1]
+          })
+        }
         appMap[id] = {
           ...appMap[id],
           ...app,
+          ...devProps,
           prodId: app.appId,
           prodRev: app._rev,
         }
diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts
index cd2ea415a4..878a81e110 100644
--- a/packages/server/src/api/controllers/backup.ts
+++ b/packages/server/src/api/controllers/backup.ts
@@ -9,7 +9,7 @@ export async function exportAppDump(ctx: any) {
   excludeRows = isQsTrue(excludeRows)
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
-  ctx.body = await sdk.apps.exports.streamBackup(appId, excludeRows)
+  ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()
diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index 55aa6bb548..5de5141d74 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -35,7 +35,7 @@ exports.exportApps = async ctx => {
     // only export the dev apps as they will be the latest, the user can republish the apps
     // in their self hosted environment
     if (isDevAppID(appId)) {
-      allDBs[app.name] = await sdk.apps.exports.exportDB(appId)
+      allDBs[app.name] = await sdk.apps.exports.exportApp(appId)
     }
   }
   const filename = `cloud-export-${new Date().getTime()}.txt`
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 08213c2cf8..f60dc12971 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -5,7 +5,7 @@ require("svelte/register")
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
-const { ObjectStoreBuckets } = require("../../../constants")
+const { ObjectStoreBuckets, ATTACHMENT_PATH } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
 const {
   loadHandlebarsFile,
@@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) {
 
     return prepareUpload({
       file,
-      s3Key: `${ctx.appId}/attachments/${processedFileName}`,
+      s3Key: `${ctx.appId}/${ATTACHMENT_PATH}/${processedFileName}`,
       bucket: ObjectStoreBuckets.APPS,
     })
   })
diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js
index c002c10f7b..c1a992f70e 100644
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@@ -1,6 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { UserStatus } = require("@budibase/backend-core/constants")
-const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore")
+const { objectStore } = require("@budibase/backend-core")
 
 exports.JobQueues = {
   AUTOMATIONS: "automationQueue",
@@ -209,6 +209,8 @@ exports.AutomationErrors = {
 }
 
 // pass through the list from the auth/core lib
-exports.ObjectStoreBuckets = ObjectStoreBuckets
+exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets
+
+exports.ATTACHMENT_PATH = "attachments"
 
 exports.MAX_AUTOMATION_RECURRING_ERRORS = 5
diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts
index 70c4446273..bf7ea81cbc 100644
--- a/packages/server/src/sdk/app/export.ts
+++ b/packages/server/src/sdk/app/export.ts
@@ -1,7 +1,10 @@
-import { closeDB, dangerousGetDB, doWithDB } from "@budibase/backend-core/db"
+import { db as dbCore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../utilities/budibaseDir"
-import { streamUpload } from "../../utilities/fileSystem/utilities"
-import { ObjectStoreBuckets } from "../../constants"
+import {
+  streamUpload,
+  retrieveDirectory,
+} from "../../utilities/fileSystem/utilities"
+import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants"
 import {
   LINK_USER_METADATA_PREFIX,
   TABLE_ROW_PREFIX,
@@ -25,16 +28,16 @@ export async function exportDB(
 ) {
   // streaming a DB dump is a bit more complicated, can't close DB
   if (opts?.stream) {
-    const db = dangerousGetDB(dbName)
+    const db = dbCore.dangerousGetDB(dbName)
     const memStream = new MemoryStream()
     memStream.on("end", async () => {
-      await closeDB(db)
+      await dbCore.closeDB(db)
     })
     db.dump(memStream, { filter: opts?.filter })
     return memStream
   }
 
-  return doWithDB(dbName, async (db: any) => {
+  return dbCore.doWithDB(dbName, async (db: any) => {
     // Write the dump to file if required
     if (opts?.exportName) {
       const path = join(budibaseTempDir(), opts?.exportName)
@@ -49,18 +52,17 @@ export async function exportDB(
           fs.createReadStream(path)
         )
       }
-
       return fs.createReadStream(path)
+    } else {
+      // Stringify the dump in memory if required
+      const memStream = new MemoryStream()
+      let appString = ""
+      memStream.on("data", (chunk: any) => {
+        appString += chunk.toString()
+      })
+      await db.dump(memStream, { filter: opts?.filter })
+      return appString
     }
-
-    // Stringify the dump in memory if required
-    const memStream = new MemoryStream()
-    let appString = ""
-    memStream.on("data", (chunk: any) => {
-      appString += chunk.toString()
-    })
-    await db.dump(memStream, { filter: opts?.filter })
-    return appString
   })
 }
 
@@ -81,12 +83,17 @@ function defineFilter(excludeRows?: boolean) {
  * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} either a string or a stream of the backup
  */
-async function backupAppData(
+export async function exportApp(
   appId: string,
-  config: any,
+  config?: any,
   excludeRows?: boolean
 ) {
-  return await exportDB(appId, {
+  const attachmentsPath = `${dbCore.getProdAppID(appId)}/${ATTACHMENT_PATH}`
+  const tmpPath = await retrieveDirectory(
+    ObjectStoreBuckets.APPS,
+    attachmentsPath
+  )
+  await exportDB(appId, {
     ...config,
     filter: defineFilter(excludeRows),
   })
@@ -98,16 +105,6 @@ async function backupAppData(
  * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-export async function streamBackup(appId: string, excludeRows: boolean) {
-  return await backupAppData(appId, { stream: true }, excludeRows)
-}
-
-/**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
- */
-export async function performBackup(appId: string, backupName: string) {
-  return await backupAppData(appId, { exportName: backupName })
+export async function streamExportApp(appId: string, excludeRows: boolean) {
+  return await exportApp(appId, { stream: true }, excludeRows)
 }
diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js
index 1c804c0142..01ba58f5bc 100644
--- a/packages/server/src/utilities/fileSystem/utilities.js
+++ b/packages/server/src/utilities/fileSystem/utilities.js
@@ -6,6 +6,7 @@ const {
   streamUpload,
   retrieve,
   retrieveToTmp,
+  retrieveDirectory,
   deleteFolder,
   uploadDirectory,
   downloadTarball,
@@ -27,6 +28,7 @@ exports.upload = upload
 exports.streamUpload = streamUpload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
+exports.retrieveDirectory = retrieveDirectory
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index 3b98c8ef52..ce03a12587 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -99,16 +99,7 @@ export const paginatedUsers = async ({
  */
 export const getUser = async (userId: string) => {
   const db = tenancy.getGlobalDB()
-  let user
-  try {
-    user = await db.get(userId)
-  } catch (err: any) {
-    // no user found, just return nothing
-    if (err.status === 404) {
-      return {}
-    }
-    throw err
-  }
+  let user = await db.get(userId)
   if (user) {
     delete user.password
   }

From 7c71f76b705374506ca24fed38e05216f15e9b21 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 11 Oct 2022 18:21:58 +0100
Subject: [PATCH 07/46] Export to tarball through tmp.

---
 packages/server/src/api/controllers/cloud.js  | 119 +++++++++---------
 packages/server/src/sdk/app/export.ts         | 116 +++++++++++------
 .../server/src/utilities/fileSystem/index.js  |   4 +
 3 files changed, 136 insertions(+), 103 deletions(-)

diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index 5de5141d74..d766a7987f 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -1,24 +1,9 @@
 const env = require("../../environment")
 const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db")
-const { sendTempFile, readFileSync } = require("../../utilities/fileSystem")
-const { stringToReadStream } = require("../../utilities")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { create } = require("./application")
-const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils")
+const { streamFile } = require("../../utilities/fileSystem")
+const { DocumentType, isDevAppID } = require("../../db/utils")
 const sdk = require("../../sdk")
 
-async function createApp(appName, appImport) {
-  const ctx = {
-    request: {
-      body: {
-        templateString: appImport,
-        name: appName,
-      },
-    },
-  }
-  return create(ctx)
-}
-
 exports.exportApps = async ctx => {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
@@ -27,29 +12,18 @@ exports.exportApps = async ctx => {
   const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentType.USER),
   })
-  let allDBs = {
-    global: globalDBString,
-  }
-  for (let app of apps) {
-    const appId = app.appId || app._id
-    // only export the dev apps as they will be the latest, the user can republish the apps
-    // in their self hosted environment
-    if (isDevAppID(appId)) {
-      allDBs[app.name] = await sdk.apps.exports.exportApp(appId)
-    }
-  }
-  const filename = `cloud-export-${new Date().getTime()}.txt`
-  ctx.attachment(filename)
-  ctx.body = sendTempFile(JSON.stringify(allDBs))
-}
-
-async function getAllDocType(db, docType) {
-  const response = await db.allDocs(
-    getDocParams(docType, null, {
-      include_docs: true,
-    })
+  // only export the dev apps as they will be the latest, the user can republish the apps
+  // in their self-hosted environment
+  let appIds = apps
+    .map(app => app.appId || app._id)
+    .filter(appId => isDevAppID(appId))
+  const tmpPath = await sdk.apps.exports.exportMultipleApps(
+    appIds,
+    globalDBString
   )
-  return response.rows.map(row => row.doc)
+  const filename = `cloud-export-${new Date().getTime()}.tar.gz`
+  ctx.attachment(filename)
+  ctx.body = streamFile(tmpPath)
 }
 
 async function hasBeenImported() {
@@ -77,30 +51,51 @@ exports.importApps = async ctx => {
       "Import file is required and environment must be fresh to import apps."
     )
   }
-  const importFile = ctx.request.files.importFile
-  const importString = readFileSync(importFile.path)
-  const dbs = JSON.parse(importString)
-  const globalDbImport = dbs.global
-  // remove from the list of apps
-  delete dbs.global
-  const globalDb = getGlobalDB()
-  // load the global db first
-  await globalDb.load(stringToReadStream(globalDbImport))
-  for (let [appName, appImport] of Object.entries(dbs)) {
-    await createApp(appName, appImport)
-  }
 
-  // if there are any users make sure to remove them
-  let users = await getAllDocType(globalDb, DocumentType.USER)
-  let userDeletionPromises = []
-  for (let user of users) {
-    userDeletionPromises.push(globalDb.remove(user._id, user._rev))
-  }
-  if (userDeletionPromises.length > 0) {
-    await Promise.all(userDeletionPromises)
-  }
-
-  await globalDb.bulkDocs(users)
+  // TODO: IMPLEMENT TARBALL EXTRACTION, APP IMPORT, ATTACHMENT IMPORT AND GLOBAL DB IMPORT
+  // async function getAllDocType(db, docType) {
+  //   const response = await db.allDocs(
+  //     getDocParams(docType, null, {
+  //       include_docs: true,
+  //     })
+  //   )
+  //   return response.rows.map(row => row.doc)
+  // }
+  // async function createApp(appName, appImport) {
+  //   const ctx = {
+  //     request: {
+  //       body: {
+  //         templateString: appImport,
+  //         name: appName,
+  //       },
+  //     },
+  //   }
+  //   return create(ctx)
+  // }
+  // const importFile = ctx.request.files.importFile
+  // const importString = readFileSync(importFile.path)
+  // const dbs = JSON.parse(importString)
+  // const globalDbImport = dbs.global
+  // // remove from the list of apps
+  // delete dbs.global
+  // const globalDb = getGlobalDB()
+  // // load the global db first
+  // await globalDb.load(stringToReadStream(globalDbImport))
+  // for (let [appName, appImport] of Object.entries(dbs)) {
+  //   await createApp(appName, appImport)
+  // }
+  //
+  // // if there are any users make sure to remove them
+  // let users = await getAllDocType(globalDb, DocumentType.USER)
+  // let userDeletionPromises = []
+  // for (let user of users) {
+  //   userDeletionPromises.push(globalDb.remove(user._id, user._rev))
+  // }
+  // if (userDeletionPromises.length > 0) {
+  //   await Promise.all(userDeletionPromises)
+  // }
+  //
+  // await globalDb.bulkDocs(users)
   ctx.body = {
     message: "Apps successfully imported.",
   }
diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts
index bf7ea81cbc..ad71062483 100644
--- a/packages/server/src/sdk/app/export.ts
+++ b/packages/server/src/sdk/app/export.ts
@@ -1,9 +1,7 @@
 import { db as dbCore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../utilities/budibaseDir"
-import {
-  streamUpload,
-  retrieveDirectory,
-} from "../../utilities/fileSystem/utilities"
+import { retrieveDirectory } from "../../utilities/fileSystem/utilities"
+import { streamFile } from "../../utilities/fileSystem"
 import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants"
 import {
   LINK_USER_METADATA_PREFIX,
@@ -11,10 +9,35 @@ import {
   USER_METDATA_PREFIX,
 } from "../../db/utils"
 import fs from "fs"
-import env from "../../environment"
 import { join } from "path"
+const uuid = require("uuid/v4")
+const tar = require("tar")
 const MemoryStream = require("memorystream")
 
+const DB_EXPORT_FILE = "db.txt"
+const GLOBAL_DB_EXPORT_FILE = "global.txt"
+type ExportOpts = {
+  filter?: any
+  exportPath?: string
+  tar?: boolean
+  excludeRows?: boolean
+}
+
+function tarFiles(cwd: string, files: string[], exportName?: string) {
+  exportName = exportName ? `${exportName}.tar.gz` : "export.tar.gz"
+  tar.create(
+    {
+      sync: true,
+      gzip: true,
+      file: exportName,
+      recursive: true,
+      cwd,
+    },
+    files
+  )
+  return join(cwd, exportName)
+}
+
 /**
  * Exports a DB to either file or a variable (memory).
  * @param {string} dbName the DB which is to be exported.
@@ -22,36 +45,13 @@ const MemoryStream = require("memorystream")
  * a filter function or the name of the export.
  * @return {*} either a readable stream or a string
  */
-export async function exportDB(
-  dbName: string,
-  opts: { stream?: boolean; filter?: any; exportName?: string } = {}
-) {
-  // streaming a DB dump is a bit more complicated, can't close DB
-  if (opts?.stream) {
-    const db = dbCore.dangerousGetDB(dbName)
-    const memStream = new MemoryStream()
-    memStream.on("end", async () => {
-      await dbCore.closeDB(db)
-    })
-    db.dump(memStream, { filter: opts?.filter })
-    return memStream
-  }
-
+export async function exportDB(dbName: string, opts: ExportOpts = {}) {
   return dbCore.doWithDB(dbName, async (db: any) => {
     // Write the dump to file if required
-    if (opts?.exportName) {
-      const path = join(budibaseTempDir(), opts?.exportName)
+    if (opts?.exportPath) {
+      const path = opts?.exportPath
       const writeStream = fs.createWriteStream(path)
       await db.dump(writeStream, { filter: opts?.filter })
-
-      // Upload the dump to the object store if self-hosted
-      if (env.SELF_HOSTED) {
-        await streamUpload(
-          ObjectStoreBuckets.BACKUPS,
-          join(dbName, opts?.exportName),
-          fs.createReadStream(path)
-        )
-      }
       return fs.createReadStream(path)
     } else {
       // Stringify the dump in memory if required
@@ -79,24 +79,57 @@ function defineFilter(excludeRows?: boolean) {
  * Local utility to back up the database state for an app, excluding global user
  * data or user relationships.
  * @param {string} appId The app to back up
- * @param {object} config Config to send to export DB
- * @param {boolean} excludeRows Flag to state whether the export should include data.
+ * @param {object} config Config to send to export DB/attachment export
  * @returns {*} either a string or a stream of the backup
  */
-export async function exportApp(
-  appId: string,
-  config?: any,
-  excludeRows?: boolean
-) {
-  const attachmentsPath = `${dbCore.getProdAppID(appId)}/${ATTACHMENT_PATH}`
+export async function exportApp(appId: string, config?: ExportOpts) {
+  const prodAppId = dbCore.getProdAppID(appId)
+  const attachmentsPath = `${prodAppId}/${ATTACHMENT_PATH}`
+  // export attachments to tmp
   const tmpPath = await retrieveDirectory(
     ObjectStoreBuckets.APPS,
     attachmentsPath
   )
+  // move out of app directory, simplify structure
+  fs.renameSync(join(tmpPath, attachmentsPath), join(tmpPath, ATTACHMENT_PATH))
+  // remove the old app directory created by object export
+  fs.rmdirSync(join(tmpPath, prodAppId))
+  // enforce an export of app DB to the tmp path
+  const dbPath = join(tmpPath, DB_EXPORT_FILE)
   await exportDB(appId, {
     ...config,
-    filter: defineFilter(excludeRows),
+    filter: defineFilter(config?.excludeRows),
+    exportPath: dbPath,
   })
+  // if tar requested, return where the tarball is
+  if (config?.tar) {
+    // now the tmpPath contains both the DB export and attachments, tar this
+    return tarFiles(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE])
+  }
+  // tar not requested, turn the directory where export is
+  else {
+    return tmpPath
+  }
+}
+
+export async function exportMultipleApps(
+  appIds: string[],
+  globalDbContents?: string
+) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  let exportPromises: Promise<void>[] = []
+  const exportAndMove = async (appId: string) => {
+    const path = await exportApp(appId)
+    await fs.promises.rename(path, join(tmpPath, appId))
+  }
+  for (let appId of appIds) {
+    exportPromises.push(exportAndMove(appId))
+  }
+  await Promise.all(exportPromises)
+  if (globalDbContents) {
+    fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
+  }
+  return tarFiles(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE])
 }
 
 /**
@@ -106,5 +139,6 @@ export async function exportApp(
  * @returns {*} a readable stream of the backup which is written in real time
  */
 export async function streamExportApp(appId: string, excludeRows: boolean) {
-  return await exportApp(appId, { stream: true }, excludeRows)
+  const tmpPath = await exportApp(appId, { excludeRows, tar: true })
+  return streamFile(tmpPath)
 }
diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js
index 656ba4816c..69bf38c819 100644
--- a/packages/server/src/utilities/fileSystem/index.js
+++ b/packages/server/src/utilities/fileSystem/index.js
@@ -112,6 +112,10 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
+exports.streamFile = path => {
+  return fs.createReadStream(path)
+}
+
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.
  * @param {string} fileContents contents which will be written to a temp file.

From f237befbce935481aca9c6760d58c4d55d5e0756 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 11 Oct 2022 19:28:13 +0100
Subject: [PATCH 08/46] Some fixes + cleanup of tmp directory.

---
 .../backend-core/src/objectStore/index.ts     |  1 +
 packages/server/src/api/controllers/backup.ts |  2 +-
 packages/server/src/sdk/app/export.ts         | 43 +++++++++++++------
 3 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts
index 903ba28ed1..4c88166375 100644
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@@ -263,6 +263,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => {
 
 export const retrieveDirectory = async (bucketName: string, path: string) => {
   let writePath = join(budibaseTempDir(), v4())
+  fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
   let fullObjects = await Promise.all(
     objects.map(obj => retrieve(bucketName, obj.Key!))
diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts
index 878a81e110..7fe197de76 100644
--- a/packages/server/src/api/controllers/backup.ts
+++ b/packages/server/src/api/controllers/backup.ts
@@ -7,7 +7,7 @@ export async function exportAppDump(ctx: any) {
   let { appId, excludeRows } = ctx.query
   const appName = decodeURI(ctx.query.appname)
   excludeRows = isQsTrue(excludeRows)
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz`
   ctx.attachment(backupIdentifier)
   ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows)
 
diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts
index ad71062483..68aa873129 100644
--- a/packages/server/src/sdk/app/export.ts
+++ b/packages/server/src/sdk/app/export.ts
@@ -23,19 +23,19 @@ type ExportOpts = {
   excludeRows?: boolean
 }
 
-function tarFiles(cwd: string, files: string[], exportName?: string) {
-  exportName = exportName ? `${exportName}.tar.gz` : "export.tar.gz"
+function tarFilesToTmp(tmpDir: string, files: string[]) {
+  const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`)
   tar.create(
     {
       sync: true,
       gzip: true,
-      file: exportName,
+      file: exportFile,
       recursive: true,
-      cwd,
+      cwd: tmpDir,
     },
     files
   )
-  return join(cwd, exportName)
+  return exportFile
 }
 
 /**
@@ -52,7 +52,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
       const path = opts?.exportPath
       const writeStream = fs.createWriteStream(path)
       await db.dump(writeStream, { filter: opts?.filter })
-      return fs.createReadStream(path)
+      return path
     } else {
       // Stringify the dump in memory if required
       const memStream = new MemoryStream()
@@ -90,10 +90,16 @@ export async function exportApp(appId: string, config?: ExportOpts) {
     ObjectStoreBuckets.APPS,
     attachmentsPath
   )
-  // move out of app directory, simplify structure
-  fs.renameSync(join(tmpPath, attachmentsPath), join(tmpPath, ATTACHMENT_PATH))
-  // remove the old app directory created by object export
-  fs.rmdirSync(join(tmpPath, prodAppId))
+  const downloadedPath = join(tmpPath, attachmentsPath),
+    tmpAttachmentPath = join(tmpPath, ATTACHMENT_PATH)
+  if (fs.existsSync(downloadedPath)) {
+    // move out of app directory, simplify structure
+    fs.renameSync(downloadedPath, tmpAttachmentPath)
+    // remove the old app directory created by object export
+    fs.rmdirSync(join(tmpPath, prodAppId))
+  } else {
+    fs.mkdirSync(tmpAttachmentPath)
+  }
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
   await exportDB(appId, {
@@ -104,7 +110,10 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    return tarFiles(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE])
+    const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE])
+    // cleanup the tmp export files as tarball returned
+    fs.rmSync(tmpPath, { recursive: true, force: true })
+    return tarPath
   }
   // tar not requested, turn the directory where export is
   else {
@@ -112,6 +121,13 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   }
 }
 
+/**
+ * Export all apps + global DB (if supplied) to a single tarball, this includes
+ * the attachments for each app as well.
+ * @param {string[]} appIds The IDs of the apps to be exported.
+ * @param {string} globalDbContents The contents of the global DB to export as well.
+ * @return {string} The path to the tarball.
+ */
 export async function exportMultipleApps(
   appIds: string[],
   globalDbContents?: string
@@ -129,7 +145,10 @@ export async function exportMultipleApps(
   if (globalDbContents) {
     fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
   }
-  return tarFiles(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE])
+  const tarPath = tarFilesToTmp(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE])
+  // clear up the tmp path now tarball generated
+  fs.rmSync(tmpPath, { recursive: true, force: true })
+  return tarPath
 }
 
 /**

From 19133f08e6d3190b8600dccd6cfd7a34d242db57 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 11 Oct 2022 20:25:22 +0100
Subject: [PATCH 09/46] Adding import functionality - still need to update the
 attachments URL.

---
 .../backend-core/src/objectStore/index.ts     | 26 ++++---
 .../server/src/api/controllers/application.ts | 13 +---
 .../src/api/controllers/static/index.ts       |  4 +-
 packages/server/src/constants/index.js        |  2 +-
 packages/server/src/sdk/app/constants.ts      |  4 ++
 .../src/sdk/app/{export.ts => exports.ts}     | 15 ++--
 packages/server/src/sdk/app/imports.ts        | 71 +++++++++++++++++++
 packages/server/src/sdk/app/index.ts          |  3 +-
 .../server/src/utilities/fileSystem/index.js  | 15 ----
 9 files changed, 107 insertions(+), 46 deletions(-)
 create mode 100644 packages/server/src/sdk/app/constants.ts
 rename packages/server/src/sdk/app/{export.ts => exports.ts} (93%)
 create mode 100644 packages/server/src/sdk/app/imports.ts

diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts
index 4c88166375..e210f9dccc 100644
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@@ -36,16 +36,16 @@ const STRING_CONTENT_TYPES = [
 ]
 
 // does normal sanitization and then swaps dev apps to apps
-export function sanitizeKey(input: any) {
+export function sanitizeKey(input: string) {
   return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
 }
 
 // simply handles the dev app to app conversion
-export function sanitizeBucket(input: any) {
+export function sanitizeBucket(input: string) {
   return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
 }
 
-function publicPolicy(bucketName: any) {
+function publicPolicy(bucketName: string) {
   return {
     Version: "2012-10-17",
     Statement: [
@@ -73,7 +73,7 @@ const PUBLIC_BUCKETS = [
  * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (bucket: any) => {
+export const ObjectStore = (bucket: string) => {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -295,7 +295,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   return objectStore.deleteObject(params)
 }
 
-export const deleteFiles = async (bucketName: any, filepaths: any) => {
+export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -311,8 +311,8 @@ export const deleteFiles = async (bucketName: any, filepaths: any) => {
  * Delete a path, including everything within.
  */
 export const deleteFolder = async (
-  bucketName: any,
-  folder: any
+  bucketName: string,
+  folder: string
 ): Promise<any> => {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
@@ -345,9 +345,9 @@ export const deleteFolder = async (
 }
 
 export const uploadDirectory = async (
-  bucketName: any,
-  localPath: any,
-  bucketPath: any
+  bucketName: string,
+  localPath: string,
+  bucketPath: string
 ) => {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
@@ -379,7 +379,11 @@ exports.downloadTarballDirect = async (
   await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
 }
 
-export const downloadTarball = async (url: any, bucketName: any, path: any) => {
+export const downloadTarball = async (
+  url: string,
+  bucketName: string,
+  path: string
+) => {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index a7caf85e94..87bc7d0033 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -5,11 +5,7 @@ import {
   createRoutingView,
   createAllSearchIndex,
 } from "../../db/views/staticViews"
-import {
-  getTemplateStream,
-  createApp,
-  deleteApp,
-} from "../../utilities/fileSystem"
+import { createApp, deleteApp } from "../../utilities/fileSystem"
 import {
   generateAppID,
   getLayoutParams,
@@ -50,6 +46,7 @@ import { errors, events, migrations } from "@budibase/backend-core"
 import { App, Layout, Screen, MigrationType } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import { enrichPluginURLs } from "../../utilities/plugins"
+import sdk from "../../sdk"
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -153,11 +150,7 @@ async function createInstance(template: any) {
       throw "Error loading database dump from memory."
     }
   } else if (template && template.useTemplate === "true") {
-    /* istanbul ignore next */
-    const { ok } = await db.load(await getTemplateStream(template))
-    if (!ok) {
-      throw "Error loading database dump from template."
-    }
+    await sdk.apps.imports.importApp(appId, db, template)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index f60dc12971..ec2ea70820 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -5,7 +5,7 @@ require("svelte/register")
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
-const { ObjectStoreBuckets, ATTACHMENT_PATH } = require("../../../constants")
+const { ObjectStoreBuckets, ATTACHMENT_DIR } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
 const {
   loadHandlebarsFile,
@@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) {
 
     return prepareUpload({
       file,
-      s3Key: `${ctx.appId}/${ATTACHMENT_PATH}/${processedFileName}`,
+      s3Key: `${ctx.appId}/${ATTACHMENT_DIR}/${processedFileName}`,
       bucket: ObjectStoreBuckets.APPS,
     })
   })
diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js
index c1a992f70e..b9362cecf6 100644
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@@ -211,6 +211,6 @@ exports.AutomationErrors = {
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 
-exports.ATTACHMENT_PATH = "attachments"
+exports.ATTACHMENT_DIR = "attachments"
 
 exports.MAX_AUTOMATION_RECURRING_ERRORS = 5
diff --git a/packages/server/src/sdk/app/constants.ts b/packages/server/src/sdk/app/constants.ts
new file mode 100644
index 0000000000..e831172683
--- /dev/null
+++ b/packages/server/src/sdk/app/constants.ts
@@ -0,0 +1,4 @@
+import { ATTACHMENT_DIR as attachmentDir } from "../../constants"
+export const DB_EXPORT_FILE = "db.txt"
+export const ATTACHMENT_DIR = attachmentDir
+export const GLOBAL_DB_EXPORT_FILE = "global.txt"
diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/exports.ts
similarity index 93%
rename from packages/server/src/sdk/app/export.ts
rename to packages/server/src/sdk/app/exports.ts
index 68aa873129..1ecf69de68 100644
--- a/packages/server/src/sdk/app/export.ts
+++ b/packages/server/src/sdk/app/exports.ts
@@ -2,20 +2,23 @@ import { db as dbCore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../utilities/budibaseDir"
 import { retrieveDirectory } from "../../utilities/fileSystem/utilities"
 import { streamFile } from "../../utilities/fileSystem"
-import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants"
+import { ObjectStoreBuckets } from "../../constants"
 import {
   LINK_USER_METADATA_PREFIX,
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../db/utils"
+import {
+  DB_EXPORT_FILE,
+  GLOBAL_DB_EXPORT_FILE,
+  ATTACHMENT_DIR,
+} from "./constants"
 import fs from "fs"
 import { join } from "path"
 const uuid = require("uuid/v4")
 const tar = require("tar")
 const MemoryStream = require("memorystream")
 
-const DB_EXPORT_FILE = "db.txt"
-const GLOBAL_DB_EXPORT_FILE = "global.txt"
 type ExportOpts = {
   filter?: any
   exportPath?: string
@@ -84,14 +87,14 @@ function defineFilter(excludeRows?: boolean) {
  */
 export async function exportApp(appId: string, config?: ExportOpts) {
   const prodAppId = dbCore.getProdAppID(appId)
-  const attachmentsPath = `${prodAppId}/${ATTACHMENT_PATH}`
+  const attachmentsPath = `${prodAppId}/${ATTACHMENT_DIR}`
   // export attachments to tmp
   const tmpPath = await retrieveDirectory(
     ObjectStoreBuckets.APPS,
     attachmentsPath
   )
   const downloadedPath = join(tmpPath, attachmentsPath),
-    tmpAttachmentPath = join(tmpPath, ATTACHMENT_PATH)
+    tmpAttachmentPath = join(tmpPath, ATTACHMENT_DIR)
   if (fs.existsSync(downloadedPath)) {
     // move out of app directory, simplify structure
     fs.renameSync(downloadedPath, tmpAttachmentPath)
@@ -110,7 +113,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE])
+    const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_DIR, DB_EXPORT_FILE])
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
     return tarPath
diff --git a/packages/server/src/sdk/app/imports.ts b/packages/server/src/sdk/app/imports.ts
new file mode 100644
index 0000000000..96629db39a
--- /dev/null
+++ b/packages/server/src/sdk/app/imports.ts
@@ -0,0 +1,71 @@
+import { db as dbCore } from "@budibase/backend-core"
+import { budibaseTempDir } from "../../utilities/budibaseDir"
+import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants"
+import { uploadDirectory } from "../../utilities/fileSystem/utilities"
+import { ObjectStoreBuckets } from "../../constants"
+import { join } from "path"
+import fs from "fs"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+
+type TemplateType = {
+  file?: {
+    type: string
+    path: string
+  }
+  key?: string
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns a fs read stream which can be loaded into the database.
+ */
+async function getTemplateStream(template: TemplateType) {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else if (template.key) {
+    const [type, name] = template.key.split("/")
+    const tmpPath = await exports.downloadTemplate(type, name)
+    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
+  }
+}
+
+export async function importApp(
+  appId: string,
+  db: PouchDB.Database,
+  template: TemplateType
+) {
+  let prodAppId = dbCore.getProdAppID(appId)
+  let dbStream: any
+  if (template.file && template.file.type === "application/gzip") {
+    const tmpPath = join(budibaseTempDir(), uuid())
+    fs.mkdirSync(tmpPath)
+    // extract the tarball
+    tar.extract({
+      sync: true,
+      cwd: tmpPath,
+      file: template.file.path,
+    })
+    const attachmentPath = join(tmpPath, ATTACHMENT_DIR)
+    // have to handle object import
+    if (fs.existsSync(attachmentPath)) {
+      await uploadDirectory(
+        ObjectStoreBuckets.APPS,
+        attachmentPath,
+        join(prodAppId, ATTACHMENT_DIR)
+      )
+    }
+    dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE))
+  } else {
+    dbStream = await getTemplateStream(template)
+  }
+  // @ts-ignore
+  const { ok } = await db.load(dbStream)
+  if (!ok) {
+    throw "Error loading database dump from template."
+  } else {
+    // TODO: need to iterate over attachments and update their URLs
+  }
+  return ok
+}
diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts
index 3927539bc8..5ac7c04b2c 100644
--- a/packages/server/src/sdk/app/index.ts
+++ b/packages/server/src/sdk/app/index.ts
@@ -1 +1,2 @@
-export * as exports from "./export"
+export * as exports from "./exports"
+export * as imports from "./imports"
diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js
index 69bf38c819..1eb8a481e5 100644
--- a/packages/server/src/utilities/fileSystem/index.js
+++ b/packages/server/src/utilities/fileSystem/index.js
@@ -74,21 +74,6 @@ exports.checkDevelopmentEnvironment = () => {
   }
 }
 
-/**
- * This function manages temporary template files which are stored by Koa.
- * @param {Object} template The template object retrieved from the Koa context object.
- * @returns {Object} Returns an fs read stream which can be loaded into the database.
- */
-exports.getTemplateStream = async template => {
-  if (template.file) {
-    return fs.createReadStream(template.file.path)
-  } else {
-    const [type, name] = template.key.split("/")
-    const tmpPath = await exports.downloadTemplate(type, name)
-    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
-  }
-}
-
 /**
  * Used to retrieve a handlebars file from the system which will be used as a template.
  * This is allowable as the template handlebars files should be static and identical across

From 8d7f40e443a6ee81f6cdfdc8651f98b4b45d927b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 12 Oct 2022 17:02:23 +0100
Subject: [PATCH 10/46] Refactoring more to Typescript, adding the ability to
 use the _find API of CouchDB.

---
 .../src/db/{index.js => index.ts}             |  54 ++++--
 packages/backend-core/src/db/utils.ts         |  30 +---
 .../server/src/api/controllers/application.ts |   2 +-
 packages/server/src/api/controllers/backup.ts |   2 +-
 packages/server/src/api/controllers/cloud.js  |   7 +-
 .../src/api/controllers/table/bulkFormula.js  |   7 +-
 .../src/api/controllers/table/external.js     |   8 +-
 .../server/src/api/controllers/table/index.js |   8 +-
 .../src/api/controllers/table/internal.ts     |  10 +-
 .../server/src/api/controllers/table/utils.ts |  40 -----
 .../server/src/api/controllers/view/index.js  |   4 +-
 .../server/src/automations/automationUtils.js |   4 +-
 .../functions/backfill/app/tables.ts          |  19 +--
 .../src/sdk/app/{ => backups}/constants.ts    |   0
 .../src/sdk/app/{ => backups}/exports.ts      |  10 +-
 .../src/sdk/app/{ => backups}/imports.ts      |  33 +++-
 packages/server/src/sdk/app/backups/index.ts  |   7 +
 packages/server/src/sdk/app/index.ts          |   2 -
 packages/server/src/sdk/app/tables/index.ts   |  60 +++++++
 packages/server/src/sdk/index.ts              |   6 +-
 packages/types/package.json                   |   3 +-
 packages/types/src/core/db.ts                 |  22 +++
 packages/types/src/core/index.ts              |   1 +
 packages/types/src/documents/app/table.ts     |   1 +
 packages/types/src/index.ts                   |   1 +
 packages/types/yarn.lock                      | 158 ++++++++++++++++++
 26 files changed, 369 insertions(+), 130 deletions(-)
 rename packages/backend-core/src/db/{index.js => index.ts} (58%)
 rename packages/server/src/sdk/app/{ => backups}/constants.ts (100%)
 rename packages/server/src/sdk/app/{ => backups}/exports.ts (94%)
 rename packages/server/src/sdk/app/{ => backups}/imports.ts (62%)
 create mode 100644 packages/server/src/sdk/app/backups/index.ts
 delete mode 100644 packages/server/src/sdk/app/index.ts
 create mode 100644 packages/server/src/sdk/app/tables/index.ts
 create mode 100644 packages/types/src/core/db.ts
 create mode 100644 packages/types/src/core/index.ts

diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.ts
similarity index 58%
rename from packages/backend-core/src/db/index.js
rename to packages/backend-core/src/db/index.ts
index aa6f7ebc2c..016155033a 100644
--- a/packages/backend-core/src/db/index.js
+++ b/packages/backend-core/src/db/index.ts
@@ -1,8 +1,11 @@
-const pouch = require("./pouch")
-const env = require("../environment")
+import pouch from "./pouch"
+import env from "../environment"
+import { checkSlashesInUrl } from "../helpers"
+import fetch from "node-fetch"
+import { PouchOptions, CouchFindOptions } from "@budibase/types"
 
-const openDbs = []
-let PouchDB
+const openDbs: string[] = []
+let PouchDB: any
 let initialised = false
 const dbList = new Set()
 
@@ -14,8 +17,8 @@ if (env.MEMORY_LEAK_CHECK) {
 }
 
 const put =
-  dbPut =>
-  async (doc, options = {}) => {
+  (dbPut: any) =>
+  async (doc: any, options = {}) => {
     if (!doc.createdAt) {
       doc.createdAt = new Date().toISOString()
     }
@@ -29,7 +32,7 @@ const checkInitialised = () => {
   }
 }
 
-exports.init = opts => {
+export async function init(opts: PouchOptions) {
   PouchDB = pouch.getPouch(opts)
   initialised = true
 }
@@ -37,7 +40,7 @@ exports.init = opts => {
 // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
 // this function is prone to leaks, should only be used
 // in situations that using the function doWithDB does not work
-exports.dangerousGetDB = (dbName, opts) => {
+export async function dangerousGetDB(dbName: string, opts: any) {
   checkInitialised()
   if (env.isTest()) {
     dbList.add(dbName)
@@ -53,7 +56,7 @@ exports.dangerousGetDB = (dbName, opts) => {
 
 // use this function if you have called dangerousGetDB - close
 // the databases you've opened once finished
-exports.closeDB = async db => {
+export async function closeDB(db: PouchDB.Database) {
   if (!db || env.isTest()) {
     return
   }
@@ -71,7 +74,7 @@ exports.closeDB = async db => {
 // we have to use a callback for this so that we can close
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
-exports.doWithDB = async (dbName, cb, opts = {}) => {
+export async function doWithDB(dbName: string, cb: any, opts = {}) {
   const db = exports.dangerousGetDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
   // async operations have been completed
@@ -82,10 +85,39 @@ exports.doWithDB = async (dbName, cb, opts = {}) => {
   }
 }
 
-exports.allDbs = () => {
+export function allDbs() {
   if (!env.isTest()) {
     throw new Error("Cannot be used outside test environment.")
   }
   checkInitialised()
   return [...dbList]
 }
+
+export async function directCouchQuery(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  let { url, cookie } = pouch.getCouchInfo()
+  const couchUrl = `${url}/${path}`
+  const params: any = {
+    method: method,
+    headers: {
+      Authorization: cookie,
+    },
+  }
+  if (body && method !== "GET") {
+    params.body = body
+  }
+  const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
+  if (response.status < 300) {
+    return await response.json()
+  } else {
+    throw "Cannot connect to CouchDB instance"
+  }
+}
+
+export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
+  const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
+  return { rows: json.docs, bookmark: json.bookmark }
+}
diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts
index 1c4be7e366..c78c11217e 100644
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@@ -4,11 +4,8 @@ import env from "../environment"
 import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants"
 import { getTenantId, getGlobalDB } from "../context"
 import { getGlobalDBName } from "./tenancy"
-import fetch from "node-fetch"
-import { doWithDB, allDbs } from "./index"
-import { getCouchInfo } from "./pouch"
+import { doWithDB, allDbs, directCouchQuery } from "./index"
 import { getAppMetadata } from "../cache/appMetadata"
-import { checkSlashesInUrl } from "../helpers"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
 import * as events from "../events"
@@ -209,22 +206,11 @@ export async function getAllDbs(opts = { efficient: false }) {
     return allDbs()
   }
   let dbs: any[] = []
-  let { url, cookie } = getCouchInfo()
-  async function addDbs(couchUrl: string) {
-    const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), {
-      method: "GET",
-      headers: {
-        Authorization: cookie,
-      },
-    })
-    if (response.status === 200) {
-      let json = await response.json()
-      dbs = dbs.concat(json)
-    } else {
-      throw "Cannot connect to CouchDB instance"
-    }
+  async function addDbs(couchPath: string) {
+    const json = await directCouchQuery(couchPath)
+    dbs = dbs.concat(json)
   }
-  let couchUrl = `${url}/_all_dbs`
+  let couchPath = "/_all_dbs"
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
     // just get all DBs when:
@@ -232,12 +218,12 @@ export async function getAllDbs(opts = { efficient: false }) {
     // - default tenant
     //    - apps dbs don't contain tenant id
     //    - non-default tenant dbs are filtered out application side in getAllApps
-    await addDbs(couchUrl)
+    await addDbs(couchPath)
   } else {
     // get prod apps
-    await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP, tenantId))
+    await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP, tenantId))
     // get dev apps
-    await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP_DEV, tenantId))
+    await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP_DEV, tenantId))
     // add global db name
     dbs.push(getGlobalDBName(tenantId))
   }
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 87bc7d0033..f3dca51f72 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -150,7 +150,7 @@ async function createInstance(template: any) {
       throw "Error loading database dump from memory."
     }
   } else if (template && template.useTemplate === "true") {
-    await sdk.apps.imports.importApp(appId, db, template)
+    await sdk.backups.importApp(appId, db, template)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts
index 7fe197de76..0ffda2c733 100644
--- a/packages/server/src/api/controllers/backup.ts
+++ b/packages/server/src/api/controllers/backup.ts
@@ -9,7 +9,7 @@ export async function exportAppDump(ctx: any) {
   excludeRows = isQsTrue(excludeRows)
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz`
   ctx.attachment(backupIdentifier)
-  ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows)
+  ctx.body = await sdk.backups.streamExportApp(appId, excludeRows)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()
diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index d766a7987f..323c7409db 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -9,7 +9,7 @@ exports.exportApps = async ctx => {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
   const apps = await getAllApps({ all: true })
-  const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), {
+  const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentType.USER),
   })
   // only export the dev apps as they will be the latest, the user can republish the apps
@@ -17,10 +17,7 @@ exports.exportApps = async ctx => {
   let appIds = apps
     .map(app => app.appId || app._id)
     .filter(appId => isDevAppID(appId))
-  const tmpPath = await sdk.apps.exports.exportMultipleApps(
-    appIds,
-    globalDBString
-  )
+  const tmpPath = await sdk.backups.exportMultipleApps(appIds, globalDBString)
   const filename = `cloud-export-${new Date().getTime()}.tar.gz`
   ctx.attachment(filename)
   ctx.body = streamFile(tmpPath)
diff --git a/packages/server/src/api/controllers/table/bulkFormula.js b/packages/server/src/api/controllers/table/bulkFormula.js
index d736c126f2..733c16d455 100644
--- a/packages/server/src/api/controllers/table/bulkFormula.js
+++ b/packages/server/src/api/controllers/table/bulkFormula.js
@@ -1,10 +1,11 @@
 const { FieldTypes, FormulaTypes } = require("../../../constants")
-const { getAllInternalTables, clearColumns } = require("./utils")
+const { clearColumns } = require("./utils")
 const { doesContainStrings } = require("@budibase/string-templates")
 const { cloneDeep } = require("lodash/fp")
 const { isEqual, uniq } = require("lodash")
 const { updateAllFormulasInTable } = require("../row/staticFormula")
 const { getAppDB } = require("@budibase/backend-core/context")
+const sdk = require("../../../sdk")
 
 function isStaticFormula(column) {
   return (
@@ -39,7 +40,7 @@ function getFormulaThatUseColumn(table, columnNames) {
  */
 async function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) {
   // start by retrieving all tables, remove the current table from the list
-  const tables = (await getAllInternalTables()).filter(
+  const tables = (await sdk.tables.getAllInternalTables()).filter(
     tbl => tbl._id !== table._id
   )
   const schemaToUse = oldTable ? oldTable.schema : table.schema
@@ -99,7 +100,7 @@ async function updateRelatedFormulaLinksOnTables(
 ) {
   const db = getAppDB()
   // start by retrieving all tables, remove the current table from the list
-  const tables = (await getAllInternalTables()).filter(
+  const tables = (await sdk.tables.getAllInternalTables()).filter(
     tbl => tbl._id !== table._id
   )
   // clone the tables, so we can compare at end
diff --git a/packages/server/src/api/controllers/table/external.js b/packages/server/src/api/controllers/table/external.js
index d919e9dad7..fe9270fe1d 100644
--- a/packages/server/src/api/controllers/table/external.js
+++ b/packages/server/src/api/controllers/table/external.js
@@ -3,7 +3,6 @@ const {
   breakExternalTableId,
 } = require("../../../integrations/utils")
 const {
-  getTable,
   generateForeignKey,
   generateJunctionTableName,
   foreignKeyStructure,
@@ -20,6 +19,7 @@ const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
+const sdk = require("../../../sdk")
 
 async function makeTableRequest(
   datasource,
@@ -181,7 +181,7 @@ exports.save = async function (ctx) {
 
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await getTable(ctx.request.body._id)
+    oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
 
   if (hasTypeChanged(tableToSave, oldTable)) {
@@ -281,7 +281,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const tableToDelete = await getTable(ctx.params.tableId)
+  const tableToDelete = await sdk.tables.getTable(ctx.params.tableId)
   if (!tableToDelete || !tableToDelete.created) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
@@ -303,7 +303,7 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkImport = async function (ctx) {
-  const table = await getTable(ctx.params.tableId)
+  const table = await sdk.tables.getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
   if (!dataImport || !dataImport.schema || !dataImport.csvString) {
     ctx.throw(400, "Provided data import information is invalid.")
diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js
index e6192457af..3a20f4dff6 100644
--- a/packages/server/src/api/controllers/table/index.js
+++ b/packages/server/src/api/controllers/table/index.js
@@ -4,8 +4,8 @@ const csvParser = require("../../../utilities/csvParser")
 const { isExternalTable, isSQL } = require("../../../integrations/utils")
 const { getDatasourceParams } = require("../../../db/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
-const { getTable, getAllInternalTables } = require("./utils")
 const { events } = require("@budibase/backend-core")
+const sdk = require("../../../sdk")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -23,7 +23,7 @@ function pickApi({ tableId, table }) {
 exports.fetch = async function (ctx) {
   const db = getAppDB()
 
-  const internal = await getAllInternalTables()
+  const internal = await sdk.tables.getAllInternalTables()
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {
@@ -50,7 +50,7 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const tableId = ctx.params.tableId
-  ctx.body = await getTable(tableId)
+  ctx.body = await sdk.tables.getTable(tableId)
 }
 
 exports.save = async function (ctx) {
@@ -101,7 +101,7 @@ exports.validateCSVSchema = async function (ctx) {
   const { csvString, schema = {}, tableId } = ctx.request.body
   let existingTable
   if (tableId) {
-    existingTable = await getTable(tableId)
+    existingTable = await sdk.tables.getTable(tableId)
   }
   let result = await csvParser.parse(csvString, schema)
   if (existingTable) {
diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index 7e55c71aea..a50009b1f6 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -1,12 +1,7 @@
 import { updateLinks, EventType } from "../../../db/linkedRows"
 import { getRowParams, generateTableID } from "../../../db/utils"
 import { FieldTypes } from "../../../constants"
-import {
-  TableSaveFunctions,
-  hasTypeChanged,
-  getTable,
-  handleDataImport,
-} from "./utils"
+import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils"
 const { getAppDB } = require("@budibase/backend-core/context")
 import { isTest } from "../../../environment"
 import {
@@ -19,6 +14,7 @@ import { quotas } from "@budibase/pro"
 import { isEqual } from "lodash"
 import { cloneDeep } from "lodash/fp"
 import env from "../../../environment"
+import sdk from "../../../sdk"
 
 function checkAutoColumns(table: Table, oldTable: Table) {
   if (!table.schema) {
@@ -188,7 +184,7 @@ export async function destroy(ctx: any) {
 }
 
 export async function bulkImport(ctx: any) {
-  const table = await getTable(ctx.params.tableId)
+  const table = await sdk.tables.getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
   await handleDataImport(ctx.user, table, dataImport)
   return table
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index c2aa2f47c9..79c551b63a 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -256,46 +256,6 @@ class TableSaveFunctions {
   }
 }
 
-export async function getAllInternalTables() {
-  const db = getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map((tableDoc: any) => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
-}
-
-export async function getAllExternalTables(datasourceId: any) {
-  const db = getAppDB()
-  const datasource = await db.get(datasourceId)
-  if (!datasource || !datasource.entities) {
-    throw "Datasource is not configured fully."
-  }
-  return datasource.entities
-}
-
-export async function getExternalTable(datasourceId: any, tableName: any) {
-  const entities = await getAllExternalTables(datasourceId)
-  return entities[tableName]
-}
-
-export async function getTable(tableId: any) {
-  const db = getAppDB()
-  if (isExternalTable(tableId)) {
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
-    const table = await getExternalTable(datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
-  } else {
-    return db.get(tableId)
-  }
-}
-
 export async function checkForViewUpdates(
   table: any,
   rename: any,
diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js
index b2c3a84c59..91657cfc21 100644
--- a/packages/server/src/api/controllers/view/index.js
+++ b/packages/server/src/api/controllers/view/index.js
@@ -3,12 +3,12 @@ const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
-const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
 const { DocumentType } = require("../../../db/utils")
 const { cloneDeep, isEqual } = require("lodash")
+const sdk = require("../../../sdk")
 
 exports.fetch = async ctx => {
   ctx.body = await getViews()
@@ -144,7 +144,7 @@ exports.exportView = async ctx => {
 
   let schema = view && view.meta && view.meta.schema
   const tableId = ctx.params.tableId || view.meta.tableId
-  const table = await getTable(tableId)
+  const table = await sdk.tables.getTable(tableId)
   if (!schema) {
     schema = table.schema
   }
diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js
index 0646e453c2..06a79e6ab4 100644
--- a/packages/server/src/automations/automationUtils.js
+++ b/packages/server/src/automations/automationUtils.js
@@ -1,10 +1,10 @@
-const { getTable } = require("../api/controllers/table/utils")
 const {
   findHBSBlocks,
   decodeJSBinding,
   isJSBinding,
   encodeJSBinding,
 } = require("@budibase/string-templates")
+const sdk = require("../sdk")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -64,7 +64,7 @@ exports.cleanInputValues = (inputs, schema) => {
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
 exports.cleanUpRow = async (tableId, row) => {
-  let table = await getTable(tableId)
+  let table = await sdk.tables.getTable(tableId)
   return exports.cleanInputValues(row, { properties: table.schema })
 }
 
diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts
index 150a3b4d4a..b6d896a1ca 100644
--- a/packages/server/src/migrations/functions/backfill/app/tables.ts
+++ b/packages/server/src/migrations/functions/backfill/app/tables.ts
@@ -1,18 +1,11 @@
 import { events } from "@budibase/backend-core"
-import { getTableParams } from "../../../../db/utils"
-import { Table } from "@budibase/types"
+import sdk from "../../../../sdk"
 
-const getTables = async (appDb: any): Promise<Table[]> => {
-  const response = await appDb.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const tables = await getTables(appDb)
+export const backfill = async (
+  appDb: PouchDB.Database,
+  timestamp: string | number
+) => {
+  const tables = await sdk.tables.getAllInternalTables(appDb)
 
   for (const table of tables) {
     await events.table.created(table, timestamp)
diff --git a/packages/server/src/sdk/app/constants.ts b/packages/server/src/sdk/app/backups/constants.ts
similarity index 100%
rename from packages/server/src/sdk/app/constants.ts
rename to packages/server/src/sdk/app/backups/constants.ts
diff --git a/packages/server/src/sdk/app/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
similarity index 94%
rename from packages/server/src/sdk/app/exports.ts
rename to packages/server/src/sdk/app/backups/exports.ts
index 1ecf69de68..adf39f7e15 100644
--- a/packages/server/src/sdk/app/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -1,13 +1,13 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { budibaseTempDir } from "../../utilities/budibaseDir"
-import { retrieveDirectory } from "../../utilities/fileSystem/utilities"
-import { streamFile } from "../../utilities/fileSystem"
-import { ObjectStoreBuckets } from "../../constants"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
+import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
+import { streamFile } from "../../../utilities/fileSystem"
+import { ObjectStoreBuckets } from "../../../constants"
 import {
   LINK_USER_METADATA_PREFIX,
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
-} from "../../db/utils"
+} from "../../../db/utils"
 import {
   DB_EXPORT_FILE,
   GLOBAL_DB_EXPORT_FILE,
diff --git a/packages/server/src/sdk/app/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
similarity index 62%
rename from packages/server/src/sdk/app/imports.ts
rename to packages/server/src/sdk/app/backups/imports.ts
index 96629db39a..0ea7eeaa1b 100644
--- a/packages/server/src/sdk/app/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -1,10 +1,12 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { budibaseTempDir } from "../../utilities/budibaseDir"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants"
-import { uploadDirectory } from "../../utilities/fileSystem/utilities"
-import { ObjectStoreBuckets } from "../../constants"
+import { uploadDirectory } from "../../../utilities/fileSystem/utilities"
+import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
+import sdk from "../../"
+import { CouchFindOptions, Row } from "@budibase/types"
 const uuid = require("uuid/v4")
 const tar = require("tar")
 
@@ -64,8 +66,29 @@ export async function importApp(
   const { ok } = await db.load(dbStream)
   if (!ok) {
     throw "Error loading database dump from template."
-  } else {
-    // TODO: need to iterate over attachments and update their URLs
+  }
+  // iterate through attachment documents and update them
+  const tables = await sdk.tables.getAllInternalTables(db)
+  for (let table of tables) {
+    const attachmentCols: string[] = []
+    for (let [key, column] of Object.entries(table.schema)) {
+      if (column.type === FieldTypes.ATTACHMENT) {
+        attachmentCols.push(key)
+      }
+    }
+    // no attachment columns, nothing to do
+    if (attachmentCols.length === 0) {
+      continue
+    }
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const params: CouchFindOptions = { selector: {} }
+    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+    const { rows } = await dbCore.directCouchFind(db.name, params)
+    for (let row of rows) {
+      // TODO:
+    }
+    // write back the updated attachments
+    await db.bulkDocs(rows)
   }
   return ok
 }
diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts
new file mode 100644
index 0000000000..fe7e4e1049
--- /dev/null
+++ b/packages/server/src/sdk/app/backups/index.ts
@@ -0,0 +1,7 @@
+import * as exportApps from "./exports"
+import * as importApps from "./imports"
+
+export default {
+  ...exportApps,
+  ...importApps,
+}
diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts
deleted file mode 100644
index 5ac7c04b2c..0000000000
--- a/packages/server/src/sdk/app/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export * as exports from "./exports"
-export * as imports from "./imports"
diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts
new file mode 100644
index 0000000000..eeaf869055
--- /dev/null
+++ b/packages/server/src/sdk/app/tables/index.ts
@@ -0,0 +1,60 @@
+import { getAppDB } from "@budibase/backend-core/context"
+import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  breakExternalTableId,
+  isExternalTable,
+  isSQL,
+} from "../../../integrations/utils"
+import { Table } from "@budibase/types"
+
+async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
+  if (!db) {
+    db = getAppDB() as PouchDB.Database
+  }
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+  const db = getAppDB()
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return datasource.entities
+}
+
+async function getExternalTable(
+  datasourceId: any,
+  tableName: any
+): Promise<Table> {
+  const entities = await getAllExternalTables(datasourceId)
+  return entities[tableName]
+}
+
+async function getTable(tableId: any): Promise<Table> {
+  const db = getAppDB()
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    const datasource = await db.get(datasourceId)
+    const table = await getExternalTable(datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
+  } else {
+    return db.get(tableId)
+  }
+}
+
+export default {
+  getAllInternalTables,
+  getAllExternalTables,
+  getExternalTable,
+  getTable,
+}
diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts
index 9199c1cc72..8cf9d21c41 100644
--- a/packages/server/src/sdk/index.ts
+++ b/packages/server/src/sdk/index.ts
@@ -1,5 +1,7 @@
-import * as apps from "./app"
+import { default as backups } from "./app/backups"
+import { default as tables } from "./app/tables"
 
 export default {
-  apps,
+  backups,
+  tables,
 }
diff --git a/packages/types/package.json b/packages/types/package.json
index 93271fb93e..0787093e46 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -16,6 +16,7 @@
     "@types/koa": "2.13.4",
     "@types/node": "14.18.20",
     "rimraf": "3.0.2",
-    "typescript": "4.7.3"
+    "typescript": "4.7.3",
+    "@types/pouchdb": "6.4.0"
   }
 }
diff --git a/packages/types/src/core/db.ts b/packages/types/src/core/db.ts
new file mode 100644
index 0000000000..6d47a5c36b
--- /dev/null
+++ b/packages/types/src/core/db.ts
@@ -0,0 +1,22 @@
+export type PouchOptions = {
+  inMemory: boolean
+  replication: boolean
+  onDisk: boolean
+  find: boolean
+}
+
+export enum SortOption {
+  ASCENDING = "asc",
+  DESCENDING = "desc",
+}
+
+export type CouchFindOptions = {
+  selector: PouchDB.Find.Selector
+  fields?: string[]
+  sort?: {
+    [key: string]: SortOption
+  }[]
+  limit?: number
+  skip?: number
+  bookmark?: string
+}
diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts
new file mode 100644
index 0000000000..9071393365
--- /dev/null
+++ b/packages/types/src/core/index.ts
@@ -0,0 +1 @@
+export * from "./db"
diff --git a/packages/types/src/documents/app/table.ts b/packages/types/src/documents/app/table.ts
index 72cff4f056..b98c562852 100644
--- a/packages/types/src/documents/app/table.ts
+++ b/packages/types/src/documents/app/table.ts
@@ -49,4 +49,5 @@ export interface Table extends Document {
   sourceId?: string
   relatedFormula?: string[]
   constrained?: string[]
+  sql?: boolean
 }
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index 4adb2fda97..92d2ceb050 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -1,3 +1,4 @@
 export * from "./documents"
 export * from "./sdk"
 export * from "./api"
+export * from "./core"
diff --git a/packages/types/yarn.lock b/packages/types/yarn.lock
index c80ff652ba..f225ffc442 100644
--- a/packages/types/yarn.lock
+++ b/packages/types/yarn.lock
@@ -39,6 +39,13 @@
     "@types/keygrip" "*"
     "@types/node" "*"
 
+"@types/debug@*":
+  version "4.1.7"
+  resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82"
+  integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==
+  dependencies:
+    "@types/ms" "*"
+
 "@types/express-serve-static-core@^4.17.18":
   version "4.17.29"
   resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.29.tgz#2a1795ea8e9e9c91b4a4bbe475034b20c1ec711c"
@@ -113,6 +120,11 @@
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
   integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==
 
+"@types/ms@*":
+  version "0.7.31"
+  resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
+  integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
+
 "@types/node@*":
   version "18.0.6"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.6.tgz#0ba49ac517ad69abe7a1508bc9b3a5483df9d5d7"
@@ -123,6 +135,152 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
   integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
 
+"@types/pouchdb-adapter-cordova-sqlite@*":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz#49e5ee6df7cc0c23196fcb340f43a560e74eb1d6"
+  integrity sha512-nqlXpW1ho3KBg1mUQvZgH2755y3z/rw4UA7ZJCPMRTHofxGMY8izRVw5rHBL4/7P615or0J2udpRYxgkT3D02g==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-fruitdown@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-fruitdown/-/pouchdb-adapter-fruitdown-6.1.3.tgz#9b140ad9645cc56068728acf08ec19ac0046658e"
+  integrity sha512-Wz1Z1JLOW1hgmFQjqnSkmyyfH7by/iWb4abKn684WMvQfmxx6BxKJpJ4+eulkVPQzzgMMSgU1MpnQOm9FgRkbw==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-http@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-http/-/pouchdb-adapter-http-6.1.3.tgz#6e592d5f48deb6274a21ddac1498dd308096bcf3"
+  integrity sha512-9Z4TLbF/KJWy/D2sWRPBA+RNU0odQimfdvlDX+EY7rGcd3aVoH8qjD/X0Xcd/0dfBH5pKrNIMFFQgW/TylRCmA==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-idb@*":
+  version "6.1.4"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-idb/-/pouchdb-adapter-idb-6.1.4.tgz#cb9a18864585d600820cd325f007614c5c3989cd"
+  integrity sha512-KIAXbkF4uYUz0ZwfNEFLtEkK44mEWopAsD76UhucH92XnJloBysav+TjI4FFfYQyTjoW3S1s6V+Z14CUJZ0F6w==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-leveldb@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-leveldb/-/pouchdb-adapter-leveldb-6.1.3.tgz#17c7e75d75b992050bca15991e97fba575c61bb3"
+  integrity sha512-ex8NFqQGFwEpFi7AaZ5YofmuemfZNsL3nTFZBUCAKYMBkazQij1pe2ILLStSvJr0XS0qxgXjCEW19T5Wqiiskg==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-localstorage@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-localstorage/-/pouchdb-adapter-localstorage-6.1.3.tgz#0dde02ba6b9d6073a295a20196563942ba9a54bd"
+  integrity sha512-oor040tye1KKiGLWYtIy7rRT7C2yoyX3Tf6elEJRpjOA7Ja/H8lKc4LaSh9ATbptIcES6MRqZDxtp7ly9hsW3Q==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-memory@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-memory/-/pouchdb-adapter-memory-6.1.3.tgz#9eabdbc890fcf58960ee8b68b8685f837e75c844"
+  integrity sha512-gVbsIMzDzgZYThFVT4eVNsmuZwVm/4jDxP1sjlgc3qtDIxbtBhGgyNfcskwwz9Zu5Lv1avkDsIWvcxQhnvRlHg==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-node-websql@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-node-websql/-/pouchdb-adapter-node-websql-6.1.3.tgz#aa18bc68af8cf509acd12c400010dcd5fab2243d"
+  integrity sha512-F/P+os6Jsa7CgHtH64+Z0HfwIcj0hIRB5z8gNhF7L7dxPWoAfkopK5H2gydrP3sQrlGyN4WInF+UJW/Zu1+FKg==
+  dependencies:
+    "@types/pouchdb-adapter-websql" "*"
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-adapter-websql@*":
+  version "6.1.4"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-websql/-/pouchdb-adapter-websql-6.1.4.tgz#359fbe42ccac0ac90b492ddb8c32fafd0aa96d79"
+  integrity sha512-zMJQCtXC40hBsIDRn0GhmpeGMK0f9l/OGWfLguvczROzxxcOD7REI+e6SEmX7gJKw5JuMvlfuHzkQwjmvSJbtg==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-browser@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-browser/-/pouchdb-browser-6.1.3.tgz#8f33d6ef58d6817d1f6d36979148a1c7f63244d8"
+  integrity sha512-EdYowrWxW9SWBMX/rux2eq7dbHi5Zeyzz+FF/IAsgQKnUxgeCO5VO2j4zTzos0SDyJvAQU+EYRc11r7xGn5tvA==
+  dependencies:
+    "@types/pouchdb-adapter-http" "*"
+    "@types/pouchdb-adapter-idb" "*"
+    "@types/pouchdb-adapter-websql" "*"
+    "@types/pouchdb-core" "*"
+    "@types/pouchdb-mapreduce" "*"
+    "@types/pouchdb-replication" "*"
+
+"@types/pouchdb-core@*":
+  version "7.0.10"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-core/-/pouchdb-core-7.0.10.tgz#d1ea1549e7fad6cb579f71459b1bc27252e06a5a"
+  integrity sha512-mKhjLlWWXyV3PTTjDhzDV1kc2dolO7VYFa75IoKM/hr8Er9eo8RIbS7mJLfC8r/C3p6ihZu9yZs1PWC1LQ0SOA==
+  dependencies:
+    "@types/debug" "*"
+    "@types/pouchdb-find" "*"
+
+"@types/pouchdb-find@*":
+  version "7.3.0"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-find/-/pouchdb-find-7.3.0.tgz#b917030e9f4bf6e56bf8c3b9fe4b2a25e989009a"
+  integrity sha512-sFPli5tBjGX9UfXioik1jUzPdcN84eV82n0lmEFuoPepWqkLjQcyri0eOa++HYOaNPyMDhKFBqEALEZivK2dRg==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-http@*":
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-http/-/pouchdb-http-6.1.3.tgz#09576c0d409da1f8dee34ec5b768415e2472ea52"
+  integrity sha512-0e9E5SqNOyPl/3FnEIbENssB4FlJsNYuOy131nxrZk36S+y1R/6qO7ZVRypWpGTqBWSuVd7gCsq2UDwO/285+w==
+  dependencies:
+    "@types/pouchdb-adapter-http" "*"
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-mapreduce@*":
+  version "6.1.7"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-mapreduce/-/pouchdb-mapreduce-6.1.7.tgz#9ab32d1e0f234f1bf6d1e4c5d7e216e9e23ac0a3"
+  integrity sha512-WzBwm7tmO9QhfRzVaWT4v6JQSS/fG2OoUDrWrhX87rPe2Pn6laPvdK5li6myNRxCoI/l5e8Jd+oYBAFnaiFucA==
+  dependencies:
+    "@types/pouchdb-core" "*"
+
+"@types/pouchdb-node@*":
+  version "6.1.4"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-node/-/pouchdb-node-6.1.4.tgz#5214c0169fcfd2237d373380bbd65a934feb5dfb"
+  integrity sha512-wnTCH8X1JOPpNOfVhz8HW0AvmdHh6pt40MuRj0jQnK7QEHsHS79WujsKTKSOF8QXtPwpvCNSsI7ut7H7tfxxJQ==
+  dependencies:
+    "@types/pouchdb-adapter-http" "*"
+    "@types/pouchdb-adapter-leveldb" "*"
+    "@types/pouchdb-core" "*"
+    "@types/pouchdb-mapreduce" "*"
+    "@types/pouchdb-replication" "*"
+
+"@types/pouchdb-replication@*":
+  version "6.4.4"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb-replication/-/pouchdb-replication-6.4.4.tgz#743406c90f13a988fa3e346ea74ce40acd170d00"
+  integrity sha512-BsE5LKpjJK4iAf6Fx5kyrMw+33V+Ip7uWldUnU2BYrrvtR+MLD22dcImm7DZN1st2wPPb91i0XEnQzvP0w1C/Q==
+  dependencies:
+    "@types/pouchdb-core" "*"
+    "@types/pouchdb-find" "*"
+
+"@types/pouchdb@6.4.0":
+  version "6.4.0"
+  resolved "https://registry.yarnpkg.com/@types/pouchdb/-/pouchdb-6.4.0.tgz#f9c41ca64b23029f9bf2eb4bf6956e6431cb79f8"
+  integrity sha512-eGCpX+NXhd5VLJuJMzwe3L79fa9+IDTrAG3CPaf4s/31PD56hOrhDJTSmRELSXuiqXr6+OHzzP0PldSaWsFt7w==
+  dependencies:
+    "@types/pouchdb-adapter-cordova-sqlite" "*"
+    "@types/pouchdb-adapter-fruitdown" "*"
+    "@types/pouchdb-adapter-http" "*"
+    "@types/pouchdb-adapter-idb" "*"
+    "@types/pouchdb-adapter-leveldb" "*"
+    "@types/pouchdb-adapter-localstorage" "*"
+    "@types/pouchdb-adapter-memory" "*"
+    "@types/pouchdb-adapter-node-websql" "*"
+    "@types/pouchdb-adapter-websql" "*"
+    "@types/pouchdb-browser" "*"
+    "@types/pouchdb-core" "*"
+    "@types/pouchdb-http" "*"
+    "@types/pouchdb-mapreduce" "*"
+    "@types/pouchdb-node" "*"
+    "@types/pouchdb-replication" "*"
+
 "@types/qs@*":
   version "6.9.7"
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"

From f5dd87f8f9d6c6077ec35000661a7a57f87bb3ed Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 12 Oct 2022 17:34:17 +0100
Subject: [PATCH 11/46] Adding first pass of attachments updating.

---
 packages/backend-core/src/db/index.ts         |  4 +-
 .../server/src/sdk/app/backups/constants.ts   |  2 +-
 .../server/src/sdk/app/backups/imports.ts     | 84 +++++++++++++------
 packages/types/src/documents/app/row.ts       |  8 ++
 4 files changed, 71 insertions(+), 27 deletions(-)

diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts
index 016155033a..2728b83682 100644
--- a/packages/backend-core/src/db/index.ts
+++ b/packages/backend-core/src/db/index.ts
@@ -32,7 +32,7 @@ const checkInitialised = () => {
   }
 }
 
-export async function init(opts: PouchOptions) {
+export async function init(opts?: PouchOptions) {
   PouchDB = pouch.getPouch(opts)
   initialised = true
 }
@@ -40,7 +40,7 @@ export async function init(opts: PouchOptions) {
 // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
 // this function is prone to leaks, should only be used
 // in situations that using the function doWithDB does not work
-export async function dangerousGetDB(dbName: string, opts: any) {
+export async function dangerousGetDB(dbName: string, opts?: any) {
   checkInitialised()
   if (env.isTest()) {
     dbList.add(dbName)
diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts
index e831172683..f022168846 100644
--- a/packages/server/src/sdk/app/backups/constants.ts
+++ b/packages/server/src/sdk/app/backups/constants.ts
@@ -1,4 +1,4 @@
-import { ATTACHMENT_DIR as attachmentDir } from "../../constants"
+import { ATTACHMENT_DIR as attachmentDir } from "../../../constants"
 export const DB_EXPORT_FILE = "db.txt"
 export const ATTACHMENT_DIR = attachmentDir
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index 0ea7eeaa1b..297274e26b 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -1,4 +1,5 @@
 import { db as dbCore } from "@budibase/backend-core"
+import { TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants"
 import { uploadDirectory } from "../../../utilities/fileSystem/utilities"
@@ -6,7 +7,7 @@ import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-import { CouchFindOptions, Row } from "@budibase/types"
+import { CouchFindOptions, RowAttachment } from "@budibase/types"
 const uuid = require("uuid/v4")
 const tar = require("tar")
 
@@ -18,6 +19,63 @@ type TemplateType = {
   key?: string
 }
 
+async function updateAttachmentColumns(
+  prodAppId: string,
+  db: PouchDB.Database
+) {
+  // iterate through attachment documents and update them
+  const tables = await sdk.tables.getAllInternalTables(db)
+  for (let table of tables) {
+    const attachmentCols: string[] = []
+    for (let [key, column] of Object.entries(table.schema)) {
+      if (column.type === FieldTypes.ATTACHMENT) {
+        attachmentCols.push(key)
+      }
+    }
+    // no attachment columns, nothing to do
+    if (attachmentCols.length === 0) {
+      continue
+    }
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const params: CouchFindOptions = {
+      selector: {
+        _id: {
+          $regex: `^${TABLE_ROW_PREFIX}`,
+        },
+      },
+    }
+    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+    const { rows } = await dbCore.directCouchFind(db.name, params)
+    for (let row of rows) {
+      for (let column of attachmentCols) {
+        if (!Array.isArray(row[column])) {
+          continue
+        }
+        row[column] = row[column].map((attachment: RowAttachment) => {
+          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+          const urlParts = attachment.url.split("/")
+          // drop the first empty element
+          urlParts.shift()
+          // get the prefix
+          const prefix = urlParts.shift()
+          // remove the app ID
+          urlParts.shift()
+          // add new app ID
+          urlParts.unshift(prodAppId)
+          const key = urlParts.join("/")
+          return {
+            ...attachment,
+            key,
+            url: `/${prefix}/${key}`,
+          }
+        })
+      }
+    }
+    // write back the updated attachments
+    await db.bulkDocs(rows)
+  }
+}
+
 /**
  * This function manages temporary template files which are stored by Koa.
  * @param {Object} template The template object retrieved from the Koa context object.
@@ -67,28 +125,6 @@ export async function importApp(
   if (!ok) {
     throw "Error loading database dump from template."
   }
-  // iterate through attachment documents and update them
-  const tables = await sdk.tables.getAllInternalTables(db)
-  for (let table of tables) {
-    const attachmentCols: string[] = []
-    for (let [key, column] of Object.entries(table.schema)) {
-      if (column.type === FieldTypes.ATTACHMENT) {
-        attachmentCols.push(key)
-      }
-    }
-    // no attachment columns, nothing to do
-    if (attachmentCols.length === 0) {
-      continue
-    }
-    // use the CouchDB Mango query API to lookup rows that have attachments
-    const params: CouchFindOptions = { selector: {} }
-    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
-    const { rows } = await dbCore.directCouchFind(db.name, params)
-    for (let row of rows) {
-      // TODO:
-    }
-    // write back the updated attachments
-    await db.bulkDocs(rows)
-  }
+  await updateAttachmentColumns(prodAppId, db)
   return ok
 }
diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index ee5c0231e7..2cac32279b 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -16,6 +16,14 @@ export enum FieldType {
   INTERNAL = "internal",
 }
 
+export interface RowAttachment {
+  size: number
+  name: string
+  url: string
+  extension: string
+  key: string
+}
+
 export interface Row extends Document {
   type?: string
   tableId?: string

From 4da37058511480aa92d8e81df59bb7fc53af9f46 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 12 Oct 2022 17:37:52 +0100
Subject: [PATCH 12/46] Quick fixes to DB TS conversion.

---
 packages/backend-core/src/db/index.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts
index 2728b83682..7a2337359a 100644
--- a/packages/backend-core/src/db/index.ts
+++ b/packages/backend-core/src/db/index.ts
@@ -40,7 +40,7 @@ export async function init(opts?: PouchOptions) {
 // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
 // this function is prone to leaks, should only be used
 // in situations that using the function doWithDB does not work
-export async function dangerousGetDB(dbName: string, opts?: any) {
+export function dangerousGetDB(dbName: string, opts?: any) {
   checkInitialised()
   if (env.isTest()) {
     dbList.add(dbName)
@@ -75,13 +75,13 @@ export async function closeDB(db: PouchDB.Database) {
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
 export async function doWithDB(dbName: string, cb: any, opts = {}) {
-  const db = exports.dangerousGetDB(dbName, opts)
+  const db = dangerousGetDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
   // async operations have been completed
   try {
     return await cb(db)
   } finally {
-    await exports.closeDB(db)
+    await closeDB(db)
   }
 }
 

From 2eae3f2a6c954eb49053b60615633f6ec7dfac9c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 12 Oct 2022 17:57:31 +0100
Subject: [PATCH 13/46] Fixes for find functionality after testing.

---
 packages/backend-core/src/db/index.ts | 11 ++++++++++-
 packages/backend-core/src/db/utils.ts | 17 ++++++++---------
 packages/server/src/sdk/index.ts      |  8 +++++++-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts
index 7a2337359a..8374aecd8d 100644
--- a/packages/backend-core/src/db/index.ts
+++ b/packages/backend-core/src/db/index.ts
@@ -107,7 +107,8 @@ export async function directCouchQuery(
     },
   }
   if (body && method !== "GET") {
-    params.body = body
+    params.body = JSON.stringify(body)
+    params.headers["Content-Type"] = "application/json"
   }
   const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
   if (response.status < 300) {
@@ -117,6 +118,14 @@ export async function directCouchQuery(
   }
 }
 
+export async function directCouchAllDbs(queryString?: string) {
+  let couchPath = "/_all_dbs"
+  if (queryString) {
+    couchPath += `?${queryString}`
+  }
+  return await directCouchQuery(couchPath)
+}
+
 export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
   const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
   return { rows: json.docs, bookmark: json.bookmark }
diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts
index c78c11217e..8d824d60bb 100644
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@@ -4,7 +4,7 @@ import env from "../environment"
 import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants"
 import { getTenantId, getGlobalDB } from "../context"
 import { getGlobalDBName } from "./tenancy"
-import { doWithDB, allDbs, directCouchQuery } from "./index"
+import { doWithDB, allDbs, directCouchQuery, directCouchAllDbs } from "./index"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
@@ -188,9 +188,9 @@ export function getRoleParams(roleId = null, otherProps = {}) {
   return getDocParams(DocumentType.ROLE, roleId, otherProps)
 }
 
-export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) {
+export function getStartEndKeyURL(baseKey: any, tenantId = null) {
   const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
-  return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
+  return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
 }
 
 /**
@@ -206,11 +206,10 @@ export async function getAllDbs(opts = { efficient: false }) {
     return allDbs()
   }
   let dbs: any[] = []
-  async function addDbs(couchPath: string) {
-    const json = await directCouchQuery(couchPath)
+  async function addDbs(queryString?: string) {
+    const json = await directCouchAllDbs(queryString)
     dbs = dbs.concat(json)
   }
-  let couchPath = "/_all_dbs"
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
     // just get all DBs when:
@@ -218,12 +217,12 @@ export async function getAllDbs(opts = { efficient: false }) {
     // - default tenant
     //    - apps dbs don't contain tenant id
     //    - non-default tenant dbs are filtered out application side in getAllApps
-    await addDbs(couchPath)
+    await addDbs()
   } else {
     // get prod apps
-    await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP, tenantId))
+    await addDbs(getStartEndKeyURL(DocumentType.APP, tenantId))
     // get dev apps
-    await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP_DEV, tenantId))
+    await addDbs(getStartEndKeyURL(DocumentType.APP_DEV, tenantId))
     // add global db name
     dbs.push(getGlobalDBName(tenantId))
   }
diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts
index 8cf9d21c41..85c01cdb44 100644
--- a/packages/server/src/sdk/index.ts
+++ b/packages/server/src/sdk/index.ts
@@ -1,7 +1,13 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 
-export default {
+const toExport = {
   backups,
   tables,
 }
+
+// default export for TS
+export default toExport
+
+// default export for JS
+module.exports = toExport

From d1c9a56e9a01cad3bae1c139a3cb29bb409ef2b2 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 12 Oct 2022 19:15:28 +0100
Subject: [PATCH 14/46] Full import implementation - needs further testing,
 untars the file with all apps, then adds each of them individually.

---
 .../pages/builder/portal/apps/index.svelte    |   3 +-
 packages/server/src/api/controllers/cloud.js  | 123 +++++++++++-------
 .../server/src/sdk/app/backups/exports.ts     |  17 ++-
 .../server/src/sdk/app/backups/imports.ts     |  36 +++--
 4 files changed, 112 insertions(+), 67 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte
index a2fa954389..0b760c4b4a 100644
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@@ -21,7 +21,6 @@
   import { API } from "api"
   import { onMount } from "svelte"
   import { apps, auth, admin, templates, licensing } from "stores/portal"
-  import download from "downloadjs"
   import { goto } from "@roxi/routify"
   import AppRow from "components/start/AppRow.svelte"
   import { AppStatus } from "constants"
@@ -140,7 +139,7 @@
 
   const initiateAppsExport = () => {
     try {
-      download(`/api/cloud/export`)
+      window.location = `/api/cloud/export`
       notifications.success("Apps exported successfully")
     } catch (err) {
       notifications.error(`Error exporting apps: ${err}`)
diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index 323c7409db..9397bc69a6 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -1,9 +1,45 @@
 const env = require("../../environment")
 const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { streamFile } = require("../../utilities/fileSystem")
-const { DocumentType, isDevAppID } = require("../../db/utils")
+const { stringToReadStream } = require("../../utilities")
+const {
+  getDocParams,
+  DocumentType,
+  isDevAppID,
+  APP_PREFIX,
+} = require("../../db/utils")
+const { create } = require("./application")
+const { join } = require("path")
+const fs = require("fs")
 const sdk = require("../../sdk")
 
+async function createApp(appName, appDirectory) {
+  const ctx = {
+    request: {
+      body: {
+        useTemplate: true,
+        name: appName,
+      },
+      files: {
+        templateFile: {
+          path: appDirectory,
+        },
+      },
+    },
+  }
+  return create(ctx)
+}
+
+async function getAllDocType(db, docType) {
+  const response = await db.allDocs(
+    getDocParams(docType, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc)
+}
+
 exports.exportApps = async ctx => {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
@@ -14,10 +50,13 @@ exports.exportApps = async ctx => {
   })
   // only export the dev apps as they will be the latest, the user can republish the apps
   // in their self-hosted environment
-  let appIds = apps
-    .map(app => app.appId || app._id)
-    .filter(appId => isDevAppID(appId))
-  const tmpPath = await sdk.backups.exportMultipleApps(appIds, globalDBString)
+  let appMetadata = apps
+    .filter(app => isDevAppID(app.appId || app._id))
+    .map(app => ({ appId: app.appId || app._id, name: app.name }))
+  const tmpPath = await sdk.backups.exportMultipleApps(
+    appMetadata,
+    globalDBString
+  )
   const filename = `cloud-export-${new Date().getTime()}.tar.gz`
   ctx.attachment(filename)
   ctx.body = streamFile(tmpPath)
@@ -48,51 +87,37 @@ exports.importApps = async ctx => {
       "Import file is required and environment must be fresh to import apps."
     )
   }
+  if (ctx.request.files.importFile.type !== "application/gzip") {
+    ctx.throw(400, "Import file must be a gzipped tarball.")
+  }
 
-  // TODO: IMPLEMENT TARBALL EXTRACTION, APP IMPORT, ATTACHMENT IMPORT AND GLOBAL DB IMPORT
-  // async function getAllDocType(db, docType) {
-  //   const response = await db.allDocs(
-  //     getDocParams(docType, null, {
-  //       include_docs: true,
-  //     })
-  //   )
-  //   return response.rows.map(row => row.doc)
-  // }
-  // async function createApp(appName, appImport) {
-  //   const ctx = {
-  //     request: {
-  //       body: {
-  //         templateString: appImport,
-  //         name: appName,
-  //       },
-  //     },
-  //   }
-  //   return create(ctx)
-  // }
-  // const importFile = ctx.request.files.importFile
-  // const importString = readFileSync(importFile.path)
-  // const dbs = JSON.parse(importString)
-  // const globalDbImport = dbs.global
-  // // remove from the list of apps
-  // delete dbs.global
-  // const globalDb = getGlobalDB()
-  // // load the global db first
-  // await globalDb.load(stringToReadStream(globalDbImport))
-  // for (let [appName, appImport] of Object.entries(dbs)) {
-  //   await createApp(appName, appImport)
-  // }
-  //
-  // // if there are any users make sure to remove them
-  // let users = await getAllDocType(globalDb, DocumentType.USER)
-  // let userDeletionPromises = []
-  // for (let user of users) {
-  //   userDeletionPromises.push(globalDb.remove(user._id, user._rev))
-  // }
-  // if (userDeletionPromises.length > 0) {
-  //   await Promise.all(userDeletionPromises)
-  // }
-  //
-  // await globalDb.bulkDocs(users)
+  // initially get all the app databases out of the tarball
+  const tmpPath = sdk.backups.untarFile(ctx.request.file.importFile)
+  const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath)
+  const appNames = fs
+    .readdirSync(tmpPath)
+    .filter(dir => dir.startsWith(APP_PREFIX))
+
+  const globalDb = getGlobalDB()
+  // load the global db first
+  await globalDb.load(stringToReadStream(globalDbImport))
+  const appCreationPromises = []
+  for (let appName of appNames) {
+    appCreationPromises.push(createApp(appName, join(tmpPath, appName)))
+  }
+  await Promise.all(appCreationPromises)
+
+  // if there are any users make sure to remove them
+  let users = await getAllDocType(globalDb, DocumentType.USER)
+  let userDeletionPromises = []
+  for (let user of users) {
+    userDeletionPromises.push(globalDb.remove(user._id, user._rev))
+  }
+  if (userDeletionPromises.length > 0) {
+    await Promise.all(userDeletionPromises)
+  }
+
+  await globalDb.bulkDocs(users)
   ctx.body = {
     message: "Apps successfully imported.",
   }
diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index adf39f7e15..5a028c27a9 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -127,28 +127,33 @@ export async function exportApp(appId: string, config?: ExportOpts) {
 /**
  * Export all apps + global DB (if supplied) to a single tarball, this includes
  * the attachments for each app as well.
- * @param {string[]} appIds The IDs of the apps to be exported.
+ * @param {object[]} appMetadata The IDs and names of apps to export.
  * @param {string} globalDbContents The contents of the global DB to export as well.
  * @return {string} The path to the tarball.
  */
 export async function exportMultipleApps(
-  appIds: string[],
+  appMetadata: { appId: string; name: string }[],
   globalDbContents?: string
 ) {
   const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
   let exportPromises: Promise<void>[] = []
-  const exportAndMove = async (appId: string) => {
+  // export each app to a directory, then move it into the complete export
+  const exportAndMove = async (appId: string, appName: string) => {
     const path = await exportApp(appId)
     await fs.promises.rename(path, join(tmpPath, appId))
   }
-  for (let appId of appIds) {
-    exportPromises.push(exportAndMove(appId))
+  for (let metadata of appMetadata) {
+    exportPromises.push(exportAndMove(metadata.appId, metadata.name))
   }
+  // wait for all exports to finish
   await Promise.all(exportPromises)
+  // add the global DB contents
   if (globalDbContents) {
     fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
   }
-  const tarPath = tarFilesToTmp(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE])
+  const appNames = appMetadata.map(metadata => metadata.name)
+  const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE])
   // clear up the tmp path now tarball generated
   fs.rmSync(tmpPath, { recursive: true, force: true })
   return tarPath
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index 297274e26b..13d8e7aab0 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -1,7 +1,11 @@
 import { db as dbCore } from "@budibase/backend-core"
 import { TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  ATTACHMENT_DIR,
+  GLOBAL_DB_EXPORT_FILE,
+} from "./constants"
 import { uploadDirectory } from "../../../utilities/fileSystem/utilities"
 import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
 import { join } from "path"
@@ -91,6 +95,22 @@ async function getTemplateStream(template: TemplateType) {
   }
 }
 
+export function untarFile(file: { path: string }) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
+  // extract the tarball
+  tar.extract({
+    sync: true,
+    cwd: tmpPath,
+    file: file.path,
+  })
+  return tmpPath
+}
+
+export function getGlobalDBFile(tmpPath: string) {
+  return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8")
+}
+
 export async function importApp(
   appId: string,
   db: PouchDB.Database,
@@ -98,15 +118,11 @@ export async function importApp(
 ) {
   let prodAppId = dbCore.getProdAppID(appId)
   let dbStream: any
-  if (template.file && template.file.type === "application/gzip") {
-    const tmpPath = join(budibaseTempDir(), uuid())
-    fs.mkdirSync(tmpPath)
-    // extract the tarball
-    tar.extract({
-      sync: true,
-      cwd: tmpPath,
-      file: template.file.path,
-    })
+  const isTar = template.file && template.file.type === "application/gzip"
+  const isDirectory =
+    template.file && fs.lstatSync(template.file.path).isDirectory()
+  if (template.file && (isTar || isDirectory)) {
+    const tmpPath = isTar ? untarFile(template.file) : template.file.path
     const attachmentPath = join(tmpPath, ATTACHMENT_DIR)
     // have to handle object import
     if (fs.existsSync(attachmentPath)) {

From 907b838db366b3079801372e57f07d5a40a81e43 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 13 Oct 2022 15:46:53 +0100
Subject: [PATCH 15/46] Fixing multi-app import from cloud.

---
 packages/server/src/api/controllers/cloud.js  | 24 ++++++-------------
 .../server/src/sdk/app/backups/exports.ts     |  2 +-
 .../server/src/sdk/app/backups/imports.ts     |  6 ++++-
 3 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index 9397bc69a6..27b576f181 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -3,15 +3,9 @@ const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db")
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { streamFile } = require("../../utilities/fileSystem")
 const { stringToReadStream } = require("../../utilities")
-const {
-  getDocParams,
-  DocumentType,
-  isDevAppID,
-  APP_PREFIX,
-} = require("../../db/utils")
+const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils")
 const { create } = require("./application")
 const { join } = require("path")
-const fs = require("fs")
 const sdk = require("../../sdk")
 
 async function createApp(appName, appDirectory) {
@@ -41,9 +35,9 @@ async function getAllDocType(db, docType) {
 }
 
 exports.exportApps = async ctx => {
-  if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
-    ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
-  }
+  // if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
+  //   ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
+  // }
   const apps = await getAllApps({ all: true })
   const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentType.USER),
@@ -92,20 +86,16 @@ exports.importApps = async ctx => {
   }
 
   // initially get all the app databases out of the tarball
-  const tmpPath = sdk.backups.untarFile(ctx.request.file.importFile)
+  const tmpPath = sdk.backups.untarFile(ctx.request.files.importFile)
   const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath)
-  const appNames = fs
-    .readdirSync(tmpPath)
-    .filter(dir => dir.startsWith(APP_PREFIX))
+  const appNames = sdk.backups.getListOfAppsInMulti(tmpPath)
 
   const globalDb = getGlobalDB()
   // load the global db first
   await globalDb.load(stringToReadStream(globalDbImport))
-  const appCreationPromises = []
   for (let appName of appNames) {
-    appCreationPromises.push(createApp(appName, join(tmpPath, appName)))
+    await createApp(appName, join(tmpPath, appName))
   }
-  await Promise.all(appCreationPromises)
 
   // if there are any users make sure to remove them
   let users = await getAllDocType(globalDb, DocumentType.USER)
diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index 5a028c27a9..4656a83d51 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -141,7 +141,7 @@ export async function exportMultipleApps(
   // export each app to a directory, then move it into the complete export
   const exportAndMove = async (appId: string, appName: string) => {
     const path = await exportApp(appId)
-    await fs.promises.rename(path, join(tmpPath, appId))
+    await fs.promises.rename(path, join(tmpPath, appName))
   }
   for (let metadata of appMetadata) {
     exportPromises.push(exportAndMove(metadata.appId, metadata.name))
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index 13d8e7aab0..d09c4b3f02 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -1,5 +1,5 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { APP_PREFIX, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import {
   DB_EXPORT_FILE,
@@ -111,6 +111,10 @@ export function getGlobalDBFile(tmpPath: string) {
   return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8")
 }
 
+export function getListOfAppsInMulti(tmpPath: string) {
+  return fs.readdirSync(tmpPath).filter(dir => dir !== GLOBAL_DB_EXPORT_FILE)
+}
+
 export async function importApp(
   appId: string,
   db: PouchDB.Database,

From 35525bfedd9b03db51e22e29567c2817f0ae1e13 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 13 Oct 2022 17:27:04 +0100
Subject: [PATCH 16/46] Moving queue implementation into backend-core - so that
 pro can access.

---
 packages/backend-core/package.json            |   1 +
 packages/backend-core/src/index.ts            |   2 +
 packages/backend-core/src/queue/constants.ts  |   4 +
 .../backend-core/src/queue/inMemoryQueue.ts   | 127 +++++++++++++++++
 packages/backend-core/src/queue/index.ts      |   2 +
 packages/backend-core/src/queue/queue.ts      |  47 +++++++
 packages/backend-core/yarn.lock               | 129 +++++++++++++++++-
 packages/server/src/automations/bullboard.js  |  33 +----
 packages/server/src/constants/index.js        |   4 -
 9 files changed, 314 insertions(+), 35 deletions(-)
 create mode 100644 packages/backend-core/src/queue/constants.ts
 create mode 100644 packages/backend-core/src/queue/inMemoryQueue.ts
 create mode 100644 packages/backend-core/src/queue/index.ts
 create mode 100644 packages/backend-core/src/queue/queue.ts

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index abcf72491a..404b7da346 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -26,6 +26,7 @@
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",
     "bcryptjs": "2.4.3",
+    "bull": "^4.10.1",
     "dotenv": "16.0.1",
     "emitter-listener": "1.1.2",
     "ioredis": "4.28.0",
diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index 42cad17620..659a56c051 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -19,6 +19,7 @@ import pino from "./pino"
 import * as middleware from "./middleware"
 import plugins from "./plugin"
 import encryption from "./security/encryption"
+import * as queue from "./queue"
 
 // mimic the outer package exports
 import * as db from "./pkg/db"
@@ -63,6 +64,7 @@ const core = {
   ...errorClasses,
   middleware,
   encryption,
+  queue,
 }
 
 export = core
diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts
new file mode 100644
index 0000000000..d8fb3121a3
--- /dev/null
+++ b/packages/backend-core/src/queue/constants.ts
@@ -0,0 +1,4 @@
+export enum JobQueue {
+  AUTOMATIONS = "automationQueue",
+  APP_BACKUPS = "appBackupQueue",
+}
diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts
new file mode 100644
index 0000000000..80ee7362e4
--- /dev/null
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@@ -0,0 +1,127 @@
+import events from "events"
+
+/**
+ * Bull works with a Job wrapper around all messages that contains a lot more information about
+ * the state of the message, this object constructor implements the same schema of Bull jobs
+ * for the sake of maintaining API consistency.
+ * @param {string} queue The name of the queue which the message will be carried on.
+ * @param {object} message The JSON message which will be passed back to the consumer.
+ * @returns {Object} A new job which can now be put onto the queue, this is mostly an
+ * internal structure so that an in memory queue can be easily swapped for a Bull queue.
+ */
+function newJob(queue: string, message: any) {
+  return {
+    timestamp: Date.now(),
+    queue: queue,
+    data: message,
+  }
+}
+
+/**
+ * This is designed to replicate Bull (https://github.com/OptimalBits/bull) in memory as a sort of mock.
+ * It is relatively simple, using an event emitter internally to register when messages are available
+ * to the consumers - in can support many inputs and many consumers.
+ */
+class InMemoryQueue {
+  _name: string
+  _opts?: any
+  _messages: any[]
+  _emitter: EventEmitter
+  /**
+   * The constructor the queue, exactly the same as that of Bulls.
+   * @param {string} name The name of the queue which is being configured.
+   * @param {object|null} opts This is not used by the in memory queue as there is no real use
+   * case when in memory, but is the same API as Bull
+   */
+  constructor(name: string, opts = null) {
+    this._name = name
+    this._opts = opts
+    this._messages = []
+    this._emitter = new events.EventEmitter()
+  }
+
+  /**
+   * Same callback API as Bull, each callback passed to this will consume messages as they are
+   * available. Please note this is a queue service, not a notification service, so each
+   * consumer will receive different messages.
+   * @param {function<object>} func The callback function which will return a "Job", the same
+   * as the Bull API, within this job the property "data" contains the JSON message. Please
+   * note this is incredibly limited compared to Bull as in reality the Job would contain
+   * a lot more information about the queue and current status of Bull cluster.
+   */
+  process(func: any) {
+    this._emitter.on("message", async () => {
+      if (this._messages.length <= 0) {
+        return
+      }
+      let msg = this._messages.shift()
+      let resp = func(msg)
+      if (resp.then != null) {
+        await resp
+      }
+    })
+  }
+
+  // simply puts a message to the queue and emits to the queue for processing
+  /**
+   * Simple function to replicate the add message functionality of Bull, putting
+   * a new message on the queue. This then emits an event which will be used to
+   * return the message to a consumer (if one is attached).
+   * @param {object} msg A message to be transported over the queue, this should be
+   * a JSON message as this is required by Bull.
+   * @param {boolean} repeat serves no purpose for the import queue.
+   */
+  // eslint-disable-next-line no-unused-vars
+  add(msg: any, repeat: boolean) {
+    if (typeof msg !== "object") {
+      throw "Queue only supports carrying JSON."
+    }
+    this._messages.push(newJob(this._name, msg))
+    this._emitter.emit("message")
+  }
+
+  /**
+   * replicating the close function from bull, which waits for jobs to finish.
+   */
+  async close() {
+    return []
+  }
+
+  /**
+   * This removes a cron which has been implemented, this is part of Bull API.
+   * @param {string} cronJobId The cron which is to be removed.
+   */
+  removeRepeatableByKey(cronJobId: string) {
+    // TODO: implement for testing
+    console.log(cronJobId)
+  }
+
+  /**
+   * Implemented for tests
+   */
+  getRepeatableJobs() {
+    return []
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  removeJobs(pattern: string) {
+    // no-op
+  }
+
+  /**
+   * Implemented for tests
+   */
+  async clean() {
+    return []
+  }
+
+  async getJob() {
+    return {}
+  }
+
+  on() {
+    // do nothing
+  }
+}
+
+export = InMemoryQueue
diff --git a/packages/backend-core/src/queue/index.ts b/packages/backend-core/src/queue/index.ts
new file mode 100644
index 0000000000..b7d565ba13
--- /dev/null
+++ b/packages/backend-core/src/queue/index.ts
@@ -0,0 +1,2 @@
+export * from "./queue"
+export * from "./constants"
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
new file mode 100644
index 0000000000..de2c738ca4
--- /dev/null
+++ b/packages/backend-core/src/queue/queue.ts
@@ -0,0 +1,47 @@
+import env from "../environment"
+import { getRedisOptions } from "../redis/utils"
+import { JobQueue } from "./constants"
+import inMemoryQueue from "./inMemoryQueue"
+import BullQueue from "bull"
+import InMemoryQueue from "./inMemoryQueue"
+const { opts, redisProtocolUrl } = getRedisOptions()
+
+const CLEANUP_PERIOD_MS = 60 * 1000
+let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
+let cleanupInterval: NodeJS.Timeout
+
+async function cleanup() {
+  for (let queue of QUEUES) {
+    await queue.clean(CLEANUP_PERIOD_MS, "completed")
+  }
+}
+
+export function createQueue(jobQueue: JobQueue) {
+  const queueConfig: any = redisProtocolUrl || { redis: opts }
+  let queue: any
+  if (env.isTest()) {
+    queue = new BullQueue(jobQueue, queueConfig)
+  } else {
+    queue = new inMemoryQueue(jobQueue, queueConfig)
+  }
+  QUEUES.push(queue)
+  if (!cleanupInterval) {
+    cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
+  return queue
+}
+
+exports.shutdown = async () => {
+  if (QUEUES.length) {
+    clearInterval(cleanupInterval)
+    for (let queue of QUEUES) {
+      await queue.close()
+    }
+    QUEUES = []
+  }
+  console.log("Queues shutdown")
+}
diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock
index 6bc9b63728..31d25320c7 100644
--- a/packages/backend-core/yarn.lock
+++ b/packages/backend-core/yarn.lock
@@ -291,6 +291,11 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/types@2.0.30-alpha.3":
+  version "2.0.30-alpha.3"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2"
+  integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg==
+
 "@hapi/hoek@^9.0.0":
   version "9.3.0"
   resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb"
@@ -543,6 +548,36 @@
     semver "^7.3.5"
     tar "^6.1.11"
 
+"@msgpackr-extract/msgpackr-extract-darwin-arm64@2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.1.2.tgz#9571b87be3a3f2c46de05585470bc4f3af2f6f00"
+  integrity sha512-TyVLn3S/+ikMDsh0gbKv2YydKClN8HaJDDpONlaZR+LVJmsxLFUgA+O7zu59h9+f9gX1aj/ahw9wqa6rosmrYQ==
+
+"@msgpackr-extract/msgpackr-extract-darwin-x64@2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.1.2.tgz#bfbc6936ede2955218f5621a675679a5fe8e6f4c"
+  integrity sha512-YPXtcVkhmVNoMGlqp81ZHW4dMxK09msWgnxtsDpSiZwTzUBG2N+No2bsr7WMtBKCVJMSD6mbAl7YhKUqkp/Few==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm64@2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.1.2.tgz#22555e28382af2922e7450634c8a2f240bb9eb82"
+  integrity sha512-vHZ2JiOWF2+DN9lzltGbhtQNzDo8fKFGrf37UJrgqxU0yvtERrzUugnfnX1wmVfFhSsF8OxrfqiNOUc5hko1Zg==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm@2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.1.2.tgz#ffb6ae1beea7ac572b6be6bf2a8e8162ebdd8be7"
+  integrity sha512-42R4MAFeIeNn+L98qwxAt360bwzX2Kf0ZQkBBucJ2Ircza3asoY4CDbgiu9VWklq8gWJVSJSJBwDI+c/THiWkA==
+
+"@msgpackr-extract/msgpackr-extract-linux-x64@2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.1.2.tgz#7caf62eebbfb1345de40f75e89666b3d4194755f"
+  integrity sha512-RjRoRxg7Q3kPAdUSC5EUUPlwfMkIVhmaRTIe+cqHbKrGZ4M6TyCA/b5qMaukQ/1CHWrqYY2FbKOAU8Hg0pQFzg==
+
+"@msgpackr-extract/msgpackr-extract-win32-x64@2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.1.2.tgz#f2d8b9ddd8d191205ed26ce54aba3dfc5ae3e7c9"
+  integrity sha512-rIZVR48zA8hGkHIK7ED6+ZiXsjRCcAVBJbm8o89OKAMTmEAQ2QvoOxoiu3w2isAaWwzgtQIOFIqHwvZDyLKCvw==
+
 "@shopify/jest-koa-mocks@5.0.1":
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/@shopify/jest-koa-mocks/-/jest-koa-mocks-5.0.1.tgz#fba490b6b7985fbb571eb9974897d396a3642e94"
@@ -1497,6 +1532,21 @@ buffer@^5.5.0, buffer@^5.6.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
+bull@^4.10.1:
+  version "4.10.1"
+  resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f"
+  integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g==
+  dependencies:
+    cron-parser "^4.2.1"
+    debuglog "^1.0.0"
+    get-port "^5.1.1"
+    ioredis "^4.28.5"
+    lodash "^4.17.21"
+    msgpackr "^1.5.2"
+    p-timeout "^3.2.0"
+    semver "^7.3.2"
+    uuid "^8.3.0"
+
 cache-content-type@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/cache-content-type/-/cache-content-type-1.0.1.tgz#035cde2b08ee2129f4a8315ea8f00a00dba1453c"
@@ -1764,6 +1814,13 @@ core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
   integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
 
+cron-parser@^4.2.1:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.6.0.tgz#404c3fdbff10ae80eef6b709555d577ef2fd2e0d"
+  integrity sha512-guZNLMGUgg6z4+eGhmHGw7ft+v6OQeuHzd1gcLxCo9Yg/qoxmG3nindp2/uwGCLizEisf2H0ptqeVXeoCpP6FA==
+  dependencies:
+    luxon "^3.0.1"
+
 cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -1837,6 +1894,11 @@ debug@~3.1.0:
   dependencies:
     ms "2.0.0"
 
+debuglog@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
+  integrity sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==
+
 decimal.js@^10.2.1:
   version "10.3.1"
   resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783"
@@ -2318,6 +2380,11 @@ get-package-type@^0.1.0:
   resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a"
   integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==
 
+get-port@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193"
+  integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==
+
 get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
@@ -2652,6 +2719,23 @@ ioredis@4.28.0:
     redis-parser "^3.0.0"
     standard-as-callback "^2.1.0"
 
+ioredis@^4.28.5:
+  version "4.28.5"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f"
+  integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==
+  dependencies:
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.1"
+    denque "^1.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.flatten "^4.4.0"
+    lodash.isarguments "^3.1.0"
+    p-map "^2.1.0"
+    redis-commands "1.7.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 is-arrayish@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
@@ -3725,6 +3809,11 @@ ltgt@2.2.1, ltgt@^2.1.2, ltgt@~2.2.0:
   resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5"
   integrity sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==
 
+luxon@^3.0.1:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929"
+  integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw==
+
 make-dir@^3.0.0, make-dir@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@@ -3872,6 +3961,27 @@ ms@^2.1.1, ms@^2.1.3:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
+msgpackr-extract@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.1.2.tgz#56272030f3e163e1b51964ef8b1cd5e7240c03ed"
+  integrity sha512-cmrmERQFb19NX2JABOGtrKdHMyI6RUyceaPBQ2iRz9GnDkjBWFjNJC0jyyoOfZl2U/LZE3tQCCQc4dlRyA8mcA==
+  dependencies:
+    node-gyp-build-optional-packages "5.0.3"
+  optionalDependencies:
+    "@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.1.2"
+    "@msgpackr-extract/msgpackr-extract-darwin-x64" "2.1.2"
+    "@msgpackr-extract/msgpackr-extract-linux-arm" "2.1.2"
+    "@msgpackr-extract/msgpackr-extract-linux-arm64" "2.1.2"
+    "@msgpackr-extract/msgpackr-extract-linux-x64" "2.1.2"
+    "@msgpackr-extract/msgpackr-extract-win32-x64" "2.1.2"
+
+msgpackr@^1.5.2:
+  version "1.7.2"
+  resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261"
+  integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ==
+  optionalDependencies:
+    msgpackr-extract "^2.1.2"
+
 napi-macros@~2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b"
@@ -3919,6 +4029,11 @@ node-forge@^0.7.1:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac"
   integrity sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==
 
+node-gyp-build-optional-packages@5.0.3:
+  version "5.0.3"
+  resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
+  integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==
+
 node-gyp-build@~4.1.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.1.1.tgz#d7270b5d86717068d114cc57fff352f96d745feb"
@@ -4075,6 +4190,11 @@ p-cancelable@^1.0.0:
   resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc"
   integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==
 
+p-finally@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
+  integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==
+
 p-limit@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
@@ -4094,6 +4214,13 @@ p-map@^2.1.0:
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
   integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
 
+p-timeout@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
+  integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
+  dependencies:
+    p-finally "^1.0.0"
+
 p-try@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
@@ -5360,7 +5487,7 @@ uuid@8.1.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d"
   integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg==
 
-uuid@8.3.2, uuid@^8.3.2:
+uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js
index 3aac6c4fed..b0ccf14634 100644
--- a/packages/server/src/automations/bullboard.js
+++ b/packages/server/src/automations/bullboard.js
@@ -1,37 +1,15 @@
 const { createBullBoard } = require("@bull-board/api")
 const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
-const env = require("../environment")
-const Queue = env.isTest()
-  ? require("../utilities/queue/inMemoryQueue")
-  : require("bull")
-const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/backend-core/redis")
-const { opts, redisProtocolUrl } = utils.getRedisOptions()
+const { queue } = require("@budibase/backend-core")
 const listeners = require("./listeners")
 
-const CLEANUP_PERIOD_MS = 60 * 1000
-const queueConfig = redisProtocolUrl || { redis: opts }
-let cleanupInternal = null
-
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+let automationQueue = queue.createQueue(queue.JobQueues.AUTOMATIONS)
 listeners.addListeners(automationQueue)
 
-async function cleanup() {
-  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
-}
-
 const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  // cleanup the events every 5 minutes
-  if (!cleanupInternal) {
-    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
-    // fire off an initial cleanup
-    cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
-    })
-  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
@@ -48,12 +26,7 @@ exports.init = () => {
 }
 
 exports.shutdown = async () => {
-  if (automationQueue) {
-    clearInterval(cleanupInternal)
-    await automationQueue.close()
-    automationQueue = null
-  }
-  console.log("Bull shutdown")
+  await queue.shutdown()
 }
 
 exports.queue = automationQueue
diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js
index b9362cecf6..73bffeabd6 100644
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@@ -2,10 +2,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { UserStatus } = require("@budibase/backend-core/constants")
 const { objectStore } = require("@budibase/backend-core")
 
-exports.JobQueues = {
-  AUTOMATIONS: "automationQueue",
-}
-
 const FilterTypes = {
   STRING: "string",
   FUZZY: "fuzzy",

From b6ca14aa85bf83784da4d23e546bbbc99eea177f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 13 Oct 2022 17:39:26 +0100
Subject: [PATCH 17/46] Renaming some queue to automationQueue, getting build
 working.

---
 packages/backend-core/package.json           |  1 +
 packages/backend-core/yarn.lock              |  7 +++++++
 packages/server/src/automations/bullboard.js |  4 ++--
 packages/server/src/automations/index.js     |  8 ++++----
 packages/server/src/automations/triggers.js  |  8 ++++----
 packages/server/src/automations/utils.ts     | 14 +++++++-------
 6 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 404b7da346..eb90fa4159 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -64,6 +64,7 @@
   },
   "devDependencies": {
     "@types/chance": "1.1.3",
+    "@types/ioredis": "^4.28.10",
     "@types/jest": "27.5.1",
     "@types/koa": "2.0.52",
     "@types/lodash": "4.14.180",
diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock
index 31d25320c7..22b93d7b3e 100644
--- a/packages/backend-core/yarn.lock
+++ b/packages/backend-core/yarn.lock
@@ -768,6 +768,13 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
   integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
 
+"@types/ioredis@^4.28.10":
+  version "4.28.10"
+  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
+  integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44"
diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js
index b0ccf14634..9dfa7546ec 100644
--- a/packages/server/src/automations/bullboard.js
+++ b/packages/server/src/automations/bullboard.js
@@ -4,7 +4,7 @@ const { KoaAdapter } = require("@bull-board/koa")
 const { queue } = require("@budibase/backend-core")
 const listeners = require("./listeners")
 
-let automationQueue = queue.createQueue(queue.JobQueues.AUTOMATIONS)
+let automationQueue = queue.createQueue(queue.JobQueue.AUTOMATIONS)
 listeners.addListeners(automationQueue)
 
 const PATH_PREFIX = "/bulladmin"
@@ -29,4 +29,4 @@ exports.shutdown = async () => {
   await queue.shutdown()
 }
 
-exports.queue = automationQueue
+exports.automationQueue = automationQueue
diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js
index 2baa868890..521991dd2c 100644
--- a/packages/server/src/automations/index.js
+++ b/packages/server/src/automations/index.js
@@ -1,5 +1,5 @@
 const { processEvent } = require("./utils")
-const { queue, shutdown } = require("./bullboard")
+const { automationQueue, shutdown } = require("./bullboard")
 const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
@@ -8,7 +8,7 @@ const { ACTION_DEFINITIONS } = require("./actions")
  */
 exports.init = async function () {
   // this promise will not complete
-  const promise = queue.process(async job => {
+  const promise = automationQueue.process(async job => {
     await processEvent(job)
   })
   // on init we need to trigger any reboot automations
@@ -17,13 +17,13 @@ exports.init = async function () {
 }
 
 exports.getQueues = () => {
-  return [queue]
+  return [automationQueue]
 }
 
 exports.shutdown = () => {
   return shutdown()
 }
 
-exports.queue = queue
+exports.automationQueue = automationQueue
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS
 exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS
diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js
index 395390113a..6a4bbd8da6 100644
--- a/packages/server/src/automations/triggers.js
+++ b/packages/server/src/automations/triggers.js
@@ -4,7 +4,7 @@ const { coerce } = require("../utilities/rowProcessor")
 const { definitions } = require("./triggerInfo")
 const { isDevAppID } = require("../db/utils")
 // need this to call directly, so we can get a response
-const { queue } = require("./bullboard")
+const { automationQueue } = require("./bullboard")
 const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
@@ -56,7 +56,7 @@ async function queueRelevantRowAutomations(event, eventType) {
         automationTrigger.inputs &&
         automationTrigger.inputs.tableId === event.row.tableId
       ) {
-        await queue.add({ automation, event }, JOB_OPTS)
+        await automationQueue.add({ automation, event }, JOB_OPTS)
       }
     }
   })
@@ -110,7 +110,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data, JOB_OPTS)
+    return automationQueue.add(data, JOB_OPTS)
   }
 }
 
@@ -136,7 +136,7 @@ exports.rebootTrigger = async () => {
               timestamp: Date.now(),
             },
           }
-          rebootEvents.push(queue.add(job, JOB_OPTS))
+          rebootEvents.push(automationQueue.add(job, JOB_OPTS))
         }
       }
       await Promise.all(rebootEvents)
diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index 7e19486798..54b8078f30 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -1,7 +1,7 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
 import * as webhooks from "../api/controllers/webhook"
-import { queue } from "./bullboard"
+import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
 import { MetadataTypes, WebhookType } from "../constants"
@@ -79,12 +79,12 @@ export function removeDeprecated(definitions: any) {
 // end the repetition and the job itself
 export async function disableAllCrons(appId: any) {
   const promises = []
-  const jobs = await queue.getRepeatableJobs()
+  const jobs = await automationQueue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
-      promises.push(queue.removeRepeatableByKey(job.key))
+      promises.push(automationQueue.removeRepeatableByKey(job.key))
       if (job.id) {
-        promises.push(queue.removeJobs(job.id))
+        promises.push(automationQueue.removeJobs(job.id))
       }
     }
   }
@@ -92,8 +92,8 @@ export async function disableAllCrons(appId: any) {
 }
 
 export async function disableCron(jobId: string, jobKey: string) {
-  await queue.removeRepeatableByKey(jobKey)
-  await queue.removeJobs(jobId)
+  await automationQueue.removeRepeatableByKey(jobKey)
+  await automationQueue.removeJobs(jobId)
   console.log(`jobId=${jobId} disabled`)
 }
 
@@ -141,7 +141,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
   ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job: any = await queue.add(
+    const job: any = await automationQueue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },

From d620e54fdbb1e8ffdd5da6492adc36bc46b7fa45 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 13 Oct 2022 17:55:05 +0100
Subject: [PATCH 18/46] Handling listeners as part of queue creation, rather
 than external part.

---
 .../src/queue}/listeners.ts                   | 62 +++++++++++--------
 packages/backend-core/src/queue/queue.ts      | 12 ++--
 packages/server/src/automations/bullboard.js  |  8 ++-
 3 files changed, 50 insertions(+), 32 deletions(-)
 rename packages/{server/src/automations => backend-core/src/queue}/listeners.ts (52%)

diff --git a/packages/server/src/automations/listeners.ts b/packages/backend-core/src/queue/listeners.ts
similarity index 52%
rename from packages/server/src/automations/listeners.ts
rename to packages/backend-core/src/queue/listeners.ts
index 9f8667bd29..c5db628ef0 100644
--- a/packages/server/src/automations/listeners.ts
+++ b/packages/backend-core/src/queue/listeners.ts
@@ -1,78 +1,90 @@
-import { Queue, Job, JobId } from "bull"
-import { AutomationEvent } from "../definitions/automations"
-import * as automation from "../threads/automation"
+import { Job, JobId, Queue } from "bull"
+import { JobQueue } from "./constants"
 
-export const addListeners = (queue: Queue) => {
-  logging(queue)
-  handleStalled(queue)
+export type StalledFn = (job: Job) => Promise<void>
+
+export const addListeners = (
+  queue: Queue,
+  jobQueue: JobQueue,
+  removeStalled?: StalledFn
+) => {
+  logging(queue, jobQueue)
+  if (removeStalled) {
+    handleStalled(queue, removeStalled)
+  }
 }
 
-const handleStalled = (queue: Queue) => {
+const handleStalled = (queue: Queue, removeStalled: StalledFn) => {
   queue.on("stalled", async (job: Job) => {
-    await automation.removeStalled(job as AutomationEvent)
+    await removeStalled(job)
   })
 }
 
-const logging = (queue: Queue) => {
+const logging = (queue: Queue, jobQueue: JobQueue) => {
+  let eventType: string
+  switch (jobQueue) {
+    case JobQueue.AUTOMATIONS:
+      eventType = "automation-event"
+      break
+    case JobQueue.APP_BACKUPS:
+      eventType = "app-backup-event"
+      break
+  }
   if (process.env.NODE_DEBUG?.includes("bull")) {
     queue
       .on("error", (error: any) => {
         // An error occurred.
-        console.error(`automation-event=error error=${JSON.stringify(error)}`)
+        console.error(`${eventType}=error error=${JSON.stringify(error)}`)
       })
       .on("waiting", (jobId: JobId) => {
         // A Job is waiting to be processed as soon as a worker is idling.
-        console.log(`automation-event=waiting jobId=${jobId}`)
+        console.log(`${eventType}=waiting jobId=${jobId}`)
       })
       .on("active", (job: Job, jobPromise: any) => {
         // A job has started. You can use `jobPromise.cancel()`` to abort it.
-        console.log(`automation-event=active jobId=${job.id}`)
+        console.log(`${eventType}=active jobId=${job.id}`)
       })
       .on("stalled", (job: Job) => {
         // A job has been marked as stalled. This is useful for debugging job
         // workers that crash or pause the event loop.
         console.error(
-          `automation-event=stalled jobId=${job.id} job=${JSON.stringify(job)}`
+          `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}`
         )
       })
       .on("progress", (job: Job, progress: any) => {
         // A job's progress was updated!
         console.log(
-          `automation-event=progress jobId=${job.id} progress=${progress}`
+          `${eventType}=progress jobId=${job.id} progress=${progress}`
         )
       })
       .on("completed", (job: Job, result) => {
         // A job successfully completed with a `result`.
-        console.log(
-          `automation-event=completed jobId=${job.id} result=${result}`
-        )
+        console.log(`${eventType}=completed jobId=${job.id} result=${result}`)
       })
       .on("failed", (job, err: any) => {
         // A job failed with reason `err`!
-        console.log(`automation-event=failed jobId=${job.id} error=${err}`)
+        console.log(`${eventType}=failed jobId=${job.id} error=${err}`)
       })
       .on("paused", () => {
         // The queue has been paused.
-        console.log(`automation-event=paused`)
+        console.log(`${eventType}=paused`)
       })
       .on("resumed", (job: Job) => {
         // The queue has been resumed.
-        console.log(`automation-event=paused jobId=${job.id}`)
+        console.log(`${eventType}=paused jobId=${job.id}`)
       })
       .on("cleaned", (jobs: Job[], type: string) => {
         // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
         // jobs, and `type` is the type of jobs cleaned.
-        console.log(
-          `automation-event=cleaned length=${jobs.length} type=${type}`
-        )
+        console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`)
       })
       .on("drained", () => {
         // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
-        console.log(`automation-event=drained`)
+        console.log(`${eventType}=drained`)
       })
       .on("removed", (job: Job) => {
         // A job successfully removed.
-        console.log(`automation-event=removed jobId=${job.id}`)
+        console.log(`${eventType}=removed jobId=${job.id}`)
       })
   }
 }
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index de2c738ca4..8f7ac79e7a 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -1,9 +1,9 @@
 import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
-import inMemoryQueue from "./inMemoryQueue"
-import BullQueue from "bull"
 import InMemoryQueue from "./inMemoryQueue"
+import BullQueue from "bull"
+import { addListeners, StalledFn } from "./listeners"
 const { opts, redisProtocolUrl } = getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000
@@ -16,14 +16,18 @@ async function cleanup() {
   }
 }
 
-export function createQueue(jobQueue: JobQueue) {
+export function createQueue(
+  jobQueue: JobQueue,
+  removeStalled: StalledFn
+): BullQueue.Queue {
   const queueConfig: any = redisProtocolUrl || { redis: opts }
   let queue: any
   if (env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
   } else {
-    queue = new inMemoryQueue(jobQueue, queueConfig)
+    queue = new InMemoryQueue(jobQueue, queueConfig)
   }
+  addListeners(queue, jobQueue, removeStalled)
   QUEUES.push(queue)
   if (!cleanupInterval) {
     cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS)
diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js
index 9dfa7546ec..af2243f13d 100644
--- a/packages/server/src/automations/bullboard.js
+++ b/packages/server/src/automations/bullboard.js
@@ -2,10 +2,12 @@ const { createBullBoard } = require("@bull-board/api")
 const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
 const { queue } = require("@budibase/backend-core")
-const listeners = require("./listeners")
+const automation = require("../threads/automation")
 
-let automationQueue = queue.createQueue(queue.JobQueue.AUTOMATIONS)
-listeners.addListeners(automationQueue)
+let automationQueue = queue.createQueue(
+  queue.JobQueue.AUTOMATIONS,
+  automation.removeStalled
+)
 
 const PATH_PREFIX = "/bulladmin"
 

From b702c7482a5bdf8270137ead8ffaec4fbb6abc40 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 14 Oct 2022 13:26:42 +0100
Subject: [PATCH 19/46] Fixes for cronjob stop - correctly handle this without
 stalled job handle.

---
 packages/backend-core/src/queue/listeners.ts  | 21 +++++++--
 packages/backend-core/src/queue/queue.ts      |  2 +-
 packages/server/src/automations/utils.ts      | 10 ++--
 .../server/src/definitions/automations.ts     | 12 -----
 packages/server/src/threads/automation.ts     | 47 ++++++++-----------
 5 files changed, 44 insertions(+), 48 deletions(-)

diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts
index c5db628ef0..0e06fb5ef0 100644
--- a/packages/backend-core/src/queue/listeners.ts
+++ b/packages/backend-core/src/queue/listeners.ts
@@ -3,24 +3,35 @@ import { JobQueue } from "./constants"
 
 export type StalledFn = (job: Job) => Promise<void>
 
-export const addListeners = (
+export function addListeners(
   queue: Queue,
   jobQueue: JobQueue,
   removeStalled?: StalledFn
-) => {
+) {
   logging(queue, jobQueue)
   if (removeStalled) {
     handleStalled(queue, removeStalled)
   }
 }
 
-const handleStalled = (queue: Queue, removeStalled: StalledFn) => {
+function handleStalled(queue: Queue, removeStalled?: StalledFn) {
   queue.on("stalled", async (job: Job) => {
-    await removeStalled(job)
+    if (removeStalled) {
+      await removeStalled(job)
+    } else if (job.opts.repeat) {
+      const jobId = job.id
+      const repeatJobs = await queue.getRepeatableJobs()
+      for (let repeatJob of repeatJobs) {
+        if (repeatJob.id === jobId) {
+          await queue.removeRepeatableByKey(repeatJob.key)
+        }
+      }
+      console.log(`jobId=${jobId} disabled`)
+    }
   })
 }
 
-const logging = (queue: Queue, jobQueue: JobQueue) => {
+function logging(queue: Queue, jobQueue: JobQueue) {
   let eventType: string
   switch (jobQueue) {
     case JobQueue.AUTOMATIONS:
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index 8f7ac79e7a..d83e421456 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -18,7 +18,7 @@ async function cleanup() {
 
 export function createQueue(
   jobQueue: JobQueue,
-  removeStalled: StalledFn
+  removeStalled?: StalledFn
 ): BullQueue.Queue {
   const queueConfig: any = redisProtocolUrl || { redis: opts }
   let queue: any
diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts
index 54b8078f30..0eebcb21cf 100644
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@@ -91,9 +91,13 @@ export async function disableAllCrons(appId: any) {
   return Promise.all(promises)
 }
 
-export async function disableCron(jobId: string, jobKey: string) {
-  await automationQueue.removeRepeatableByKey(jobKey)
-  await automationQueue.removeJobs(jobId)
+export async function disableCronById(jobId: number | string) {
+  const repeatJobs = await automationQueue.getRepeatableJobs()
+  for (let repeatJob of repeatJobs) {
+    if (repeatJob.id === jobId) {
+      await automationQueue.removeRepeatableByKey(repeatJob.key)
+    }
+  }
   console.log(`jobId=${jobId} disabled`)
 }
 
diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts
index ed1455c049..877a1b4579 100644
--- a/packages/server/src/definitions/automations.ts
+++ b/packages/server/src/definitions/automations.ts
@@ -27,18 +27,6 @@ export interface TriggerOutput {
   timestamp?: number
 }
 
-export interface AutomationEvent {
-  data: {
-    automation: Automation
-    event: any
-  }
-  opts?: {
-    repeat?: {
-      jobId: string
-    }
-  }
-}
-
 export interface AutomationContext extends AutomationResults {
   steps: any[]
   trigger: any
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 64ae9439d8..b4b290462e 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -1,6 +1,11 @@
 import { default as threadUtils } from "./utils"
+import { Job } from "bull"
 threadUtils.threadSetup()
-import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils"
+import {
+  isRecurring,
+  disableCronById,
+  isErrorInOutput,
+} from "../automations/utils"
 import { default as actions } from "../automations/actions"
 import { default as automationUtils } from "../automations/automationUtils"
 import { default as AutomationEmitter } from "../events/AutomationEmitter"
@@ -13,7 +18,6 @@ import {
   LoopStep,
   LoopStepType,
   LoopInput,
-  AutomationEvent,
   TriggerOutput,
   AutomationContext,
   AutomationMetadata,
@@ -73,19 +77,16 @@ class Orchestrator {
   _automation: Automation
   _emitter: any
   _context: AutomationContext
-  _repeat?: { jobId: string; jobKey: string }
+  _job: Job
   executionOutput: AutomationContext
 
-  constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) {
+  constructor(job: Job) {
+    let automation = job.data.automation,
+      triggerOutput = job.data.event
     const metadata = triggerOutput.metadata
     this._chainCount = metadata ? metadata.automationChainCount : 0
     this._appId = triggerOutput.appId as string
-    if (opts?.repeat) {
-      this._repeat = {
-        jobId: opts.repeat.jobId,
-        jobKey: opts.repeat.key,
-      }
-    }
+    this._job = job
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
@@ -134,7 +135,7 @@ class Orchestrator {
   }
 
   async stopCron(reason: string) {
-    if (!this._repeat) {
+    if (!this._job.opts.repeat) {
       return
     }
     logWarn(
@@ -142,7 +143,7 @@ class Orchestrator {
     )
     const automation = this._automation
     const trigger = automation.definition.trigger
-    await disableCron(this._repeat?.jobId, this._repeat?.jobKey)
+    await disableCronById(this._job.id)
     this.updateExecutionOutput(
       trigger.id,
       trigger.stepId,
@@ -156,7 +157,7 @@ class Orchestrator {
   }
 
   async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this._repeat) {
+    if (!metadata.errorCount || !this._job.opts.repeat) {
       return false
     }
     if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
@@ -475,17 +476,13 @@ class Orchestrator {
   }
 }
 
-export function execute(input: AutomationEvent, callback: WorkerCallback) {
-  const appId = input.data.event.appId
+export function execute(job: Job, callback: WorkerCallback) {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     try {
       const response = await automationOrchestrator.execute()
       callback(null, response)
@@ -495,17 +492,13 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) {
   })
 }
 
-export const removeStalled = async (input: AutomationEvent) => {
-  const appId = input.data.event.appId
+export const removeStalled = async (job: Job) => {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     await automationOrchestrator.stopCron("stalled")
   })
 }

From 0bd2a18e46f09c51a75d19f6161a1a3c0bfb358a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 14 Oct 2022 19:24:03 +0100
Subject: [PATCH 20/46] Main types and work for the CRUD operations of app
 backup backend in pro + the listeners to handle exporting apps from the
 server.

---
 packages/backend-core/package.json            |  1 +
 packages/backend-core/src/db/constants.ts     |  2 +
 .../backend-core/src/objectStore/index.ts     |  3 +-
 packages/backend-core/src/queue/constants.ts  |  4 +-
 packages/backend-core/src/queue/listeners.ts  |  4 +-
 packages/backend-core/src/queue/queue.ts      |  4 +-
 packages/backend-core/yarn.lock               | 10 ++++-
 packages/server/src/app.ts                    |  2 +
 packages/server/src/automations/bullboard.js  |  2 +-
 packages/server/src/sdk/app/backups/backup.ts | 39 +++++++++++++++++++
 packages/server/src/sdk/app/backups/index.ts  |  2 +
 packages/types/src/api/web/app/backup.ts      |  1 +
 packages/types/src/documents/app/backup.ts    | 37 +++++++++++++-----
 13 files changed, 93 insertions(+), 18 deletions(-)
 create mode 100644 packages/server/src/sdk/app/backups/backup.ts

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index eb90fa4159..17977decba 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -63,6 +63,7 @@
     ]
   },
   "devDependencies": {
+    "@types/bull": "^3.15.9",
     "@types/chance": "1.1.3",
     "@types/ioredis": "^4.28.10",
     "@types/jest": "27.5.1",
diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts
index a61e8a2af2..8f8f45638b 100644
--- a/packages/backend-core/src/db/constants.ts
+++ b/packages/backend-core/src/db/constants.ts
@@ -21,6 +21,7 @@ export enum ViewName {
   ACCOUNT_BY_EMAIL = "account_by_email",
   PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
   USER_BY_GROUP = "by_group_user",
+  APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
 
 export const DeprecatedViews = {
@@ -49,6 +50,7 @@ export enum DocumentType {
   TABLE = "ta",
   DATASOURCE = "datasource",
   DATASOURCE_PLUS = "datasource_plus",
+  APP_BACKUP = "backup",
 }
 
 export const StaticDatabases = {
diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts
index e210f9dccc..8453c9aee6 100644
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@@ -27,6 +27,7 @@ const CONTENT_TYPE_MAP: any = {
   css: "text/css",
   js: "application/javascript",
   json: "application/json",
+  gz: "application/gzip",
 }
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
@@ -149,7 +150,7 @@ export const upload = async ({
   type,
   metadata,
 }: any) => {
-  const extension = [...filename.split(".")].pop()
+  const extension = filename.split(".").pop()
   const fileBytes = fs.readFileSync(path)
 
   const objectStore = ObjectStore(bucketName)
diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts
index d8fb3121a3..e8323dacb8 100644
--- a/packages/backend-core/src/queue/constants.ts
+++ b/packages/backend-core/src/queue/constants.ts
@@ -1,4 +1,4 @@
 export enum JobQueue {
-  AUTOMATIONS = "automationQueue",
-  APP_BACKUPS = "appBackupQueue",
+  AUTOMATION = "automationQueue",
+  APP_BACKUP = "appBackupQueue",
 }
diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts
index 0e06fb5ef0..f264c3a84c 100644
--- a/packages/backend-core/src/queue/listeners.ts
+++ b/packages/backend-core/src/queue/listeners.ts
@@ -34,10 +34,10 @@ function handleStalled(queue: Queue, removeStalled?: StalledFn) {
 function logging(queue: Queue, jobQueue: JobQueue) {
   let eventType: string
   switch (jobQueue) {
-    case JobQueue.AUTOMATIONS:
+    case JobQueue.AUTOMATION:
       eventType = "automation-event"
       break
-    case JobQueue.APP_BACKUPS:
+    case JobQueue.APP_BACKUP:
       eventType = "app-backup-event"
       break
   }
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index d83e421456..6dd10ee091 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -16,10 +16,10 @@ async function cleanup() {
   }
 }
 
-export function createQueue(
+export function createQueue<T>(
   jobQueue: JobQueue,
   removeStalled?: StalledFn
-): BullQueue.Queue {
+): BullQueue.Queue<T> {
   const queueConfig: any = redisProtocolUrl || { redis: opts }
   let queue: any
   if (env.isTest()) {
diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock
index 22b93d7b3e..d2831ca8fe 100644
--- a/packages/backend-core/yarn.lock
+++ b/packages/backend-core/yarn.lock
@@ -698,6 +698,14 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/bull@^3.15.9":
+  version "3.15.9"
+  resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.9.tgz#e10e0901ec3762bff85716b3c580277960751c93"
+  integrity sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==
+  dependencies:
+    "@types/ioredis" "*"
+    "@types/redis" "^2.8.0"
+
 "@types/chance@1.1.3":
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea"
@@ -768,7 +776,7 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
   integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
 
-"@types/ioredis@^4.28.10":
+"@types/ioredis@*", "@types/ioredis@^4.28.10":
   version "4.28.10"
   resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
   integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts
index 3bfe8976f7..776adb602d 100644
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@@ -37,6 +37,7 @@ import {
 } from "./utilities/workerRequests"
 import { watch } from "./watch"
 import { initialise as initialiseWebsockets } from "./websocket"
+import sdk from "./sdk"
 
 const app = new Koa()
 
@@ -108,6 +109,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  await sdk.backups.init()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment
diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js
index af2243f13d..c4f33e07a9 100644
--- a/packages/server/src/automations/bullboard.js
+++ b/packages/server/src/automations/bullboard.js
@@ -5,7 +5,7 @@ const { queue } = require("@budibase/backend-core")
 const automation = require("../threads/automation")
 
 let automationQueue = queue.createQueue(
-  queue.JobQueue.AUTOMATIONS,
+  queue.JobQueue.AUTOMATION,
   automation.removeStalled
 )
 
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
new file mode 100644
index 0000000000..6a0d370653
--- /dev/null
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -0,0 +1,39 @@
+import { backups } from "@budibase/pro"
+import { objectStore, tenancy } from "@budibase/backend-core"
+import { exportApp } from "./exports"
+import { Job } from "bull"
+import fs from "fs"
+import env from "../../../environment"
+
+export async function init() {
+  await backups.addAppBackupProcessor(async (job: Job) => {
+    const appId = job.data.appId,
+      trigger = job.data.trigger,
+      name = job.data.name
+    const createdAt = new Date().toISOString()
+    const tarPath = await exportApp(appId, { tar: true })
+    let filename = `${appId}/backup-${createdAt}.tar.gz`
+    // add the tenant to the bucket path if backing up within a multi-tenant environment
+    if (env.MULTI_TENANCY) {
+      const tenantId = tenancy.getTenantIDFromAppID(appId)
+      filename = `${tenantId}/${filename}`
+    }
+    const bucket = objectStore.ObjectStoreBuckets.BACKUPS
+    const metadata = {
+      appId,
+      createdAt,
+      trigger,
+      name,
+    }
+    await objectStore.upload({
+      path: tarPath,
+      type: "application/gzip",
+      bucket,
+      filename,
+      metadata,
+    })
+    await backups.storeAppBackupMetadata(filename, metadata)
+    // clear up the tarball after uploading it
+    fs.rmSync(tarPath)
+  })
+}
diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts
index fe7e4e1049..94210c2c4c 100644
--- a/packages/server/src/sdk/app/backups/index.ts
+++ b/packages/server/src/sdk/app/backups/index.ts
@@ -1,7 +1,9 @@
 import * as exportApps from "./exports"
 import * as importApps from "./imports"
+import * as backup from "./backup"
 
 export default {
   ...exportApps,
   ...importApps,
+  ...backup,
 }
diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts
index 57ffba0d70..092cf714b5 100644
--- a/packages/types/src/api/web/app/backup.ts
+++ b/packages/types/src/api/web/app/backup.ts
@@ -4,6 +4,7 @@ export interface SearchAppBackupsRequest {
   trigger: AppBackupTrigger
   startDate: string
   endDate: string
+  page?: string
 }
 
 export interface CreateAppBackupRequest {
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index a935ed5ba1..fb8ef61e3a 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -6,16 +6,35 @@ export enum AppBackupTrigger {
   SCHEDULED = "scheduled",
 }
 
-export interface AppBackupContents {
-  datasources: string[]
-  screens: string[]
-  automations: string[]
-}
-
 export interface AppBackup extends Document {
   trigger: AppBackupTrigger
   name: string
-  date: string
-  userId: string
-  contents: AppBackupContents
+  createdAt: string
+  filename: string
+  appId: string
+  userId?: string
+  contents?: {
+    datasources: string[]
+    screens: string[]
+    automations: string[]
+  }
+}
+
+export type AppBackupFetchOpts = {
+  trigger?: AppBackupTrigger
+  limit?: number
+  page?: string
+  paginate?: boolean
+  startDate?: string
+  endDate?: string
+}
+
+export interface AppBackupQueueData {
+  trigger: AppBackupTrigger
+  name?: string
+  appId: string
+}
+
+export interface AppBackupMetadata extends AppBackupQueueData {
+  createdAt: string
 }

From 1373630b33af7739e55c6e79fe424f580616e431 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 14 Oct 2022 20:10:44 +0100
Subject: [PATCH 21/46] Minor fixes after testing manual backup system.

---
 packages/backend-core/src/context/index.ts    |  3 ++
 packages/server/src/api/routes/index.ts       | 10 ++--
 packages/server/src/sdk/app/backups/backup.ts | 50 ++++++++++---------
 packages/types/src/sdk/koa.ts                 |  9 +++-
 4 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts
index 35eeee608b..7efe0e23f7 100644
--- a/packages/backend-core/src/context/index.ts
+++ b/packages/backend-core/src/context/index.ts
@@ -53,6 +53,9 @@ export const getTenantIDFromAppID = (appId: string) => {
   if (!appId) {
     return null
   }
+  if (!isMultiTenant()) {
+    return DEFAULT_TENANT_ID
+  }
   const split = appId.split(SEPARATOR)
   const hasDev = split[1] === DocumentType.DEV
   if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts
index 1cf34d4a68..02a4900077 100644
--- a/packages/server/src/api/routes/index.ts
+++ b/packages/server/src/api/routes/index.ts
@@ -34,6 +34,8 @@ export { default as publicRoutes } from "./public"
 const appBackupRoutes = api.appBackups
 const scheduleRoutes = api.schedules
 export const mainRoutes: Router[] = [
+  appBackupRoutes,
+  backupRoutes,
   authRoutes,
   deployRoutes,
   layoutRoutes,
@@ -53,16 +55,14 @@ export const mainRoutes: Router[] = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  backupRoutes,
   metadataRoutes,
   devRoutes,
   cloudRoutes,
-  // these need to be handled last as they still use /api/:tableId
-  // this could be breaking as koa may recognise other routes as this
-  tableRoutes,
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
-  appBackupRoutes,
   scheduleRoutes,
+  // these need to be handled last as they still use /api/:tableId
+  // this could be breaking as koa may recognise other routes as this
+  tableRoutes,
 ]
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 6a0d370653..fb4bcc6022 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -10,30 +10,32 @@ export async function init() {
     const appId = job.data.appId,
       trigger = job.data.trigger,
       name = job.data.name
-    const createdAt = new Date().toISOString()
-    const tarPath = await exportApp(appId, { tar: true })
-    let filename = `${appId}/backup-${createdAt}.tar.gz`
-    // add the tenant to the bucket path if backing up within a multi-tenant environment
-    if (env.MULTI_TENANCY) {
-      const tenantId = tenancy.getTenantIDFromAppID(appId)
-      filename = `${tenantId}/${filename}`
-    }
-    const bucket = objectStore.ObjectStoreBuckets.BACKUPS
-    const metadata = {
-      appId,
-      createdAt,
-      trigger,
-      name,
-    }
-    await objectStore.upload({
-      path: tarPath,
-      type: "application/gzip",
-      bucket,
-      filename,
-      metadata,
+    const tenantId = tenancy.getTenantIDFromAppID(appId)
+    await tenancy.doInTenant(tenantId, async () => {
+      const createdAt = new Date().toISOString()
+      const tarPath = await exportApp(appId, { tar: true })
+      let filename = `${appId}/backup-${createdAt}.tar.gz`
+      // add the tenant to the bucket path if backing up within a multi-tenant environment
+      if (env.MULTI_TENANCY) {
+        filename = `${tenantId}/${filename}`
+      }
+      const bucket = objectStore.ObjectStoreBuckets.BACKUPS
+      const metadata = {
+        appId,
+        createdAt,
+        trigger,
+        name,
+      }
+      await objectStore.upload({
+        path: tarPath,
+        type: "application/gzip",
+        bucket,
+        filename,
+        metadata,
+      })
+      await backups.storeAppBackupMetadata(filename, metadata)
+      // clear up the tarball after uploading it
+      fs.rmSync(tarPath)
     })
-    await backups.storeAppBackupMetadata(filename, metadata)
-    // clear up the tarball after uploading it
-    fs.rmSync(tarPath)
   })
 }
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index 8d419d5cf1..250b176d79 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -7,7 +7,12 @@ export interface ContextUser extends User {
   license: License
 }
 
-export interface BBContext extends Context {
+export interface BBContext {
   user?: ContextUser
-  body: any
+  request: {
+    body: any
+  }
+  params: any
+  body?: any
+  redirect?: any
 }

From bb4e3ba8cfa6cc95578b111df7bcf5a4b19381bd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 17 Oct 2022 15:26:09 +0100
Subject: [PATCH 22/46] Adding in required events for app backup system.

---
 .../backend-core/src/events/publishers/backup.ts     | 12 ++++++++++++
 packages/backend-core/src/events/publishers/index.ts |  1 +
 packages/types/src/sdk/events/backup.ts              |  7 +++++++
 packages/types/src/sdk/events/event.ts               |  3 +++
 packages/types/src/sdk/events/index.ts               |  1 +
 5 files changed, 24 insertions(+)
 create mode 100644 packages/backend-core/src/events/publishers/backup.ts
 create mode 100644 packages/types/src/sdk/events/backup.ts

diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts
new file mode 100644
index 0000000000..00b4f8db69
--- /dev/null
+++ b/packages/backend-core/src/events/publishers/backup.ts
@@ -0,0 +1,12 @@
+import { AppBackup, AppBackupRevertEvent, Event } from "@budibase/types"
+import { publishEvent } from "../events"
+
+export async function appBackupRestored(backup: AppBackup) {
+  const properties: AppBackupRevertEvent = {
+    appId: backup.appId,
+    backupName: backup.name,
+    backupCreatedAt: backup.createdAt,
+  }
+
+  await publishEvent(Event.APP_BACKUP_RESTORED, properties)
+}
diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts
index 6fe42c4bda..7306312a8f 100644
--- a/packages/backend-core/src/events/publishers/index.ts
+++ b/packages/backend-core/src/events/publishers/index.ts
@@ -19,3 +19,4 @@ export * as installation from "./installation"
 export * as backfill from "./backfill"
 export * as group from "./group"
 export * as plugin from "./plugin"
+export * as backup from "./backup"
diff --git a/packages/types/src/sdk/events/backup.ts b/packages/types/src/sdk/events/backup.ts
new file mode 100644
index 0000000000..f3ddafcafc
--- /dev/null
+++ b/packages/types/src/sdk/events/backup.ts
@@ -0,0 +1,7 @@
+import { BaseEvent } from "./event"
+
+export interface AppBackupRevertEvent extends BaseEvent {
+  appId: string
+  backupName: string
+  backupCreatedAt: string
+}
diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts
index 73e5315713..71caf2bf96 100644
--- a/packages/types/src/sdk/events/event.ts
+++ b/packages/types/src/sdk/events/event.ts
@@ -168,6 +168,9 @@ export enum Event {
   PLUGIN_INIT = "plugin:init",
   PLUGIN_IMPORTED = "plugin:imported",
   PLUGIN_DELETED = "plugin:deleted",
+
+  // BACKUP
+  APP_BACKUP_RESTORED = "app:backup:restored",
 }
 
 // properties added at the final stage of the event pipeline
diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts
index cc0c2b9aa1..5abc30f5b9 100644
--- a/packages/types/src/sdk/events/index.ts
+++ b/packages/types/src/sdk/events/index.ts
@@ -20,3 +20,4 @@ export * from "./backfill"
 export * from "./identification"
 export * from "./userGroup"
 export * from "./plugin"
+export * from "./backup"

From 8003f8b28342bb33564204e74d8f5f87508141e9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 17 Oct 2022 18:50:52 +0100
Subject: [PATCH 23/46] Work for backup download endpoint.

---
 packages/types/src/documents/app/backup.ts | 2 ++
 packages/types/src/sdk/koa.ts              | 1 +
 2 files changed, 3 insertions(+)

diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index fb8ef61e3a..5fb602860e 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -10,6 +10,7 @@ export interface AppBackup extends Document {
   trigger: AppBackupTrigger
   name: string
   createdAt: string
+  createdBy?: string
   filename: string
   appId: string
   userId?: string
@@ -31,6 +32,7 @@ export type AppBackupFetchOpts = {
 
 export interface AppBackupQueueData {
   trigger: AppBackupTrigger
+  createdBy?: string
   name?: string
   appId: string
 }
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index 250b176d79..e6f4a5094e 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -15,4 +15,5 @@ export interface BBContext {
   params: any
   body?: any
   redirect?: any
+  attachment: any
 }

From f795cb0e33b727f09029934d41af9d8fc1b2b06c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 17 Oct 2022 19:42:36 +0100
Subject: [PATCH 24/46] Reformatting types to allow queue to be used for import
 and export.

---
 packages/server/src/sdk/app/backups/backup.ts | 66 ++++++++++---------
 .../server/src/sdk/app/backups/imports.ts     |  2 +-
 packages/types/src/documents/app/backup.ts    | 23 +++++--
 3 files changed, 55 insertions(+), 36 deletions(-)

diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index fb4bcc6022..cb758536bd 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -5,37 +5,41 @@ import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
 
-export async function init() {
-  await backups.addAppBackupProcessor(async (job: Job) => {
-    const appId = job.data.appId,
-      trigger = job.data.trigger,
-      name = job.data.name
-    const tenantId = tenancy.getTenantIDFromAppID(appId)
-    await tenancy.doInTenant(tenantId, async () => {
-      const createdAt = new Date().toISOString()
-      const tarPath = await exportApp(appId, { tar: true })
-      let filename = `${appId}/backup-${createdAt}.tar.gz`
-      // add the tenant to the bucket path if backing up within a multi-tenant environment
-      if (env.MULTI_TENANCY) {
-        filename = `${tenantId}/${filename}`
-      }
-      const bucket = objectStore.ObjectStoreBuckets.BACKUPS
-      const metadata = {
-        appId,
-        createdAt,
-        trigger,
-        name,
-      }
-      await objectStore.upload({
-        path: tarPath,
-        type: "application/gzip",
-        bucket,
-        filename,
-        metadata,
-      })
-      await backups.storeAppBackupMetadata(filename, metadata)
-      // clear up the tarball after uploading it
-      fs.rmSync(tarPath)
+async function importProcessor(job: Job) {}
+
+async function exportProcessor(job: Job) {
+  const appId = job.data.appId,
+    trigger = job.data.trigger,
+    name = job.data.name
+  const tenantId = tenancy.getTenantIDFromAppID(appId)
+  await tenancy.doInTenant(tenantId, async () => {
+    const createdAt = new Date().toISOString()
+    const tarPath = await exportApp(appId, { tar: true })
+    let filename = `${appId}/backup-${createdAt}.tar.gz`
+    // add the tenant to the bucket path if backing up within a multi-tenant environment
+    if (env.MULTI_TENANCY) {
+      filename = `${tenantId}/${filename}`
+    }
+    const bucket = objectStore.ObjectStoreBuckets.BACKUPS
+    const metadata = {
+      appId,
+      createdAt,
+      trigger,
+      name,
+    }
+    await objectStore.upload({
+      path: tarPath,
+      type: "application/gzip",
+      bucket,
+      filename,
+      metadata,
     })
+    await backups.storeAppBackupMetadata(filename, metadata)
+    // clear up the tarball after uploading it
+    fs.rmSync(tarPath)
   })
 }
+
+export async function init() {
+  await backups.addAppBackupProcessors(importProcessor, exportProcessor)
+}
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index d09c4b3f02..60ce63d51e 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -1,5 +1,5 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { APP_PREFIX, TABLE_ROW_PREFIX } from "../../../db/utils"
+import { TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import {
   DB_EXPORT_FILE,
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index 5fb602860e..28e927c772 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -6,6 +6,11 @@ export enum AppBackupTrigger {
   SCHEDULED = "scheduled",
 }
 
+export enum AppBackupEventType {
+  EXPORT = "export",
+  IMPORT = "import",
+}
+
 export interface AppBackup extends Document {
   trigger: AppBackupTrigger
   name: string
@@ -31,12 +36,22 @@ export type AppBackupFetchOpts = {
 }
 
 export interface AppBackupQueueData {
-  trigger: AppBackupTrigger
-  createdBy?: string
-  name?: string
+  eventType: AppBackupEventType
   appId: string
+  export?: {
+    trigger: AppBackupTrigger
+    name?: string
+    createdBy?: string
+  }
+  import?: {
+    backupId: string
+  }
 }
 
-export interface AppBackupMetadata extends AppBackupQueueData {
+export interface AppBackupMetadata {
+  appId: string
+  trigger: AppBackupTrigger
+  name?: string
+  createdBy?: string
   createdAt: string
 }

From caa4954d8ebc37cc91bede28fa9113afc78aadb0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 18 Oct 2022 16:42:25 +0100
Subject: [PATCH 25/46] Updating export/import to include the budibase client
 and manifest.

---
 packages/client/yarn.lock                     |  5 +++
 packages/server/src/sdk/app/backups/backup.ts | 15 +++++--
 .../server/src/sdk/app/backups/exports.ts     | 37 ++++++++----------
 .../server/src/sdk/app/backups/imports.ts     | 39 ++++++++++++++-----
 4 files changed, 63 insertions(+), 33 deletions(-)

diff --git a/packages/client/yarn.lock b/packages/client/yarn.lock
index 4ed5b395fe..fbe688c0b7 100644
--- a/packages/client/yarn.lock
+++ b/packages/client/yarn.lock
@@ -670,6 +670,11 @@ has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+html5-qrcode@^2.2.1:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.2.3.tgz#5acb826860365e7c7ab91e1e14528ea16a502e8a"
+  integrity sha512-9CtEz5FVT56T76entiQxyrASzBWl8Rm30NHiQH8T163Eml5LS14BoZlYel9igxbikOt7O8KhvrT3awN1Y2HMqw==
+
 htmlparser2@^6.0.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index cb758536bd..2a85424f71 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -1,16 +1,23 @@
 import { backups } from "@budibase/pro"
 import { objectStore, tenancy } from "@budibase/backend-core"
+import { AppBackupQueueData } from "@budibase/types"
 import { exportApp } from "./exports"
 import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
 
-async function importProcessor(job: Job) {}
+async function importProcessor(job: Job) {
+  const data: AppBackupQueueData = job.data
+  const appId = data.appId,
+    backupId = data.import!.backupId
+  const { path, metadata } = await backups.downloadAppBackup(backupId)
+}
 
 async function exportProcessor(job: Job) {
-  const appId = job.data.appId,
-    trigger = job.data.trigger,
-    name = job.data.name
+  const data: AppBackupQueueData = job.data
+  const appId = data.appId,
+    trigger = data.export!.trigger,
+    name = data.export!.name
   const tenantId = tenancy.getTenantIDFromAppID(appId)
   await tenancy.doInTenant(tenantId, async () => {
     const createdAt = new Date().toISOString()
diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index 4656a83d51..4ca8c439a7 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -1,6 +1,9 @@
 import { db as dbCore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
+import {
+  retrieveDirectory,
+  retrieve,
+} from "../../../utilities/fileSystem/utilities"
 import { streamFile } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import {
@@ -8,11 +11,7 @@ import {
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../../db/utils"
-import {
-  DB_EXPORT_FILE,
-  GLOBAL_DB_EXPORT_FILE,
-  ATTACHMENT_DIR,
-} from "./constants"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import fs from "fs"
 import { join } from "path"
 const uuid = require("uuid/v4")
@@ -87,21 +86,19 @@ function defineFilter(excludeRows?: boolean) {
  */
 export async function exportApp(appId: string, config?: ExportOpts) {
   const prodAppId = dbCore.getProdAppID(appId)
-  const attachmentsPath = `${prodAppId}/${ATTACHMENT_DIR}`
-  // export attachments to tmp
-  const tmpPath = await retrieveDirectory(
-    ObjectStoreBuckets.APPS,
-    attachmentsPath
-  )
-  const downloadedPath = join(tmpPath, attachmentsPath),
-    tmpAttachmentPath = join(tmpPath, ATTACHMENT_DIR)
+  const appPath = `${prodAppId}/`
+  // export bucket contents
+  const tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath)
+  const downloadedPath = join(tmpPath, appPath)
   if (fs.existsSync(downloadedPath)) {
-    // move out of app directory, simplify structure
-    fs.renameSync(downloadedPath, tmpAttachmentPath)
+    const allFiles = fs.readdirSync(downloadedPath)
+    for (let file of allFiles) {
+      const path = join(downloadedPath, file)
+      // move out of app directory, simplify structure
+      fs.renameSync(path, join(downloadedPath, "..", file))
+    }
     // remove the old app directory created by object export
-    fs.rmdirSync(join(tmpPath, prodAppId))
-  } else {
-    fs.mkdirSync(tmpAttachmentPath)
+    fs.rmdirSync(downloadedPath)
   }
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
@@ -113,7 +110,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_DIR, DB_EXPORT_FILE])
+    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
     return tarPath
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index 60ce63d51e..b63a0ee922 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -6,9 +6,12 @@ import {
   ATTACHMENT_DIR,
   GLOBAL_DB_EXPORT_FILE,
 } from "./constants"
-import { uploadDirectory } from "../../../utilities/fileSystem/utilities"
+import {
+  uploadDirectory,
+  upload,
+} from "../../../utilities/fileSystem/utilities"
 import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
-import { join } from "path"
+import { join, basename } from "path"
 import fs from "fs"
 import sdk from "../../"
 import { CouchFindOptions, RowAttachment } from "@budibase/types"
@@ -127,14 +130,32 @@ export async function importApp(
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
     const tmpPath = isTar ? untarFile(template.file) : template.file.path
-    const attachmentPath = join(tmpPath, ATTACHMENT_DIR)
+    const contents = fs.readdirSync(tmpPath)
     // have to handle object import
-    if (fs.existsSync(attachmentPath)) {
-      await uploadDirectory(
-        ObjectStoreBuckets.APPS,
-        attachmentPath,
-        join(prodAppId, ATTACHMENT_DIR)
-      )
+    if (contents.length) {
+      let promises = []
+      let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE]
+      for (let filename of contents) {
+        const path = join(tmpPath, filename)
+        if (excludedFiles.includes(filename)) {
+          continue
+        }
+        filename = join(prodAppId, filename)
+        if (fs.lstatSync(path).isDirectory()) {
+          promises.push(
+            uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
+          )
+        } else {
+          promises.push(
+            upload({
+              bucket: ObjectStoreBuckets.APPS,
+              path,
+              filename,
+            })
+          )
+        }
+      }
+      await Promise.all(promises)
     }
     dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE))
   } else {

From b52f413aa573b410d603f555ac893eeb78fe5545 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 18 Oct 2022 17:04:18 +0100
Subject: [PATCH 26/46] Removing attachment dir constant - no longer needed.

---
 .../src/api/controllers/deploy/index.ts       | 21 ++++++++++++-------
 packages/server/src/constants/index.js        |  2 --
 packages/server/src/sdk/app/backups/backup.ts |  1 +
 .../server/src/sdk/app/backups/constants.ts   |  2 --
 .../server/src/sdk/app/backups/imports.ts     |  8 ++-----
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts
index a51e7ad6ec..a1cb905930 100644
--- a/packages/server/src/api/controllers/deploy/index.ts
+++ b/packages/server/src/api/controllers/deploy/index.ts
@@ -1,23 +1,25 @@
 import Deployment from "./Deployment"
 import {
-  Replication,
-  getProdAppID,
   getDevelopmentAppID,
+  getProdAppID,
+  Replication,
 } from "@budibase/backend-core/db"
 import { DocumentType, getAutomationParams } from "../../../db/utils"
 import {
+  clearMetadata,
   disableAllCrons,
   enableCronTrigger,
-  clearMetadata,
 } from "../../../automations/utils"
 import { app as appCache } from "@budibase/backend-core/cache"
 import {
-  getAppId,
   getAppDB,
-  getProdAppDB,
+  getAppId,
   getDevAppDB,
+  getProdAppDB,
 } from "@budibase/backend-core/context"
 import { events } from "@budibase/backend-core"
+import { backups } from "@budibase/pro"
+import { AppBackupTrigger } from "@budibase/types"
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -98,13 +100,18 @@ async function initDeployedApp(prodAppId: any) {
   console.log("Enabled cron triggers for deployed app..")
 }
 
-async function deployApp(deployment: any) {
+async function deployApp(deployment: any, userId: string) {
   let replication
   try {
     const appId = getAppId()
     const devAppId = getDevelopmentAppID(appId)
     const productionAppId = getProdAppID(appId)
 
+    // trigger backup initially
+    await backups.triggerAppBackup(productionAppId, AppBackupTrigger.PUBLISH, {
+      createdBy: userId,
+    })
+
     const config: any = {
       source: devAppId,
       target: productionAppId,
@@ -205,7 +212,7 @@ const _deployApp = async function (ctx: any) {
 
   console.log("Deploying app...")
 
-  let app = await deployApp(deployment)
+  let app = await deployApp(deployment, ctx.user._id)
 
   await events.app.published(app)
   ctx.body = deployment
diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js
index 6b5bc4bf83..a3bccae754 100644
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@@ -209,6 +209,4 @@ exports.AutomationErrors = {
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 
-exports.ATTACHMENT_DIR = "attachments"
-
 exports.MAX_AUTOMATION_RECURRING_ERRORS = 5
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 2a85424f71..92f70a5a84 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -11,6 +11,7 @@ async function importProcessor(job: Job) {
   const appId = data.appId,
     backupId = data.import!.backupId
   const { path, metadata } = await backups.downloadAppBackup(backupId)
+  // start by removing app database and contents of bucket - which will be updated
 }
 
 async function exportProcessor(job: Job) {
diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts
index f022168846..2f011ea2de 100644
--- a/packages/server/src/sdk/app/backups/constants.ts
+++ b/packages/server/src/sdk/app/backups/constants.ts
@@ -1,4 +1,2 @@
-import { ATTACHMENT_DIR as attachmentDir } from "../../../constants"
 export const DB_EXPORT_FILE = "db.txt"
-export const ATTACHMENT_DIR = attachmentDir
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index b63a0ee922..b29a9eede5 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -1,17 +1,13 @@
 import { db as dbCore } from "@budibase/backend-core"
 import { TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import {
-  DB_EXPORT_FILE,
-  ATTACHMENT_DIR,
-  GLOBAL_DB_EXPORT_FILE,
-} from "./constants"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import {
   uploadDirectory,
   upload,
 } from "../../../utilities/fileSystem/utilities"
 import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
-import { join, basename } from "path"
+import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
 import { CouchFindOptions, RowAttachment } from "@budibase/types"

From bdc4e29b2da5d06d75ced004654c7422ffe6a8fc Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 18 Oct 2022 19:43:19 +0100
Subject: [PATCH 27/46] Finishing import processor - download backup, delete
 dev DB and then import over the top of this. Also includes a rollback feature
 if the backup fails to restore for whatever reason.

---
 packages/server/src/sdk/app/backups/backup.ts | 37 +++++++++++++++++--
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 92f70a5a84..cc2338e274 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -1,17 +1,48 @@
 import { backups } from "@budibase/pro"
-import { objectStore, tenancy } from "@budibase/backend-core"
+import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core"
 import { AppBackupQueueData } from "@budibase/types"
 import { exportApp } from "./exports"
+import { importApp } from "./imports"
 import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
 
+async function removeExistingApp(devId: string) {
+  const devDb = dbCore.dangerousGetDB(devId, { skip_setup: true })
+  await devDb.destroy()
+}
+
 async function importProcessor(job: Job) {
   const data: AppBackupQueueData = job.data
   const appId = data.appId,
     backupId = data.import!.backupId
-  const { path, metadata } = await backups.downloadAppBackup(backupId)
-  // start by removing app database and contents of bucket - which will be updated
+  const tenantId = tenancy.getTenantIDFromAppID(appId)
+  tenancy.doInTenant(tenantId, async () => {
+    const devAppId = dbCore.getDevAppID(appId)
+    const performImport = async (path: string) => {
+      await importApp(devAppId, dbCore.dangerousGetDB(devAppId), {
+        file: {
+          type: "application/gzip",
+          path,
+        },
+        key: path,
+      })
+    }
+    // initially export the current state to disk - incase something goes wrong
+    const backupTarPath = await exportApp(devAppId, { tar: true })
+    // get the backup ready on disk
+    const { path } = await backups.downloadAppBackup(backupId)
+    // start by removing app database and contents of bucket - which will be updated
+    await removeExistingApp(devAppId)
+    try {
+      await performImport(path)
+    } catch (err) {
+      // rollback - clear up failed import and re-import the pre-backup
+      await removeExistingApp(devAppId)
+      await performImport(backupTarPath)
+    }
+    fs.rmSync(backupTarPath)
+  })
 }
 
 async function exportProcessor(job: Job) {

From f5845a8a6e026689bb469103e3f657cbc0e8591e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 19 Oct 2022 11:47:07 +0100
Subject: [PATCH 28/46] Protection for real app IDs.

---
 packages/server/src/sdk/app/backups/exports.ts | 5 +----
 packages/types/src/sdk/koa.ts                  | 1 +
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index 4ca8c439a7..26277fcd66 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -1,9 +1,6 @@
 import { db as dbCore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import {
-  retrieveDirectory,
-  retrieve,
-} from "../../../utilities/fileSystem/utilities"
+import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
 import { streamFile } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import {
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index e6f4a5094e..1b48d95015 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -16,4 +16,5 @@ export interface BBContext {
   body?: any
   redirect?: any
   attachment: any
+  throw: any
 }

From 59bf052b70cb0d6db2709f1f97de98f03961730d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 19 Oct 2022 13:52:56 +0100
Subject: [PATCH 29/46] Fixing issue with metadata name not being provided for
 uploaded backup on publish.

---
 packages/server/src/sdk/app/backups/backup.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index cc2338e274..0f2a1aa760 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -49,7 +49,7 @@ async function exportProcessor(job: Job) {
   const data: AppBackupQueueData = job.data
   const appId = data.appId,
     trigger = data.export!.trigger,
-    name = data.export!.name
+    name = data.export!.name || `${trigger} - backup`
   const tenantId = tenancy.getTenantIDFromAppID(appId)
   await tenancy.doInTenant(tenantId, async () => {
     const createdAt = new Date().toISOString()

From 113a23ba3d1440d30e433b5c36c0d9e9ecd379b6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 19 Oct 2022 16:48:32 +0100
Subject: [PATCH 30/46] Adding statistics output for backups.

---
 .../src/events/publishers/backup.ts           |  4 +-
 packages/server/src/sdk/app/backups/backup.ts | 22 ++++--
 .../server/src/sdk/app/statistics/index.ts    | 77 +++++++++++++++++++
 packages/types/src/documents/app/backup.ts    | 26 +++----
 4 files changed, 105 insertions(+), 24 deletions(-)
 create mode 100644 packages/server/src/sdk/app/statistics/index.ts

diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts
index 00b4f8db69..bd346cad64 100644
--- a/packages/backend-core/src/events/publishers/backup.ts
+++ b/packages/backend-core/src/events/publishers/backup.ts
@@ -4,8 +4,8 @@ import { publishEvent } from "../events"
 export async function appBackupRestored(backup: AppBackup) {
   const properties: AppBackupRevertEvent = {
     appId: backup.appId,
-    backupName: backup.name,
-    backupCreatedAt: backup.createdAt,
+    backupName: backup.name!,
+    backupCreatedAt: backup.timestamp,
   }
 
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 0f2a1aa760..6f7906d1cc 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -3,6 +3,7 @@ import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core"
 import { AppBackupQueueData } from "@budibase/types"
 import { exportApp } from "./exports"
 import { importApp } from "./imports"
+import { calculateBackupStats } from "../statistics"
 import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
@@ -52,26 +53,35 @@ async function exportProcessor(job: Job) {
     name = data.export!.name || `${trigger} - backup`
   const tenantId = tenancy.getTenantIDFromAppID(appId)
   await tenancy.doInTenant(tenantId, async () => {
-    const createdAt = new Date().toISOString()
-    const tarPath = await exportApp(appId, { tar: true })
-    let filename = `${appId}/backup-${createdAt}.tar.gz`
+    const devAppId = dbCore.getDevAppID(appId),
+      prodAppId = dbCore.getProdAppID(appId)
+    const timestamp = new Date().toISOString()
+    const tarPath = await exportApp(devAppId, { tar: true })
+    const contents = await calculateBackupStats(devAppId)
+    let filename = `${prodAppId}/backup-${timestamp}.tar.gz`
     // add the tenant to the bucket path if backing up within a multi-tenant environment
     if (env.MULTI_TENANCY) {
       filename = `${tenantId}/${filename}`
     }
     const bucket = objectStore.ObjectStoreBuckets.BACKUPS
     const metadata = {
-      appId,
-      createdAt,
+      appId: prodAppId,
+      timestamp,
       trigger,
       name,
+      contents,
     }
     await objectStore.upload({
       path: tarPath,
       type: "application/gzip",
       bucket,
       filename,
-      metadata,
+      metadata: {
+        name,
+        trigger,
+        timestamp,
+        appId: prodAppId,
+      },
     })
     await backups.storeAppBackupMetadata(filename, metadata)
     // clear up the tarball after uploading it
diff --git a/packages/server/src/sdk/app/statistics/index.ts b/packages/server/src/sdk/app/statistics/index.ts
new file mode 100644
index 0000000000..3f03158264
--- /dev/null
+++ b/packages/server/src/sdk/app/statistics/index.ts
@@ -0,0 +1,77 @@
+import { context, db as dbCore } from "@budibase/backend-core"
+import {
+  getDatasourceParams,
+  getTableParams,
+  getAutomationParams,
+  getScreenParams,
+} from "../../../db/utils"
+
+async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
+  if (db) {
+    return cb(db)
+  } else {
+    const devAppId = dbCore.getDevAppID(appId)
+    return context.doInAppContext(devAppId, () => {
+      const db = context.getAppDB()
+      return cb(db)
+    })
+  }
+}
+
+export async function calculateDatasourceCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const datasourceList = await db.allDocs(getDatasourceParams())
+      const tableList = await db.allDocs(getTableParams())
+      return datasourceList.rows.length + tableList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateAutomationCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const automationList = await db.allDocs(getAutomationParams())
+      return automationList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateScreenCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const screenList = await db.allDocs(getScreenParams())
+      return screenList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateBackupStats(appId: string) {
+  return runInContext(appId, async (db: PouchDB.Database) => {
+    const promises = []
+    promises.push(calculateDatasourceCount(appId, db))
+    promises.push(calculateAutomationCount(appId, db))
+    promises.push(calculateScreenCount(appId, db))
+    const responses = await Promise.all(promises)
+    return {
+      datasources: responses[0],
+      automations: responses[1],
+      screens: responses[2],
+    }
+  })
+}
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index 28e927c772..a62b603a96 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -11,21 +11,23 @@ export enum AppBackupEventType {
   IMPORT = "import",
 }
 
-export interface AppBackup extends Document {
-  trigger: AppBackupTrigger
-  name: string
-  createdAt: string
-  createdBy?: string
-  filename: string
+export interface AppBackupMetadata {
   appId: string
-  userId?: string
-  contents?: {
+  trigger: AppBackupTrigger
+  name?: string
+  createdBy?: string
+  timestamp: string
+  contents: {
     datasources: string[]
     screens: string[]
     automations: string[]
   }
 }
 
+export interface AppBackup extends Document, AppBackupMetadata {
+  filename: string
+}
+
 export type AppBackupFetchOpts = {
   trigger?: AppBackupTrigger
   limit?: number
@@ -47,11 +49,3 @@ export interface AppBackupQueueData {
     backupId: string
   }
 }
-
-export interface AppBackupMetadata {
-  appId: string
-  trigger: AppBackupTrigger
-  name?: string
-  createdBy?: string
-  createdAt: string
-}

From 10c81c1adcdae34091ce4d6cb6c546f007735480 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 19 Oct 2022 16:57:14 +0100
Subject: [PATCH 31/46] Fixing issue with using templates.

---
 packages/server/src/sdk/app/backups/imports.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index b29a9eede5..aaab85ec92 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -6,6 +6,7 @@ import {
   uploadDirectory,
   upload,
 } from "../../../utilities/fileSystem/utilities"
+import { downloadTemplate } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
@@ -89,7 +90,7 @@ async function getTemplateStream(template: TemplateType) {
     return fs.createReadStream(template.file.path)
   } else if (template.key) {
     const [type, name] = template.key.split("/")
-    const tmpPath = await exports.downloadTemplate(type, name)
+    const tmpPath = await downloadTemplate(type, name)
     return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
   }
 }

From f11c0bbfd777e16420d205fc1b73fb998d0670c1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 19 Oct 2022 19:19:36 +0100
Subject: [PATCH 32/46] Some type updates and processor handling for backup
 document being created before backup/restore occurs.

---
 packages/server/src/sdk/app/backups/backup.ts | 24 +++++++------
 packages/types/src/documents/app/backup.ts    | 34 +++++++++++++------
 2 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 6f7906d1cc..97fe12e63d 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -1,6 +1,6 @@
 import { backups } from "@budibase/pro"
-import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core"
-import { AppBackupQueueData } from "@budibase/types"
+import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core"
+import { AppBackupQueueData, AppBackupStatus } from "@budibase/types"
 import { exportApp } from "./exports"
 import { importApp } from "./imports"
 import { calculateBackupStats } from "../statistics"
@@ -42,6 +42,11 @@ async function importProcessor(job: Job) {
       await removeExistingApp(devAppId)
       await performImport(backupTarPath)
     }
+    await backups.updateRestoreStatus(
+      data.docId,
+      data.docRev,
+      AppBackupStatus.COMPLETE
+    )
     fs.rmSync(backupTarPath)
   })
 }
@@ -64,13 +69,6 @@ async function exportProcessor(job: Job) {
       filename = `${tenantId}/${filename}`
     }
     const bucket = objectStore.ObjectStoreBuckets.BACKUPS
-    const metadata = {
-      appId: prodAppId,
-      timestamp,
-      trigger,
-      name,
-      contents,
-    }
     await objectStore.upload({
       path: tarPath,
       type: "application/gzip",
@@ -83,7 +81,13 @@ async function exportProcessor(job: Job) {
         appId: prodAppId,
       },
     })
-    await backups.storeAppBackupMetadata(filename, metadata)
+    await backups.updateBackupStatus(
+      data.docId,
+      data.docRev,
+      AppBackupStatus.COMPLETE,
+      contents,
+      filename
+    )
     // clear up the tarball after uploading it
     fs.rmSync(tarPath)
   })
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index a62b603a96..a6d75efcff 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -1,31 +1,41 @@
 import { Document } from "../document"
 
+export enum AppBackupType {
+  BACKUP = "backup",
+  RESTORE = "restore",
+}
+
+export enum AppBackupStatus {
+  STARTED = "started",
+  COMPLETE = "complete",
+  FAILED = "failed",
+}
+
 export enum AppBackupTrigger {
   PUBLISH = "publish",
   MANUAL = "manual",
   SCHEDULED = "scheduled",
 }
 
-export enum AppBackupEventType {
-  EXPORT = "export",
-  IMPORT = "import",
+export interface AppBackupContents {
+  datasources: string[]
+  screens: string[]
+  automations: string[]
 }
 
 export interface AppBackupMetadata {
   appId: string
-  trigger: AppBackupTrigger
+  trigger?: AppBackupTrigger
+  type: AppBackupType
+  status: AppBackupStatus
   name?: string
   createdBy?: string
   timestamp: string
-  contents: {
-    datasources: string[]
-    screens: string[]
-    automations: string[]
-  }
+  contents?: AppBackupContents
 }
 
 export interface AppBackup extends Document, AppBackupMetadata {
-  filename: string
+  filename?: string
 }
 
 export type AppBackupFetchOpts = {
@@ -38,8 +48,9 @@ export type AppBackupFetchOpts = {
 }
 
 export interface AppBackupQueueData {
-  eventType: AppBackupEventType
   appId: string
+  docId: string
+  docRev: string
   export?: {
     trigger: AppBackupTrigger
     name?: string
@@ -47,5 +58,6 @@ export interface AppBackupQueueData {
   }
   import?: {
     backupId: string
+    createdBy?: string
   }
 }

From f5557fd8059fd613618fd96bce125d72eee89224 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 20 Oct 2022 15:05:50 +0100
Subject: [PATCH 33/46] Fetching a list of backup/restore events includes the
 full user object.

---
 packages/backend-core/src/db/constants.ts     |  19 ++-
 packages/backend-core/src/db/utils.ts         |  74 ++++++++++-
 .../server/src/api/controllers/table/utils.ts |  13 +-
 packages/server/src/db/utils.js               | 125 +++---------------
 packages/types/src/documents/app/backup.ts    |   3 +-
 5 files changed, 108 insertions(+), 126 deletions(-)

diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts
index 8f8f45638b..446f1f7d01 100644
--- a/packages/backend-core/src/db/constants.ts
+++ b/packages/backend-core/src/db/constants.ts
@@ -31,6 +31,10 @@ export const DeprecatedViews = {
   ],
 }
 
+export enum InternalTable {
+  USER_METADATA = "ta_users",
+}
+
 export enum DocumentType {
   USER = "us",
   GROUP = "gr",
@@ -47,10 +51,23 @@ export enum DocumentType {
   AUTOMATION_LOG = "log_au",
   ACCOUNT_METADATA = "acc_metadata",
   PLUGIN = "plg",
-  TABLE = "ta",
   DATASOURCE = "datasource",
   DATASOURCE_PLUS = "datasource_plus",
   APP_BACKUP = "backup",
+  TABLE = "ta",
+  ROW = "ro",
+  AUTOMATION = "au",
+  LINK = "li",
+  WEBHOOK = "wh",
+  INSTANCE = "inst",
+  LAYOUT = "layout",
+  SCREEN = "screen",
+  QUERY = "query",
+  DEPLOYMENTS = "deployments",
+  METADATA = "metadata",
+  MEM_VIEW = "view",
+  USER_FLAG = "flag",
+  AUTOMATION_METADATA = "meta_au",
 }
 
 export const StaticDatabases = {
diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts
index 8d824d60bb..c04da5da4f 100644
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@@ -1,10 +1,16 @@
 import { newid } from "../hashing"
 import { DEFAULT_TENANT_ID, Configs } from "../constants"
 import env from "../environment"
-import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants"
+import {
+  SEPARATOR,
+  DocumentType,
+  UNICODE_MAX,
+  ViewName,
+  InternalTable,
+} from "./constants"
 import { getTenantId, getGlobalDB } from "../context"
 import { getGlobalDBName } from "./tenancy"
-import { doWithDB, allDbs, directCouchQuery, directCouchAllDbs } from "./index"
+import { doWithDB, allDbs, directCouchAllDbs } from "./index"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
@@ -40,8 +46,8 @@ export const generateAppID = (tenantId = null) => {
  * @returns {object} Parameters which can then be used with an allDocs request.
  */
 export function getDocParams(
-  docType: any,
-  docId: any = null,
+  docType: string,
+  docId?: string | null,
   otherProps: any = {}
 ) {
   if (docId == null) {
@@ -54,6 +60,28 @@ export function getDocParams(
   }
 }
 
+/**
+ * Gets the DB allDocs/query params for retrieving a row.
+ * @param {string|null} tableId The table in which the rows have been stored.
+ * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
+ * left null to get all the rows in the table.
+ * @param {object} otherProps Any other properties to add to the request.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+export function getRowParams(
+  tableId?: string | null,
+  rowId?: string | null,
+  otherProps = {}
+) {
+  if (tableId == null) {
+    return getDocParams(DocumentType.ROW, null, otherProps)
+  }
+
+  const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId
+
+  return getDocParams(DocumentType.ROW, endOfKey, otherProps)
+}
+
 /**
  * Retrieve the correct index for a view based on default design DB.
  */
@@ -61,6 +89,17 @@ export function getQueryIndex(viewName: ViewName) {
   return `database/${viewName}`
 }
 
+/**
+ * Gets a new row ID for the specified table.
+ * @param {string} tableId The table which the row is being created for.
+ * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
+ * @returns {string} The new ID which a row doc can be stored under.
+ */
+export function generateRowID(tableId: string, id?: string) {
+  id = id || newid()
+  return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}`
+}
+
 /**
  * Check if a given ID is that of a table.
  * @returns {boolean}
@@ -128,6 +167,33 @@ export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
   }
 }
 
+/**
+ * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
+ */
+export function getUserMetadataParams(userId?: string, otherProps = {}) {
+  return getRowParams(InternalTable.USER_METADATA, userId, otherProps)
+}
+
+/**
+ * Generates a new user ID based on the passed in global ID.
+ * @param {string} globalId The ID of the global user.
+ * @returns {string} The new user ID which the user doc can be stored under.
+ */
+export function generateUserMetadataID(globalId: string) {
+  return generateRowID(InternalTable.USER_METADATA, globalId)
+}
+
+/**
+ * Breaks up the ID to get the global ID.
+ */
+export function getGlobalIDFromUserMetadataID(id: string) {
+  const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+  if (!id || !id.includes(prefix)) {
+    return id
+  }
+  return id.split(prefix)[1]
+}
+
 export function getUsersByAppParams(appId: any, otherProps: any = {}) {
   const prodAppId = getProdAppID(appId)
   return {
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index 79c551b63a..6d17d12f3d 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -1,11 +1,5 @@
 import { transform } from "../../../utilities/csvParser"
-import {
-  getRowParams,
-  generateRowID,
-  InternalTables,
-  getTableParams,
-  BudibaseInternalDB,
-} from "../../../db/utils"
+import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
 import { isEqual } from "lodash"
 import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import {
@@ -17,11 +11,6 @@ import {
   SwitchableTypes,
   CanSwitchTypes,
 } from "../../../constants"
-import {
-  isExternalTable,
-  breakExternalTableId,
-  isSQL,
-} from "../../../integrations/utils"
 import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 const { getAppDB } = require("@budibase/backend-core/context")
diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js
index 046172df73..acd72cbf66 100644
--- a/packages/server/src/db/utils.js
+++ b/packages/server/src/db/utils.js
@@ -1,6 +1,7 @@
 const newid = require("./newid")
 const {
-  DocumentType: CoreDocTypes,
+  DocumentType: CoreDocType,
+  InternalTable,
   getRoleParams,
   generateRoleID,
   APP_DEV_PREFIX,
@@ -13,6 +14,12 @@ const {
   generateAppID,
   getQueryIndex,
   ViewName,
+  getDocParams,
+  getRowParams,
+  generateRowID,
+  getUserMetadataParams,
+  generateUserMetadataID,
+  getGlobalIDFromUserMetadataID,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -23,28 +30,7 @@ const AppStatus = {
   DEPLOYED: "published",
 }
 
-const DocumentType = {
-  ...CoreDocTypes,
-  TABLE: "ta",
-  ROW: "ro",
-  USER: "us",
-  AUTOMATION: "au",
-  LINK: "li",
-  WEBHOOK: "wh",
-  INSTANCE: "inst",
-  LAYOUT: "layout",
-  SCREEN: "screen",
-  QUERY: "query",
-  DEPLOYMENTS: "deployments",
-  METADATA: "metadata",
-  MEM_VIEW: "view",
-  USER_FLAG: "flag",
-  AUTOMATION_METADATA: "meta_au",
-}
-
-const InternalTables = {
-  USER_METADATA: "ta_users",
-}
+const DocumentType = CoreDocType
 
 const SearchIndexes = {
   ROWS: "rows",
@@ -64,11 +50,11 @@ exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
 exports.isDevAppID = isDevAppID
 exports.isProdAppID = isProdAppID
-exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
 exports.TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 exports.ViewName = ViewName
-exports.InternalTables = InternalTables
+exports.InternalTables = InternalTable
 exports.DocumentType = DocumentType
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
@@ -77,36 +63,15 @@ exports.AppStatus = AppStatus
 exports.BudibaseInternalDB = BudibaseInternalDB
 exports.generateAppID = generateAppID
 exports.generateDevAppID = getDevelopmentAppID
-
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
-
 exports.getQueryIndex = getQueryIndex
-
-/**
- * If creating DB allDocs/query params with only a single top level ID this can be used, this
- * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
- * More complex cases such as link docs and rows which have multiple levels of IDs that their
- * ID consists of need their own functions to build the allDocs parameters.
- * @param {string} docType The type of document which input params are being built for, e.g. user,
- * link, app, table and so on.
- * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
- * for a singular document.
- * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
- * @returns {object} Parameters which can then be used with an allDocs request.
- */
-function getDocParams(docType, docId = null, otherProps = {}) {
-  if (docId == null) {
-    docId = ""
-  }
-  return {
-    ...otherProps,
-    startkey: `${docType}${SEPARATOR}${docId}`,
-    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
-  }
-}
-
 exports.getDocParams = getDocParams
+exports.getRowParams = getRowParams
+exports.generateRowID = generateRowID
+exports.getUserMetadataParams = getUserMetadataParams
+exports.generateUserMetadataID = generateUserMetadataID
+exports.getGlobalIDFromUserMetadataID = getGlobalIDFromUserMetadataID
 
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
@@ -123,24 +88,6 @@ exports.generateTableID = () => {
   return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
 }
 
-/**
- * Gets the DB allDocs/query params for retrieving a row.
- * @param {string|null} tableId The table in which the rows have been stored.
- * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
- * left null to get all the rows in the table.
- * @param {object} otherProps Any other properties to add to the request.
- * @returns {object} Parameters which can then be used with an allDocs request.
- */
-exports.getRowParams = (tableId = null, rowId = null, otherProps = {}) => {
-  if (tableId == null) {
-    return getDocParams(DocumentType.ROW, null, otherProps)
-  }
-
-  const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId
-
-  return getDocParams(DocumentType.ROW, endOfKey, otherProps)
-}
-
 /**
  * Given a row ID this will find the table ID within it (only works for internal tables).
  * @param {string} rowId The ID of the row.
@@ -153,44 +100,6 @@ exports.getTableIDFromRowID = rowId => {
   return `${DocumentType.TABLE}${SEPARATOR}${components[0]}`
 }
 
-/**
- * Gets a new row ID for the specified table.
- * @param {string} tableId The table which the row is being created for.
- * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
- * @returns {string} The new ID which a row doc can be stored under.
- */
-exports.generateRowID = (tableId, id = null) => {
-  id = id || newid()
-  return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}`
-}
-
-/**
- * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
- */
-exports.getUserMetadataParams = (userId = null, otherProps = {}) => {
-  return exports.getRowParams(InternalTables.USER_METADATA, userId, otherProps)
-}
-
-/**
- * Generates a new user ID based on the passed in global ID.
- * @param {string} globalId The ID of the global user.
- * @returns {string} The new user ID which the user doc can be stored under.
- */
-exports.generateUserMetadataID = globalId => {
-  return exports.generateRowID(InternalTables.USER_METADATA, globalId)
-}
-
-/**
- * Breaks up the ID to get the global ID.
- */
-exports.getGlobalIDFromUserMetadataID = id => {
-  const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-  if (!id || !id.includes(prefix)) {
-    return id
-  }
-  return id.split(prefix)[1]
-}
-
 /**
  * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
  */
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index a6d75efcff..520013a8bb 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -1,4 +1,5 @@
 import { Document } from "../document"
+import { User } from "../../"
 
 export enum AppBackupType {
   BACKUP = "backup",
@@ -29,7 +30,7 @@ export interface AppBackupMetadata {
   type: AppBackupType
   status: AppBackupStatus
   name?: string
-  createdBy?: string
+  createdBy?: string | User
   timestamp: string
   contents?: AppBackupContents
 }

From 854cb23947f96ae55d1f3395b44e9c1d12b14f73 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 20 Oct 2022 19:07:10 +0100
Subject: [PATCH 34/46] Getting the import system to carry out a backup of the
 app before the restore.

---
 packages/server/src/sdk/app/backups/backup.ts | 145 +++++++++++-------
 packages/types/src/documents/app/backup.ts    |   2 +
 2 files changed, 93 insertions(+), 54 deletions(-)

diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 97fe12e63d..2741e41096 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -1,6 +1,11 @@
 import { backups } from "@budibase/pro"
 import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core"
-import { AppBackupQueueData, AppBackupStatus } from "@budibase/types"
+import {
+  AppBackupQueueData,
+  AppBackupStatus,
+  AppBackupTrigger,
+  AppBackupType,
+} from "@budibase/types"
 import { exportApp } from "./exports"
 import { importApp } from "./imports"
 import { calculateBackupStats } from "../statistics"
@@ -8,19 +13,96 @@ import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
 
+type BackupOpts = {
+  doc?: { id: string; rev: string }
+  createdBy?: string
+}
+
 async function removeExistingApp(devId: string) {
   const devDb = dbCore.dangerousGetDB(devId, { skip_setup: true })
   await devDb.destroy()
 }
 
+async function runBackup(
+  name: string,
+  trigger: AppBackupTrigger,
+  tenantId: string,
+  appId: string,
+  opts?: BackupOpts
+) {
+  const devAppId = dbCore.getDevAppID(appId),
+    prodAppId = dbCore.getProdAppID(appId)
+  const timestamp = new Date().toISOString()
+  const tarPath = await exportApp(devAppId, { tar: true })
+  const contents = await calculateBackupStats(devAppId)
+  let filename = `${prodAppId}/backup-${timestamp}.tar.gz`
+  // add the tenant to the bucket path if backing up within a multi-tenant environment
+  if (env.MULTI_TENANCY) {
+    filename = `${tenantId}/${filename}`
+  }
+  const bucket = objectStore.ObjectStoreBuckets.BACKUPS
+  await objectStore.upload({
+    path: tarPath,
+    type: "application/gzip",
+    bucket,
+    filename,
+    metadata: {
+      name,
+      trigger,
+      timestamp,
+      appId: prodAppId,
+    },
+  })
+  if (opts?.doc) {
+    await backups.updateBackupStatus(
+      opts.doc.id,
+      opts.doc.rev,
+      AppBackupStatus.COMPLETE,
+      contents,
+      filename
+    )
+  } else {
+    await backups.storeAppBackupMetadata(
+      {
+        appId: prodAppId,
+        timestamp,
+        name,
+        trigger,
+        type: AppBackupType.BACKUP,
+        status: AppBackupStatus.COMPLETE,
+        contents,
+        createdBy: opts?.createdBy,
+      },
+      { filename }
+    )
+  }
+  // clear up the tarball after uploading it
+  fs.rmSync(tarPath)
+}
+
 async function importProcessor(job: Job) {
   const data: AppBackupQueueData = job.data
   const appId = data.appId,
-    backupId = data.import!.backupId
-  const tenantId = tenancy.getTenantIDFromAppID(appId)
+    backupId = data.import!.backupId,
+    nameForBackup = data.import!.nameForBackup,
+    createdBy = data.import!.createdBy
+  const tenantId = tenancy.getTenantIDFromAppID(appId) as string
   tenancy.doInTenant(tenantId, async () => {
     const devAppId = dbCore.getDevAppID(appId)
-    const performImport = async (path: string) => {
+    // initially export the current state to disk - incase something goes wrong
+    await runBackup(
+      nameForBackup,
+      AppBackupTrigger.RESTORING,
+      tenantId,
+      appId,
+      { createdBy }
+    )
+    // get the backup ready on disk
+    const { path } = await backups.downloadAppBackup(backupId)
+    // start by removing app database and contents of bucket - which will be updated
+    await removeExistingApp(devAppId)
+    let status = AppBackupStatus.COMPLETE
+    try {
       await importApp(devAppId, dbCore.dangerousGetDB(devAppId), {
         file: {
           type: "application/gzip",
@@ -28,26 +110,10 @@ async function importProcessor(job: Job) {
         },
         key: path,
       })
-    }
-    // initially export the current state to disk - incase something goes wrong
-    const backupTarPath = await exportApp(devAppId, { tar: true })
-    // get the backup ready on disk
-    const { path } = await backups.downloadAppBackup(backupId)
-    // start by removing app database and contents of bucket - which will be updated
-    await removeExistingApp(devAppId)
-    try {
-      await performImport(path)
     } catch (err) {
-      // rollback - clear up failed import and re-import the pre-backup
-      await removeExistingApp(devAppId)
-      await performImport(backupTarPath)
+      status = AppBackupStatus.FAILED
     }
-    await backups.updateRestoreStatus(
-      data.docId,
-      data.docRev,
-      AppBackupStatus.COMPLETE
-    )
-    fs.rmSync(backupTarPath)
+    await backups.updateRestoreStatus(data.docId, data.docRev, status)
   })
 }
 
@@ -56,40 +122,11 @@ async function exportProcessor(job: Job) {
   const appId = data.appId,
     trigger = data.export!.trigger,
     name = data.export!.name || `${trigger} - backup`
-  const tenantId = tenancy.getTenantIDFromAppID(appId)
+  const tenantId = tenancy.getTenantIDFromAppID(appId) as string
   await tenancy.doInTenant(tenantId, async () => {
-    const devAppId = dbCore.getDevAppID(appId),
-      prodAppId = dbCore.getProdAppID(appId)
-    const timestamp = new Date().toISOString()
-    const tarPath = await exportApp(devAppId, { tar: true })
-    const contents = await calculateBackupStats(devAppId)
-    let filename = `${prodAppId}/backup-${timestamp}.tar.gz`
-    // add the tenant to the bucket path if backing up within a multi-tenant environment
-    if (env.MULTI_TENANCY) {
-      filename = `${tenantId}/${filename}`
-    }
-    const bucket = objectStore.ObjectStoreBuckets.BACKUPS
-    await objectStore.upload({
-      path: tarPath,
-      type: "application/gzip",
-      bucket,
-      filename,
-      metadata: {
-        name,
-        trigger,
-        timestamp,
-        appId: prodAppId,
-      },
+    return runBackup(name, trigger, tenantId, appId, {
+      doc: { id: data.docId, rev: data.docRev },
     })
-    await backups.updateBackupStatus(
-      data.docId,
-      data.docRev,
-      AppBackupStatus.COMPLETE,
-      contents,
-      filename
-    )
-    // clear up the tarball after uploading it
-    fs.rmSync(tarPath)
   })
 }
 
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index 520013a8bb..c9671e8269 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -16,6 +16,7 @@ export enum AppBackupTrigger {
   PUBLISH = "publish",
   MANUAL = "manual",
   SCHEDULED = "scheduled",
+  RESTORING = "restoring",
 }
 
 export interface AppBackupContents {
@@ -59,6 +60,7 @@ export interface AppBackupQueueData {
   }
   import?: {
     backupId: string
+    nameForBackup: string
     createdBy?: string
   }
 }

From 2a2f41a8615f1143af1012790c04175ba1c2a45a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 21 Oct 2022 16:02:13 +0100
Subject: [PATCH 35/46] Fixing issue discovered by tests.

---
 packages/backend-core/src/queue/queue.ts         |  2 +-
 .../server/src/api/controllers/deploy/index.ts   | 16 ++++++++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index 6dd10ee091..7eaafe38a8 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -22,7 +22,7 @@ export function createQueue<T>(
 ): BullQueue.Queue<T> {
   const queueConfig: any = redisProtocolUrl || { redis: opts }
   let queue: any
-  if (env.isTest()) {
+  if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
   } else {
     queue = new InMemoryQueue(jobQueue, queueConfig)
diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts
index a1cb905930..b3b875e397 100644
--- a/packages/server/src/api/controllers/deploy/index.ts
+++ b/packages/server/src/api/controllers/deploy/index.ts
@@ -20,6 +20,7 @@ import {
 import { events } from "@budibase/backend-core"
 import { backups } from "@budibase/pro"
 import { AppBackupTrigger } from "@budibase/types"
+import env from "../../../environment"
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -107,10 +108,17 @@ async function deployApp(deployment: any, userId: string) {
     const devAppId = getDevelopmentAppID(appId)
     const productionAppId = getProdAppID(appId)
 
-    // trigger backup initially
-    await backups.triggerAppBackup(productionAppId, AppBackupTrigger.PUBLISH, {
-      createdBy: userId,
-    })
+    // can't do this in test
+    if (!env.isTest()) {
+      // trigger backup initially
+      await backups.triggerAppBackup(
+        productionAppId,
+        AppBackupTrigger.PUBLISH,
+        {
+          createdBy: userId,
+        }
+      )
+    }
 
     const config: any = {
       source: devAppId,

From 5aeeed47e1bf28ab4eb151d7c996cc5da3ca3a02 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 21 Oct 2022 17:04:20 +0100
Subject: [PATCH 36/46] Type updates for searching by type + trigger on app
 backup documents.

---
 packages/types/src/api/web/app/backup.ts   | 3 ++-
 packages/types/src/documents/app/backup.ts | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts
index 092cf714b5..f16f8416ae 100644
--- a/packages/types/src/api/web/app/backup.ts
+++ b/packages/types/src/api/web/app/backup.ts
@@ -1,7 +1,8 @@
-import { AppBackupTrigger } from "../../../documents"
+import { AppBackupTrigger, AppBackupType } from "../../../documents"
 
 export interface SearchAppBackupsRequest {
   trigger: AppBackupTrigger
+  type: AppBackupType
   startDate: string
   endDate: string
   page?: string
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index c9671e8269..f7b6239b4f 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -42,6 +42,7 @@ export interface AppBackup extends Document, AppBackupMetadata {
 
 export type AppBackupFetchOpts = {
   trigger?: AppBackupTrigger
+  type?: AppBackupType
   limit?: number
   page?: string
   paginate?: boolean

From fe5d11232f0b151b14c8488b8cbad384e05a2290 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 21 Oct 2022 18:25:35 +0100
Subject: [PATCH 37/46] Update with PR comments.

---
 packages/backend-core/package.json            |   5 +-
 packages/server/package.json                  |   4 +-
 packages/server/src/sdk/app/backups/backup.ts |  14 +-
 packages/server/yarn.lock                     | 604 +++++++++++++++++-
 packages/types/src/documents/app/backup.ts    |   1 +
 packages/types/src/sdk/koa.ts                 |   1 +
 6 files changed, 594 insertions(+), 35 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index a3bb329b72..69d4968e81 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -26,7 +26,7 @@
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",
     "bcryptjs": "2.4.3",
-    "bull": "^4.10.1",
+    "bull": "4.10.1",
     "dotenv": "16.0.1",
     "emitter-listener": "1.1.2",
     "ioredis": "4.28.0",
@@ -63,9 +63,8 @@
     ]
   },
   "devDependencies": {
-    "@types/bull": "^3.15.9",
     "@types/chance": "1.1.3",
-    "@types/ioredis": "^4.28.10",
+    "@types/ioredis": "4.28.10",
     "@types/jest": "27.5.1",
     "@types/koa": "2.0.52",
     "@types/lodash": "4.14.180",
diff --git a/packages/server/package.json b/packages/server/package.json
index 539dc46ed9..42eb3b2df1 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -93,7 +93,7 @@
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
     "bcryptjs": "2.4.3",
-    "bull": "4.8.5",
+    "bull": "4.10.1",
     "chmodr": "1.2.0",
     "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
@@ -159,9 +159,9 @@
     "@jest/test-sequencer": "24.9.0",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
-    "@types/bull": "3.15.8",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
+    "@types/ioredis": "4.28.10",
     "@types/jest": "27.5.1",
     "@types/koa": "2.13.5",
     "@types/koa__router": "8.0.11",
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index 2741e41096..abb2272a99 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -89,6 +89,11 @@ async function importProcessor(job: Job) {
   const tenantId = tenancy.getTenantIDFromAppID(appId) as string
   tenancy.doInTenant(tenantId, async () => {
     const devAppId = dbCore.getDevAppID(appId)
+    const { rev } = await backups.updateRestoreStatus(
+      data.docId,
+      data.docRev,
+      AppBackupStatus.PENDING
+    )
     // initially export the current state to disk - incase something goes wrong
     await runBackup(
       nameForBackup,
@@ -113,7 +118,7 @@ async function importProcessor(job: Job) {
     } catch (err) {
       status = AppBackupStatus.FAILED
     }
-    await backups.updateRestoreStatus(data.docId, data.docRev, status)
+    await backups.updateRestoreStatus(data.docId, rev, status)
   })
 }
 
@@ -124,8 +129,13 @@ async function exportProcessor(job: Job) {
     name = data.export!.name || `${trigger} - backup`
   const tenantId = tenancy.getTenantIDFromAppID(appId) as string
   await tenancy.doInTenant(tenantId, async () => {
+    const { rev } = await backups.updateBackupStatus(
+      data.docId,
+      data.docRev,
+      AppBackupStatus.PENDING
+    )
     return runBackup(name, trigger, tenantId, appId, {
-      doc: { id: data.docId, rev: data.docRev },
+      doc: { id: data.docId, rev },
     })
   })
 }
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index 0467492533..e124ce969d 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -1130,6 +1130,59 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
+"@budibase/bbui@2.0.30-alpha.7":
+  version "2.0.30-alpha.7"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-2.0.30-alpha.7.tgz#2d0f00b3f244e9d579b146710b6a697c1fa5c3b8"
+  integrity sha512-0Ml31GFHqXUGVcIVoKHQPK79TbugtO0L7jgtQ3oaok+Z2YzWYfSudXsjDZwZrBWyKxTy9odXaodA2Me9O1x6dA==
+  dependencies:
+    "@adobe/spectrum-css-workflow-icons" "^1.2.1"
+    "@budibase/string-templates" "2.0.30-alpha.7"
+    "@spectrum-css/actionbutton" "^1.0.1"
+    "@spectrum-css/actiongroup" "^1.0.1"
+    "@spectrum-css/avatar" "^3.0.2"
+    "@spectrum-css/button" "^3.0.1"
+    "@spectrum-css/buttongroup" "^3.0.2"
+    "@spectrum-css/checkbox" "^3.0.2"
+    "@spectrum-css/dialog" "^3.0.1"
+    "@spectrum-css/divider" "^1.0.3"
+    "@spectrum-css/dropzone" "^3.0.2"
+    "@spectrum-css/fieldgroup" "^3.0.2"
+    "@spectrum-css/fieldlabel" "^3.0.1"
+    "@spectrum-css/icon" "^3.0.1"
+    "@spectrum-css/illustratedmessage" "^3.0.2"
+    "@spectrum-css/inlinealert" "^2.0.1"
+    "@spectrum-css/inputgroup" "^3.0.2"
+    "@spectrum-css/label" "^2.0.10"
+    "@spectrum-css/link" "^3.1.1"
+    "@spectrum-css/menu" "^3.0.1"
+    "@spectrum-css/modal" "^3.0.1"
+    "@spectrum-css/pagination" "^3.0.3"
+    "@spectrum-css/picker" "^1.0.1"
+    "@spectrum-css/popover" "^3.0.1"
+    "@spectrum-css/progressbar" "^1.0.2"
+    "@spectrum-css/progresscircle" "^1.0.2"
+    "@spectrum-css/radio" "^3.0.2"
+    "@spectrum-css/search" "^3.0.2"
+    "@spectrum-css/sidenav" "^3.0.2"
+    "@spectrum-css/slider" "3.0.1"
+    "@spectrum-css/statuslight" "^3.0.2"
+    "@spectrum-css/stepper" "^3.0.3"
+    "@spectrum-css/switch" "^1.0.2"
+    "@spectrum-css/table" "^3.0.1"
+    "@spectrum-css/tabs" "^3.2.12"
+    "@spectrum-css/tags" "^3.0.2"
+    "@spectrum-css/textfield" "^3.0.1"
+    "@spectrum-css/toast" "^3.0.1"
+    "@spectrum-css/tooltip" "^3.0.3"
+    "@spectrum-css/treeview" "^3.0.2"
+    "@spectrum-css/typography" "^3.0.1"
+    "@spectrum-css/underlay" "^2.0.9"
+    "@spectrum-css/vars" "^3.0.1"
+    dayjs "^1.10.4"
+    easymde "^2.16.1"
+    svelte-flatpickr "^3.2.3"
+    svelte-portal "^1.0.0"
+
 "@budibase/bbui@^0.9.139":
   version "0.9.190"
   resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.190.tgz#e1ec400ac90f556bfbc80fc23a04506f1585ea81"
@@ -1180,6 +1233,71 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
+"@budibase/client@2.0.30-alpha.7":
+  version "2.0.30-alpha.7"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-2.0.30-alpha.7.tgz#1243ba442e966574def4b146734266b5b8f9b83c"
+  integrity sha512-mSe97YjLDwm7cKt+ujT/DwedDz19Dkw/m9kyLp2PlmzlnBnzmP/H6vjHvLGBO8nDUbKoYy8MJwR3OQeiwSEimw==
+  dependencies:
+    "@budibase/bbui" "2.0.30-alpha.7"
+    "@budibase/frontend-core" "2.0.30-alpha.7"
+    "@budibase/string-templates" "2.0.30-alpha.7"
+    "@spectrum-css/button" "^3.0.3"
+    "@spectrum-css/card" "^3.0.3"
+    "@spectrum-css/divider" "^1.0.3"
+    "@spectrum-css/link" "^3.1.3"
+    "@spectrum-css/page" "^3.0.1"
+    "@spectrum-css/tag" "^3.1.4"
+    "@spectrum-css/typography" "^3.0.2"
+    "@spectrum-css/vars" "^3.0.1"
+    apexcharts "^3.22.1"
+    dayjs "^1.10.5"
+    downloadjs "1.4.7"
+    html5-qrcode "^2.2.1"
+    leaflet "^1.7.1"
+    regexparam "^1.3.0"
+    sanitize-html "^2.7.0"
+    screenfull "^6.0.1"
+    shortid "^2.2.15"
+    socket.io-client "^4.5.1"
+    svelte "^3.49.0"
+    svelte-apexcharts "^1.0.2"
+    svelte-flatpickr "^3.1.0"
+    svelte-spa-router "^3.0.5"
+
+"@budibase/frontend-core@2.0.30-alpha.7":
+  version "2.0.30-alpha.7"
+  resolved "https://registry.yarnpkg.com/@budibase/frontend-core/-/frontend-core-2.0.30-alpha.7.tgz#86f73b28db3bf9208a772296c13bc0765f898bb0"
+  integrity sha512-Vy14lDwx5MesXH3mX031QR+qUaOZmlV7PzPR0tXHAv/vxZHSc6SwUiKNdT1bpHEw0qLnm4jtsZpKGbO9FqFw+w==
+  dependencies:
+    "@budibase/bbui" "2.0.30-alpha.7"
+    lodash "^4.17.21"
+    svelte "^3.46.2"
+
+"@budibase/handlebars-helpers@^0.11.8":
+  version "0.11.8"
+  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.11.8.tgz#6953d29673a8c5c407e096c0a84890465c7ce841"
+  integrity sha512-ggWJUt0GqsHFAEup5tlWlcrmYML57nKhpNGGLzVsqXVYN8eVmf3xluYmmMe7fDYhQH0leSprrdEXmsdFQF3HAQ==
+  dependencies:
+    array-sort "^1.0.0"
+    define-property "^2.0.2"
+    extend-shallow "^3.0.2"
+    for-in "^1.0.2"
+    get-object "^0.2.0"
+    get-value "^3.0.1"
+    handlebars "^4.7.7"
+    handlebars-utils "^1.0.6"
+    has-value "^2.0.2"
+    helper-md "^0.2.2"
+    html-tag "^2.0.0"
+    is-even "^1.0.0"
+    is-glob "^4.0.1"
+    kind-of "^6.0.3"
+    micromatch "^3.1.5"
+    relative "^3.0.2"
+    striptags "^3.1.1"
+    to-gfm-code-block "^0.1.1"
+    year "^0.2.1"
+
 "@budibase/pro@2.0.30-alpha.7":
   version "2.0.30-alpha.7"
   resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.7.tgz#1432b47141b305666dc005c5fc5a02d9cd38cf6a"
@@ -1209,6 +1327,18 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
+"@budibase/string-templates@2.0.30-alpha.7":
+  version "2.0.30-alpha.7"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.0.30-alpha.7.tgz#f26ea4b16587acdbbf2368659585437b03df9f7d"
+  integrity sha512-EtWyN3TElpLE7YfHjM4hyhehmhbr6xcMt/MiT+6Yy/MZlloB5QT4x3UMKX4SriKwNSlK2UYsHLCzt7Ho1V8DqA==
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.11.8"
+    dayjs "^1.10.4"
+    handlebars "^4.7.6"
+    handlebars-utils "^1.0.6"
+    lodash "^4.17.20"
+    vm2 "^3.9.4"
+
 "@budibase/types@2.0.30-alpha.7":
   version "2.0.30-alpha.7"
   resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.7.tgz#fe85563eb42ce01869e653a790a7f213ea2d6308"
@@ -2467,6 +2597,11 @@
   resolved "https://registry.yarnpkg.com/@spectrum-css/sidenav/-/sidenav-3.0.23.tgz#c218560d472e13a3e0d1499b762df1206dcffbfd"
   integrity sha512-4IFw2/HMQJRzM0M2c5na/HeY7y5vJoGpMFBkXNpQyhW4TRo7N1rGwYQ5dRD3s4OVEWV4/rjfGV0d/qhfwKUTog==
 
+"@spectrum-css/slider@3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/slider/-/slider-3.0.1.tgz#5281e6f47eb5a4fd3d1816c138bf66d01d7f2e49"
+  integrity sha512-DI2dtMRnQuDM1miVzl3SGyR1khUEKnwdXfO5EHDFwkC3yav43F5QogkfjmjFmWWobMVovdJlAuiaaJ/IHejD0Q==
+
 "@spectrum-css/statuslight@^3.0.2":
   version "3.0.8"
   resolved "https://registry.yarnpkg.com/@spectrum-css/statuslight/-/statuslight-3.0.8.tgz#3b0ea80712573679870a85d469850230e794a0f7"
@@ -2492,6 +2627,16 @@
   resolved "https://registry.yarnpkg.com/@spectrum-css/tabs/-/tabs-3.2.16.tgz#c3f7800d8d6f7c9930c28cd01354816328bf72b1"
   integrity sha512-JUcMB/fiDG/KoyrVstlUMacFJUY4OHKqhMRuPtu9ggUXWCRbSkY8he92v6u0HwY3DuhDoOxNTK8d/PLjk/fsbg==
 
+"@spectrum-css/tabs@^3.2.12":
+  version "3.2.20"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/tabs/-/tabs-3.2.20.tgz#f8c139402619e2b9cda14da31d21f3bd9b0f14a3"
+  integrity sha512-wfKFLJpjvx22mXZht+FTK5dYw7t/kkwK/dNwM+Fa8nxn7N97WH392iy+wEgqXPU4HJmgUWxaZANITOaAw4sa0A==
+
+"@spectrum-css/tag@^3.1.4":
+  version "3.3.14"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/tag/-/tag-3.3.14.tgz#2d1ca0759da6a3a2970d14bcabf33bd4a8f63a5f"
+  integrity sha512-S4RUaxN/83Pr/SYkQHeZNh2NXmtumUEzhrsrrliI6bAt3bjs+mLresTGd9qkIX2+Ycq1JHWTr0HTga4ti1YYyA==
+
 "@spectrum-css/tags@^3.0.2":
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/@spectrum-css/tags/-/tags-3.0.3.tgz#fc76d2735cdc442de91b7eb3bee49a928c0767ac"
@@ -2656,19 +2801,18 @@
   dependencies:
     bson "*"
 
-"@types/bull@3.15.8":
-  version "3.15.8"
-  resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.8.tgz#ae2139f94490d740b37c8da5d828ce75dd82ce7c"
-  integrity sha512-8DbSPMSsZH5PWPnGEkAZLYgJEH4ghHJNKF7LB6Wr5R0/v6g+Vs+JoaA7kcvLtHE936xg2WpFPkaoaJgExOmKDw==
-  dependencies:
-    "@types/ioredis" "*"
-    "@types/redis" "^2.8.0"
-
 "@types/caseless@*":
   version "0.12.2"
   resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8"
   integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==
 
+"@types/codemirror@^5.60.4":
+  version "5.60.5"
+  resolved "https://registry.yarnpkg.com/@types/codemirror/-/codemirror-5.60.5.tgz#5b989a3b4bbe657458cf372c92b6bfda6061a2b7"
+  integrity sha512-TiECZmm8St5YxjFUp64LK0c8WU5bxMDt9YaAek1UqUb9swrSCoJhh92fWu1p3mTEqlHjhB5sY7OFBhWroJXZVg==
+  dependencies:
+    "@types/tern" "*"
+
 "@types/connect@*":
   version "3.4.35"
   resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1"
@@ -2780,7 +2924,7 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
   integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
 
-"@types/ioredis@*":
+"@types/ioredis@^4.28.10":
   version "4.28.10"
   resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
   integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
@@ -2884,6 +3028,11 @@
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a"
   integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==
 
+"@types/marked@^4.0.7":
+  version "4.0.7"
+  resolved "https://registry.yarnpkg.com/@types/marked/-/marked-4.0.7.tgz#400a76809fd08c2bbd9e25f3be06ea38c8e0a1d3"
+  integrity sha512-eEAhnz21CwvKVW+YvRvcTuFKNU9CV1qH+opcgVK3pIMI6YZzDm6gc8o2vHjldFk6MGKt5pueSB7IOpvpx5Qekw==
+
 "@types/mime@^1":
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
@@ -2973,13 +3122,6 @@
   dependencies:
     redis "*"
 
-"@types/redis@^2.8.0":
-  version "2.8.32"
-  resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
-  integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==
-  dependencies:
-    "@types/node" "*"
-
 "@types/request@^2.48.7":
   version "2.48.8"
   resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.8.tgz#0b90fde3b655ab50976cb8c5ac00faca22f5a82c"
@@ -3016,6 +3158,13 @@
     "@types/cookiejar" "*"
     "@types/node" "*"
 
+"@types/tern@*":
+  version "0.23.4"
+  resolved "https://registry.yarnpkg.com/@types/tern/-/tern-0.23.4.tgz#03926eb13dbeaf3ae0d390caf706b2643a0127fb"
+  integrity sha512-JAUw1iXGO1qaWwEOzxTKJZ/5JxVeON9kvGZ/osgZaJImBnyjyn0cjovPsf6FNLmyGY8Vw9DoXZCMlfMkMwHRWg==
+  dependencies:
+    "@types/estree" "*"
+
 "@types/tough-cookie@*":
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
@@ -3627,7 +3776,7 @@ arg@^4.1.0:
   resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
   integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
 
-argparse@^1.0.7:
+argparse@^1.0.10, argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
   integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==
@@ -3674,6 +3823,15 @@ array-equal@^1.0.0:
   resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93"
   integrity sha512-H3LU5RLiSsGXPhN+Nipar0iR0IofH+8r89G2y1tBKxQ/agagKyAjhkAFDRBfodP2caPrNKHpAWNIM/c9yeL7uA==
 
+array-sort@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/array-sort/-/array-sort-1.0.0.tgz#e4c05356453f56f53512a7d1d6123f2c54c0a88a"
+  integrity sha512-ihLeJkonmdiAsD7vpgN3CRcx2J2S0TiYW+IS/5zHBI7mKUq3ySvBdzzBfD236ubDBQFiiyG3SWCPc+msQ9KoYg==
+  dependencies:
+    default-compare "^1.0.0"
+    get-value "^2.0.6"
+    kind-of "^5.0.2"
+
 array-union@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d"
@@ -3824,6 +3982,13 @@ atomic-sleep@^1.0.0:
   resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b"
   integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==
 
+autolinker@~0.28.0:
+  version "0.28.1"
+  resolved "https://registry.yarnpkg.com/autolinker/-/autolinker-0.28.1.tgz#0652b491881879f0775dace0cdca3233942a4e47"
+  integrity sha512-zQAFO1Dlsn69eXaO6+7YZc+v84aquQKbwpzCE3L0stj56ERn9hutFxPopViLjo9G+rWwjozRhgS5KJ25Xy19cQ==
+  dependencies:
+    gulp-header "^1.7.1"
+
 available-typed-arrays@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7"
@@ -4378,10 +4543,10 @@ buffer@^5.1.0, buffer@^5.2.0, buffer@^5.2.1, buffer@^5.5.0, buffer@^5.6.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
-bull@4.8.5:
-  version "4.8.5"
-  resolved "https://registry.yarnpkg.com/bull/-/bull-4.8.5.tgz#eebafddc3249d6d5e8ced1c42b8bfa8efcc274aa"
-  integrity sha512-2Z630e4f6VsLJnWMAtfEHwIqJYmND4W3dcG48RIbXeWpvb4UnYtpe/zxEdslJu0PKrltB4IkFj5YtBsdeQRn8w==
+bull@4.10.1:
+  version "4.10.1"
+  resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f"
+  integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g==
   dependencies:
     cron-parser "^4.2.1"
     debuglog "^1.0.0"
@@ -4673,6 +4838,18 @@ co@^4.6.0:
   resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
   integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==
 
+codemirror-spell-checker@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/codemirror-spell-checker/-/codemirror-spell-checker-1.1.2.tgz#1c660f9089483ccb5113b9ba9ca19c3f4993371e"
+  integrity sha512-2Tl6n0v+GJRsC9K3MLCdLaMOmvWL0uukajNJseorZJsslaxZyZMgENocPU8R0DyoTAiKsyqiemSOZo7kjGV0LQ==
+  dependencies:
+    typo-js "*"
+
+codemirror@^5.63.1:
+  version "5.65.9"
+  resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.65.9.tgz#ec70c92aa206ee4c9853d5f1e7c4ed356cdab68c"
+  integrity sha512-19Jox5sAKpusTDgqgKB5dawPpQcY+ipQK7xoEI+MVucEF9qqFaXpeqY1KaoyGBso/wHQoDa4HMMxMjdsS3Zzzw==
+
 collect-v8-coverage@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59"
@@ -4823,6 +5000,13 @@ concat-map@0.0.1:
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
   integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==
 
+concat-with-sourcemaps@*:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/concat-with-sourcemaps/-/concat-with-sourcemaps-1.1.0.tgz#d4ea93f05ae25790951b99e7b3b09e3908a4082e"
+  integrity sha512-4gEjHJFT9e+2W/77h/DS5SGUgwDaOwprX8L/gl5+3ixnzkVJJsZWDSelmN3Oilw3LNDZjZV0yqH1hLG3k6nghg==
+  dependencies:
+    source-map "^0.6.1"
+
 condense-newlines@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/condense-newlines/-/condense-newlines-0.2.1.tgz#3de985553139475d32502c83b02f60684d24c55f"
@@ -5236,6 +5420,13 @@ deepmerge@^4.2.2:
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
   integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
 
+default-compare@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/default-compare/-/default-compare-1.0.0.tgz#cb61131844ad84d84788fb68fd01681ca7781a2f"
+  integrity sha512-QWfXlM0EkAbqOCbD/6HjdwT19j7WCkMyiRhWilc4H9/5h/RzTF9gv5LYh1+CmDV5d1rki6KAWLtQale0xt20eQ==
+  dependencies:
+    kind-of "^5.0.2"
+
 default-shell@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/default-shell/-/default-shell-1.0.1.tgz#752304bddc6174f49eb29cb988feea0b8813c8bc"
@@ -5408,11 +5599,25 @@ doctrine@3.0.0, doctrine@^3.0.0:
   dependencies:
     esutils "^2.0.2"
 
+dom-serializer@^1.0.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.4.1.tgz#de5d41b1aea290215dc45a6dae8adcf1d32e2d30"
+  integrity sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.2.0"
+    entities "^2.0.0"
+
 dom-walk@^0.1.0:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84"
   integrity sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==
 
+domelementtype@^2.0.1, domelementtype@^2.2.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d"
+  integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==
+
 domexception@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/domexception/-/domexception-1.0.1.tgz#937442644ca6a31261ef36e3ec677fe805582c90"
@@ -5427,6 +5632,22 @@ domexception@^2.0.1:
   dependencies:
     webidl-conversions "^5.0.0"
 
+domhandler@^4.0.0, domhandler@^4.2.0:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.3.1.tgz#8d792033416f59d68bc03a5aa7b018c1ca89279c"
+  integrity sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==
+  dependencies:
+    domelementtype "^2.2.0"
+
+domutils@^2.5.2:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135"
+  integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==
+  dependencies:
+    dom-serializer "^1.0.1"
+    domelementtype "^2.2.0"
+    domhandler "^4.2.0"
+
 dot-prop@^5.2.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88"
@@ -5471,6 +5692,11 @@ download@8.0.0:
     p-event "^2.1.0"
     pify "^4.0.1"
 
+downloadjs@1.4.7:
+  version "1.4.7"
+  resolved "https://registry.yarnpkg.com/downloadjs/-/downloadjs-1.4.7.tgz#f69f96f940e0d0553dac291139865a3cd0101e3c"
+  integrity sha512-LN1gO7+u9xjU5oEScGFKvXhYf7Y/empUIIEAGBs1LzUq/rg5duiDrkuH5A2lQGd5jfMOb9X9usDa2oVXwJ0U/Q==
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -5486,6 +5712,17 @@ duplexify@^4.0.0:
     readable-stream "^3.1.1"
     stream-shift "^1.0.0"
 
+easymde@^2.16.1:
+  version "2.18.0"
+  resolved "https://registry.yarnpkg.com/easymde/-/easymde-2.18.0.tgz#ff1397d07329b1a7b9187d2d0c20766fa16b3b1b"
+  integrity sha512-IxVVUxNWIoXLeqtBU4BLc+eS/ScYhT1Dcb6yF5Wchoj1iXAV+TIIDWx+NCaZhY7RcSHqDPKllbYq7nwGKILnoA==
+  dependencies:
+    "@types/codemirror" "^5.60.4"
+    "@types/marked" "^4.0.7"
+    codemirror "^5.63.1"
+    codemirror-spell-checker "1.1.2"
+    marked "^4.1.0"
+
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -5643,6 +5880,17 @@ end-stream@~0.1.0:
   dependencies:
     write-stream "~0.4.3"
 
+engine.io-client@~6.2.3:
+  version "6.2.3"
+  resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.2.3.tgz#a8cbdab003162529db85e9de31575097f6d29458"
+  integrity sha512-aXPtgF1JS3RuuKcpSrBtimSjYvrbhKW9froICH4s0F3XQWLxsKNxqzG39nnvQZQnva4CMvUK63T7shevxRyYHw==
+  dependencies:
+    "@socket.io/component-emitter" "~3.1.0"
+    debug "~4.3.1"
+    engine.io-parser "~5.0.3"
+    ws "~8.2.3"
+    xmlhttprequest-ssl "~2.0.0"
+
 engine.io-parser@~5.0.3:
   version "5.0.4"
   resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.4.tgz#0b13f704fa9271b3ec4f33112410d8f3f41d0fc0"
@@ -5672,6 +5920,16 @@ enhanced-resolve@^5.9.3:
     graceful-fs "^4.2.4"
     tapable "^2.2.0"
 
+ent@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d"
+  integrity sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==
+
+entities@^2.0.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
+  integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
+
 entities@~2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5"
@@ -6683,6 +6941,11 @@ fs-constants@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
   integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
 
+fs-exists-sync@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz#982d6893af918e72d08dec9e8673ff2b5a8d6add"
+  integrity sha512-cR/vflFyPZtrN6b38ZyWxpWdhlXrzZEBawlpBQMq7033xVY7/kg0GDMBK5jg8lDYQckdJ5x/YC88lM3C7VMsLg==
+
 fs-extra@8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0"
@@ -6831,6 +7094,14 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1:
     has "^1.0.3"
     has-symbols "^1.0.3"
 
+get-object@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/get-object/-/get-object-0.2.0.tgz#d92ff7d5190c64530cda0543dac63a3d47fe8c0c"
+  integrity sha512-7P6y6k6EzEFmO/XyUyFlXm1YLJy9xeA1x/grNV8276abX5GuwUtYgKFkRFkLixw4hf4Pz9q2vgv/8Ar42R0HuQ==
+  dependencies:
+    is-number "^2.0.2"
+    isobject "^0.2.0"
+
 get-package-type@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a"
@@ -6893,6 +7164,13 @@ get-value@^2.0.3, get-value@^2.0.6:
   resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
   integrity sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==
 
+get-value@^3.0.0, get-value@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/get-value/-/get-value-3.0.1.tgz#5efd2a157f1d6a516d7524e124ac52d0a39ef5a8"
+  integrity sha512-mKZj9JLQrwMBtj5wxi6MH8Z5eSKaERpAwjg43dPtlGI1ZVEgH/qC7T8/6R2OBSUA+zzHBZgICsVJaEIV2tKTDA==
+  dependencies:
+    isobject "^3.0.1"
+
 getopts@2.2.5:
   version "2.2.5"
   resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.5.tgz#67a0fe471cacb9c687d817cab6450b96dde8313b"
@@ -7198,7 +7476,24 @@ gtoken@^5.0.4:
     google-p12-pem "^3.1.3"
     jws "^4.0.0"
 
-handlebars@^4.7.7:
+gulp-header@^1.7.1:
+  version "1.8.12"
+  resolved "https://registry.yarnpkg.com/gulp-header/-/gulp-header-1.8.12.tgz#ad306be0066599127281c4f8786660e705080a84"
+  integrity sha512-lh9HLdb53sC7XIZOYzTXM4lFuXElv3EVkSDhsd7DoJBj7hm+Ni7D3qYbb+Rr8DuM8nRanBvkVO9d7askreXGnQ==
+  dependencies:
+    concat-with-sourcemaps "*"
+    lodash.template "^4.4.0"
+    through2 "^2.0.0"
+
+handlebars-utils@^1.0.6:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/handlebars-utils/-/handlebars-utils-1.0.6.tgz#cb9db43362479054782d86ffe10f47abc76357f9"
+  integrity sha512-d5mmoQXdeEqSKMtQQZ9WkiUcO1E3tPbWxluCK9hVgIDPzQa9WsKo3Lbe/sGflTe7TomHEeZaOgwIkyIr1kfzkw==
+  dependencies:
+    kind-of "^6.0.0"
+    typeof-article "^0.1.1"
+
+handlebars@^4.7.6, handlebars@^4.7.7:
   version "4.7.7"
   resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1"
   integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==
@@ -7292,6 +7587,14 @@ has-value@^1.0.0:
     has-values "^1.0.0"
     isobject "^3.0.0"
 
+has-value@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/has-value/-/has-value-2.0.2.tgz#d0f12e8780ba8e90e66ad1a21c707fdb67c25658"
+  integrity sha512-ybKOlcRsK2MqrM3Hmz/lQxXHZ6ejzSPzpNabKB45jb5qDgJvKPa3SdapTsTLwEb9WltgWpOmNax7i+DzNOk4TA==
+  dependencies:
+    get-value "^3.0.0"
+    has-values "^2.0.1"
+
 has-values@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771"
@@ -7305,6 +7608,13 @@ has-values@^1.0.0:
     is-number "^3.0.0"
     kind-of "^4.0.0"
 
+has-values@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/has-values/-/has-values-2.0.1.tgz#3876200ff86d8a8546a9264a952c17d5fc17579d"
+  integrity sha512-+QdH3jOmq9P8GfdjFg0eJudqx1FqU62NQJ4P16rOEHeRdl7ckgwn6uqQjzYE0ZoHVV/e5E2esuJ5Gl5+HUW19w==
+  dependencies:
+    kind-of "^6.0.2"
+
 has-yarn@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-2.1.0.tgz#137e11354a7b5bf11aa5cb649cf0c6f3ff2b2e77"
@@ -7317,6 +7627,16 @@ has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+helper-md@^0.2.2:
+  version "0.2.2"
+  resolved "https://registry.yarnpkg.com/helper-md/-/helper-md-0.2.2.tgz#c1f59d7e55bbae23362fd8a0e971607aec69d41f"
+  integrity sha512-49TaQzK+Ic7ZVTq4i1UZxRUJEmAilTk8hz7q4I0WNUaTclLR8ArJV5B3A1fe1xF2HtsDTr2gYKLaVTof/Lt84Q==
+  dependencies:
+    ent "^2.2.0"
+    extend-shallow "^2.0.1"
+    fs-exists-sync "^0.1.0"
+    remarkable "^1.6.2"
+
 homedir-polyfill@^1.0.1:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8"
@@ -7353,6 +7673,29 @@ html-escaper@^2.0.0:
   resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453"
   integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==
 
+html-tag@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/html-tag/-/html-tag-2.0.0.tgz#36c3bc8d816fd30b570d5764a497a641640c2fed"
+  integrity sha512-XxzooSo6oBoxBEUazgjdXj7VwTn/iSTSZzTYKzYY6I916tkaYzypHxy+pbVU1h+0UQ9JlVf5XkNQyxOAiiQO1g==
+  dependencies:
+    is-self-closing "^1.0.1"
+    kind-of "^6.0.0"
+
+html5-qrcode@^2.2.1:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.2.3.tgz#5acb826860365e7c7ab91e1e14528ea16a502e8a"
+  integrity sha512-9CtEz5FVT56T76entiQxyrASzBWl8Rm30NHiQH8T163Eml5LS14BoZlYel9igxbikOt7O8KhvrT3awN1Y2HMqw==
+
+htmlparser2@^6.0.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
+  integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.0.0"
+    domutils "^2.5.2"
+    entities "^2.0.0"
+
 http-assert@^1.3.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/http-assert/-/http-assert-1.5.0.tgz#c389ccd87ac16ed2dfa6246fd73b926aa00e6b8f"
@@ -7819,6 +8162,13 @@ is-docker@^2.0.0, is-docker@^2.1.1:
   resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa"
   integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==
 
+is-even@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-even/-/is-even-1.0.0.tgz#76b5055fbad8d294a86b6a949015e1c97b717c06"
+  integrity sha512-LEhnkAdJqic4Dbqn58A0y52IXoHWlsueqQkKfMfdEnIYG8A1sm/GHidKkS6yvXlMoRrkM34csHnXQtOqcb+Jzg==
+  dependencies:
+    is-odd "^0.1.2"
+
 is-extendable@^0.1.0, is-extendable@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89"
@@ -7925,6 +8275,13 @@ is-number-object@^1.0.4:
   dependencies:
     has-tostringtag "^1.0.0"
 
+is-number@^2.0.2:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f"
+  integrity sha512-QUzH43Gfb9+5yckcrSA0VBDwEtDUchrk4F6tfJZQuNzDJbEDB9cZNzSfXGQ1jqmdDY/kl41lUOWM9syA8z8jlg==
+  dependencies:
+    kind-of "^3.0.2"
+
 is-number@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195"
@@ -7947,6 +8304,13 @@ is-object@^1.0.1:
   resolved "https://registry.yarnpkg.com/is-object/-/is-object-1.0.2.tgz#a56552e1c665c9e950b4a025461da87e72f86fcf"
   integrity sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==
 
+is-odd@^0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/is-odd/-/is-odd-0.1.2.tgz#bc573b5ce371ef2aad6e6f49799b72bef13978a7"
+  integrity sha512-Ri7C2K7o5IrUU9UEI8losXJCCD/UtsaIrkR5sxIcFg4xQ9cRJXlWA5DQvTE0yDc0krvSNLsRGXN11UPS6KyfBw==
+  dependencies:
+    is-number "^3.0.0"
+
 is-path-inside@^3.0.2:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283"
@@ -7964,6 +8328,11 @@ is-plain-object@^2.0.3, is-plain-object@^2.0.4:
   dependencies:
     isobject "^3.0.1"
 
+is-plain-object@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344"
+  integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==
+
 is-potential-custom-element-name@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5"
@@ -7992,6 +8361,13 @@ is-retry-allowed@^2.2.0:
   resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz#88f34cbd236e043e71b6932d09b0c65fb7b4d71d"
   integrity sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==
 
+is-self-closing@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/is-self-closing/-/is-self-closing-1.0.1.tgz#5f406b527c7b12610176320338af0fa3896416e4"
+  integrity sha512-E+60FomW7Blv5GXTlYee2KDrnG6srxF7Xt1SjrhWUGUEsTFIqY/nq2y3DaftCsgUMdh89V07IVfhY9KIJhLezg==
+  dependencies:
+    self-closing-tags "^1.0.1"
+
 is-shared-array-buffer@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79"
@@ -8102,6 +8478,11 @@ isexe@^2.0.0:
   resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
   integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==
 
+isobject@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/isobject/-/isobject-0.2.0.tgz#a3432192f39b910b5f02cc989487836ec70aa85e"
+  integrity sha512-VaWq6XYAsbvM0wf4dyBO7WH9D7GosB7ZZlqrawI9BBiTMINBeCyqSKBa35m870MY3O4aM31pYyZi9DfGrYMJrQ==
+
 isobject@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89"
@@ -9277,7 +9658,7 @@ keyv@^3.0.0:
   dependencies:
     json-buffer "3.0.0"
 
-kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
+kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.1.0, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
   integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
@@ -9291,12 +9672,12 @@ kind-of@^4.0.0:
   dependencies:
     is-buffer "^1.1.5"
 
-kind-of@^5.0.0:
+kind-of@^5.0.0, kind-of@^5.0.2:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d"
   integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==
 
-kind-of@^6.0.0, kind-of@^6.0.2:
+kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3:
   version "6.0.3"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd"
   integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==
@@ -9511,6 +9892,11 @@ lcid@^2.0.0:
   dependencies:
     invert-kv "^2.0.0"
 
+leaflet@^1.7.1:
+  version "1.9.2"
+  resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-1.9.2.tgz#168b6c6ef1d4d1e8409bde2c4ad050c249c4dbe6"
+  integrity sha512-Kc77HQvWO+y9y2oIs3dn5h5sy2kr3j41ewdqCMEUA4N89lgfUUfOBy7wnnHEstDpefiGFObq12FdopGRMx4J7g==
+
 left-pad@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e"
@@ -9705,6 +10091,11 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+lodash._reinterpolate@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
+  integrity sha512-xYHt68QRoYGjeeM/XOE1uJtvXQAgvszfBhjV4yvsQH0u2i9I6cI6c6/eG4Hh3UAOVn0y/xAXwmTzEay49Q//HA==
+
 lodash.camelcase@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
@@ -9815,6 +10206,21 @@ lodash.sortby@^4.7.0:
   resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
   integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==
 
+lodash.template@^4.4.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.template/-/lodash.template-4.5.0.tgz#f976195cf3f347d0d5f52483569fe8031ccce8ab"
+  integrity sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==
+  dependencies:
+    lodash._reinterpolate "^3.0.0"
+    lodash.templatesettings "^4.0.0"
+
+lodash.templatesettings@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz#e481310f049d3cf6d47e912ad09313b154f0fb33"
+  integrity sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==
+  dependencies:
+    lodash._reinterpolate "^3.0.0"
+
 lodash.without@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac"
@@ -9825,7 +10231,7 @@ lodash.xor@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
   integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ==
 
-lodash@4.17.21, lodash@^4.14.0, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
+lodash@4.17.21, lodash@^4.14.0, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -9967,6 +10373,11 @@ markdown-it@^12.2.0:
     mdurl "^1.0.1"
     uc.micro "^1.0.5"
 
+marked@^4.1.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/marked/-/marked-4.1.1.tgz#2f709a4462abf65a283f2453dc1c42ab177d302e"
+  integrity sha512-0cNMnTcUJPxbA6uWmCmjWz4NJRe/0Xfk2NhXCUHjew9qJzFN20krFnsUe7QynwqOwa5m1fZ4UDg0ycKFVC0ccw==
+
 matcher@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/matcher/-/matcher-3.0.0.tgz#bd9060f4c5b70aa8041ccc6f80368760994f30ca"
@@ -10062,7 +10473,7 @@ methods@^1.1.1, methods@^1.1.2:
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==
 
-micromatch@^3.1.10, micromatch@^3.1.4:
+micromatch@^3.1.10, micromatch@^3.1.4, micromatch@^3.1.5:
   version "3.1.10"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23"
   integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==
@@ -10353,6 +10764,16 @@ nan@^2.12.1:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.16.0.tgz#664f43e45460fb98faf00edca0bb0d7b8dce7916"
   integrity sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==
 
+nanoid@^2.1.0:
+  version "2.1.11"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-2.1.11.tgz#ec24b8a758d591561531b4176a01e3ab4f0f0280"
+  integrity sha512-s/snB+WGm6uwi0WjsZdaVcuf3KJXlfGl2LcxgwkEwJF0D/BWzVWAZW/XY4bFaiR7s0Jk3FPvlnepg1H1b1UwlA==
+
+nanoid@^3.3.4:
+  version "3.3.4"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab"
+  integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==
+
 nanomatch@^1.2.9:
   version "1.2.13"
   resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119"
@@ -11009,6 +11430,11 @@ parse-passwd@^1.0.0:
   resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
   integrity sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==
 
+parse-srcset@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/parse-srcset/-/parse-srcset-1.0.2.tgz#f2bd221f6cc970a938d88556abc589caaaa2bde1"
+  integrity sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==
+
 parse5@4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
@@ -11370,6 +11796,15 @@ posix-character-classes@^0.1.0:
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
   integrity sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==
 
+postcss@^8.3.11:
+  version "8.4.18"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.18.tgz#6d50046ea7d3d66a85e0e782074e7203bc7fbca2"
+  integrity sha512-Wi8mWhncLJm11GATDaQKobXSNEYGUHeQLiQqDFG1qQ5UTDPTEvKw0Xt5NsTpktGTwLps3ByrWsBrG0rB8YQ9oA==
+  dependencies:
+    nanoid "^3.3.4"
+    picocolors "^1.0.0"
+    source-map-js "^1.0.2"
+
 postgres-array@~2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e"
@@ -12114,6 +12549,16 @@ regexp.prototype.flags@^1.4.3:
     define-properties "^1.1.3"
     functions-have-names "^1.2.2"
 
+regexparam@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
+  integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==
+
+regexparam@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
+  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
+
 regexpp@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f"
@@ -12162,6 +12607,21 @@ relative-microtime@^2.0.0:
   resolved "https://registry.yarnpkg.com/relative-microtime/-/relative-microtime-2.0.0.tgz#cceed2af095ecd72ea32011279c79e5fcc7de29b"
   integrity sha512-l18ha6HEZc+No/uK4GyAnNxgKW7nvEe35IaeN54sShMojtqik2a6GbTyuiezkjpPaqP874Z3lW5ysBo5irz4NA==
 
+relative@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/relative/-/relative-3.0.2.tgz#0dcd8ec54a5d35a3c15e104503d65375b5a5367f"
+  integrity sha512-Q5W2qeYtY9GbiR8z1yHNZ1DGhyjb4AnLEjt8iE6XfcC1QIu+FAtj3HQaO0wH28H1mX6cqNLvAqWhP402dxJGyA==
+  dependencies:
+    isobject "^2.0.0"
+
+remarkable@^1.6.2:
+  version "1.7.4"
+  resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-1.7.4.tgz#19073cb960398c87a7d6546eaa5e50d2022fcd00"
+  integrity sha512-e6NKUXgX95whv7IgddywbeN/ItCkWbISmc2DiqHJb0wTrqZIexqdco5b8Z3XZoo/48IdNVKM9ZCvTPJ4F5uvhg==
+  dependencies:
+    argparse "^1.0.10"
+    autolinker "~0.28.0"
+
 remove-trailing-separator@^1.0.1:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef"
@@ -12449,6 +12909,18 @@ sane@^4.0.3:
     minimist "^1.1.1"
     walker "~1.0.5"
 
+sanitize-html@^2.7.0:
+  version "2.7.2"
+  resolved "https://registry.yarnpkg.com/sanitize-html/-/sanitize-html-2.7.2.tgz#54c5189af75e3237d996e4b9a5e3eaad12c7f7fc"
+  integrity sha512-DggSTe7MviO+K4YTCwprG6W1vsG+IIX67yp/QY55yQqKCJYSWzCA1rZbaXzkjoKeL9+jqwm56wD6srYLtUNivg==
+  dependencies:
+    deepmerge "^4.2.2"
+    escape-string-regexp "^4.0.0"
+    htmlparser2 "^6.0.0"
+    is-plain-object "^5.0.0"
+    parse-srcset "^1.0.2"
+    postcss "^8.3.11"
+
 sanitize-s3-objectkey@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/sanitize-s3-objectkey/-/sanitize-s3-objectkey-0.0.1.tgz#efa9887cd45275b40234fb4bb12fc5754fe64e7e"
@@ -12487,6 +12959,11 @@ schema-utils@^3.1.0, schema-utils@^3.1.1:
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
+screenfull@^6.0.1:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/screenfull/-/screenfull-6.0.2.tgz#3dbe4b8c4f8f49fb8e33caa8f69d0bca730ab238"
+  integrity sha512-AQdy8s4WhNvUZ6P8F6PB21tSPIYKniic+Ogx0AacBMjKP1GUHN2E9URxQHtCusiwxudnCKkdy4GrHXPPJSkCCw==
+
 search-params@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/search-params/-/search-params-3.0.0.tgz#dbc7c243058e5a33ae1e9870be91f5aced4100d8"
@@ -12504,6 +12981,11 @@ seek-bzip@^1.0.5:
   dependencies:
     commander "^2.8.1"
 
+self-closing-tags@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/self-closing-tags/-/self-closing-tags-1.0.1.tgz#6c5fa497994bb826b484216916371accee490a5d"
+  integrity sha512-7t6hNbYMxM+VHXTgJmxwgZgLGktuXtVVD5AivWzNTdJBM4DBjnDKDzkf2SrNjihaArpeJYNjxkELBu1evI4lQA==
+
 semver-compare@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc"
@@ -12649,6 +13131,13 @@ shimmer@^1.2.0:
   resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337"
   integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==
 
+shortid@^2.2.15:
+  version "2.2.16"
+  resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608"
+  integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g==
+  dependencies:
+    nanoid "^2.1.0"
+
 side-channel@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf"
@@ -12789,6 +13278,16 @@ socket.io-adapter@~2.4.0:
   resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.4.0.tgz#b50a4a9ecdd00c34d4c8c808224daa1a786152a6"
   integrity sha512-W4N+o69rkMEGVuk2D/cvca3uYsvGlMwsySWV447y99gUPghxq42BxqLNMndb+a1mm/5/7NeXVQS7RLa2XyXvYg==
 
+socket.io-client@^4.5.1:
+  version "4.5.3"
+  resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.5.3.tgz#bed69209d001465b2fea650d2e95c1e82768ab5e"
+  integrity sha512-I/hqDYpQ6JKwtJOf5ikM+Qz+YujZPMEl6qBLhxiP0nX+TfXKhW4KZZG8lamrD6Y5ngjmYHreESVasVCgi5Kl3A==
+  dependencies:
+    "@socket.io/component-emitter" "~3.1.0"
+    debug "~4.3.2"
+    engine.io-client "~6.2.3"
+    socket.io-parser "~4.2.0"
+
 socket.io-parser@~4.2.0:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.1.tgz#01c96efa11ded938dcb21cbe590c26af5eff65e5"
@@ -12851,6 +13350,11 @@ source-list-map@^2.0.1:
   resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34"
   integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==
 
+source-map-js@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
+  integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
+
 source-map-resolve@^0.5.0:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a"
@@ -13252,6 +13756,11 @@ strip-outer@^1.0.0:
   dependencies:
     escape-string-regexp "^1.0.2"
 
+striptags@^3.1.1:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.2.0.tgz#cc74a137db2de8b0b9a370006334161f7dd67052"
+  integrity sha512-g45ZOGzHDMe2bdYMdIvdAfCQkCTDMGBazSw1ypMowwGIee7ZQ5dU0rBJ8Jqgl+jAKIv4dbeE1jscZq9wid1Tkw==
+
 style-loader@^3.3.1:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-3.3.1.tgz#057dfa6b3d4d7c7064462830f9113ed417d38575"
@@ -13351,11 +13860,23 @@ svelte-portal@^1.0.0:
   resolved "https://registry.yarnpkg.com/svelte-portal/-/svelte-portal-1.0.0.tgz#36a47c5578b1a4d9b4dc60fa32a904640ec4cdd3"
   integrity sha512-nHf+DS/jZ6jjnZSleBMSaZua9JlG5rZv9lOGKgJuaZStfevtjIlUJrkLc3vbV8QdBvPPVmvcjTlazAzfKu0v3Q==
 
+svelte-spa-router@^3.0.5:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/svelte-spa-router/-/svelte-spa-router-3.3.0.tgz#2fc0967a49dc361dfe4d38dddad6e662eed5b42c"
+  integrity sha512-cwRNe7cxD43sCvSfEeaKiNZg3FCizGxeMcf7CPiWRP3jKXjEma3vxyyuDtPOam6nWbVxl9TNM3hlE/i87ZlqcQ==
+  dependencies:
+    regexparam "2.0.1"
+
 svelte@3.49.0:
   version "3.49.0"
   resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.49.0.tgz#5baee3c672306de1070c3b7888fc2204e36a4029"
   integrity sha512-+lmjic1pApJWDfPCpUUTc1m8azDqYCG1JN9YEngrx/hUyIcFJo6VZhj0A1Ai0wqoHcEIuQy+e9tk+4uDgdtsFA==
 
+svelte@^3.46.2, svelte@^3.49.0:
+  version "3.52.0"
+  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.52.0.tgz#08259eff20904c63882b66a5d409a55e8c6743b8"
+  integrity sha512-FxcnEUOAVfr10vDU5dVgJN19IvqeHQCS1zfe8vayTfis9A2t5Fhx+JDe5uv/C3j//bB1umpLJ6quhgs9xyUbCQ==
+
 svg.draggable.js@^2.2.2:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/svg.draggable.js/-/svg.draggable.js-2.2.2.tgz#c514a2f1405efb6f0263e7958f5b68fce50603ba"
@@ -13713,6 +14234,11 @@ to-fast-properties@^2.0.0:
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
   integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=
 
+to-gfm-code-block@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/to-gfm-code-block/-/to-gfm-code-block-0.1.1.tgz#25d045a5fae553189e9637b590900da732d8aa82"
+  integrity sha512-LQRZWyn8d5amUKnfR9A9Uu7x9ss7Re8peuWR2gkh1E+ildOfv2aF26JpuDg8JtvCduu5+hOrMIH+XstZtnagqg==
+
 to-json-schema@0.2.5:
   version "0.2.5"
   resolved "https://registry.yarnpkg.com/to-json-schema/-/to-json-schema-0.2.5.tgz#ef3c3f11ad64460dcfbdbafd0fd525d69d62a98f"
@@ -13978,6 +14504,13 @@ typedarray-to-buffer@^3.1.5:
   dependencies:
     is-typedarray "^1.0.0"
 
+typeof-article@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/typeof-article/-/typeof-article-0.1.1.tgz#9f07e733c3fbb646ffa9e61c08debacd460e06af"
+  integrity sha512-Vn42zdX3FhmUrzEmitX3iYyLb+Umwpmv8fkZRIknYh84lmdrwqZA5xYaoKiIj2Rc5i/5wcDrpUmZcbk1U51vTw==
+  dependencies:
+    kind-of "^3.1.0"
+
 typeof@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/typeof/-/typeof-1.0.0.tgz#9c84403f2323ae5399167275497638ea1d2f2440"
@@ -13988,6 +14521,11 @@ typescript@4.6.2:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
   integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
 
+typo-js@*:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/typo-js/-/typo-js-1.2.2.tgz#340484d81fe518e77c81a5a770162b14492f183b"
+  integrity sha512-C7pYBQK17EjSg8tVNY91KHdUt5Nf6FMJ+c3js076quPmBML57PmNMzAcIq/2kf/hSYtFABNDIYNYlJRl5BJhGw==
+
 uc.micro@^1.0.1, uc.micro@^1.0.5:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
@@ -14317,7 +14855,7 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
-vm2@3.9.11:
+vm2@3.9.11, vm2@^3.9.4:
   version "3.9.11"
   resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.11.tgz#a880f510a606481719ec3f9803b940c5805a06fe"
   integrity sha512-PFG8iJRSjvvBdisowQ7iVF580DXb1uCIiGaXgm7tynMR1uTBlv7UJlB1zdv5KJ+Tmq1f0Upnj3fayoEOPpCBKg==
@@ -14776,6 +15314,11 @@ xmlchars@^2.2.0:
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
   integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
 
+xmlhttprequest-ssl@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67"
+  integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==
+
 xpath.js@~1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/xpath.js/-/xpath.js-1.1.0.tgz#3816a44ed4bb352091083d002a383dd5104a5ff1"
@@ -14896,6 +15439,11 @@ yauzl@^2.4.2:
     buffer-crc32 "~0.2.3"
     fd-slicer "~1.1.0"
 
+year@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/year/-/year-0.2.1.tgz#4083ae520a318b23ec86037f3000cb892bdf9bb0"
+  integrity sha512-9GnJUZ0QM4OgXuOzsKNzTJ5EOkums1Xc+3YQXp+Q+UxFjf7zLucp9dQ8QMIft0Szs1E1hUiXFim1OYfEKFq97w==
+
 ylru@^1.2.0:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.3.2.tgz#0de48017473275a4cbdfc83a1eaf67c01af8a785"
diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts
index f7b6239b4f..1dd7b45287 100644
--- a/packages/types/src/documents/app/backup.ts
+++ b/packages/types/src/documents/app/backup.ts
@@ -8,6 +8,7 @@ export enum AppBackupType {
 
 export enum AppBackupStatus {
   STARTED = "started",
+  PENDING = "pending",
   COMPLETE = "complete",
   FAILED = "failed",
 }
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index 1b48d95015..56666f7e2a 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -9,6 +9,7 @@ export interface ContextUser extends User {
 
 export interface BBContext {
   user?: ContextUser
+  status?: number
   request: {
     body: any
   }

From e1dca87507da2deae2a22261d5c23a3d613f6654 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 21 Oct 2022 18:52:39 +0100
Subject: [PATCH 38/46] Fixing test case.

---
 .../src/api/routes/tests/backup.spec.js       |  2 +-
 .../server/src/sdk/app/backups/exports.ts     | 10 +++-
 .../internal-api/TestConfiguration/screens.ts | 38 +++++++-------
 .../config/internal-api/fixtures/screens.ts   | 50 +++++++++----------
 .../internal-api/screens/screens.spec.ts      | 33 ++++++++----
 5 files changed, 75 insertions(+), 58 deletions(-)

diff --git a/packages/server/src/api/routes/tests/backup.spec.js b/packages/server/src/api/routes/tests/backup.spec.js
index 7f92a4f3d1..7863129c75 100644
--- a/packages/server/src/api/routes/tests/backup.spec.js
+++ b/packages/server/src/api/routes/tests/backup.spec.js
@@ -21,7 +21,7 @@ describe("/backups", () => {
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.text).toBeDefined()
-      expect(res.text.includes(`"db_name":"${config.getAppId()}"`)).toEqual(true)
+      expect(res.headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported.mock.calls.length).toBe(1)
     })
 
diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts
index 26277fcd66..8de51ed1e6 100644
--- a/packages/server/src/sdk/app/backups/exports.ts
+++ b/packages/server/src/sdk/app/backups/exports.ts
@@ -1,7 +1,7 @@
 import { db as dbCore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
-import { streamFile } from "../../../utilities/fileSystem"
+import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import {
   LINK_USER_METADATA_PREFIX,
@@ -11,6 +11,7 @@ import {
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import fs from "fs"
 import { join } from "path"
+import env from "../../../environment"
 const uuid = require("uuid/v4")
 const tar = require("tar")
 const MemoryStream = require("memorystream")
@@ -85,7 +86,12 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   const prodAppId = dbCore.getProdAppID(appId)
   const appPath = `${prodAppId}/`
   // export bucket contents
-  const tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath)
+  let tmpPath
+  if (!env.isTest()) {
+    tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath)
+  } else {
+    tmpPath = createTempFolder(uuid())
+  }
   const downloadedPath = join(tmpPath, appPath)
   if (fs.existsSync(downloadedPath)) {
     const allFiles = fs.readdirSync(downloadedPath)
diff --git a/qa-core/src/config/internal-api/TestConfiguration/screens.ts b/qa-core/src/config/internal-api/TestConfiguration/screens.ts
index 30c688022a..7814ffdfbc 100644
--- a/qa-core/src/config/internal-api/TestConfiguration/screens.ts
+++ b/qa-core/src/config/internal-api/TestConfiguration/screens.ts
@@ -1,23 +1,23 @@
-import { Screen } from  "@budibase/types"
+import { Screen } from "@budibase/types"
 import { Response } from "node-fetch"
 import InternalAPIClient from "./InternalAPIClient"
-  
-  export default class ScreenApi {
-    api: InternalAPIClient
-  
-    constructor(apiClient: InternalAPIClient) {
-      this.api = apiClient
-    }
-  
-    async create(body: any): Promise<[Response, Screen]> {
-      const response = await this.api.post(`/screens`, { body })
-      const json = await response.json()
-      return [response, json]
-    }
 
-    async delete(screenId: string, rev: string): Promise<[Response, Screen]> {
-      const response = await this.api.del(`/screens/${screenId}/${rev}`)
-      const json = await response.json()
-      return [response, json]
-    }
+export default class ScreenApi {
+  api: InternalAPIClient
+
+  constructor(apiClient: InternalAPIClient) {
+    this.api = apiClient
   }
+
+  async create(body: any): Promise<[Response, Screen]> {
+    const response = await this.api.post(`/screens`, { body })
+    const json = await response.json()
+    return [response, json]
+  }
+
+  async delete(screenId: string, rev: string): Promise<[Response, Screen]> {
+    const response = await this.api.del(`/screens/${screenId}/${rev}`)
+    const json = await response.json()
+    return [response, json]
+  }
+}
diff --git a/qa-core/src/config/internal-api/fixtures/screens.ts b/qa-core/src/config/internal-api/fixtures/screens.ts
index 1ebc1eb5c8..044a35ba83 100644
--- a/qa-core/src/config/internal-api/fixtures/screens.ts
+++ b/qa-core/src/config/internal-api/fixtures/screens.ts
@@ -3,32 +3,32 @@ import generator from "../../generator"
 const randomId = generator.guid()
 
 const generateScreen = (roleId: string): any => ({
-    showNavigation: true,
-    width: "Large",
-    name: randomId,
-    template: "createFromScratch",
-    props: {
-        _id: randomId,
-        _component:
-            "@budibase/standard-components/container",
-        _styles: {
-            normal: {},
-            hover: {},
-            active: {},
-            selected: {}
-        },
-        _children: [],
-        _instanceName: "New Screen",
-        direction: "column",
-        hAlign: "stretch",
-        vAlign: "top",
-        size: "grow",
-        gap: "M"
-    }, routing: {
-        route: "/test",
-        roleId: roleId,
-        homeScreen: false
+  showNavigation: true,
+  width: "Large",
+  name: randomId,
+  template: "createFromScratch",
+  props: {
+    _id: randomId,
+    _component: "@budibase/standard-components/container",
+    _styles: {
+      normal: {},
+      hover: {},
+      active: {},
+      selected: {},
     },
+    _children: [],
+    _instanceName: "New Screen",
+    direction: "column",
+    hAlign: "stretch",
+    vAlign: "top",
+    size: "grow",
+    gap: "M",
+  },
+  routing: {
+    route: "/test",
+    roleId: roleId,
+    homeScreen: false,
+  },
 })
 
 export default generateScreen
diff --git a/qa-core/src/tests/internal-api/screens/screens.spec.ts b/qa-core/src/tests/internal-api/screens/screens.spec.ts
index 68e1022cb4..1e0d647c3c 100644
--- a/qa-core/src/tests/internal-api/screens/screens.spec.ts
+++ b/qa-core/src/tests/internal-api/screens/screens.spec.ts
@@ -2,10 +2,9 @@ import TestConfiguration from "../../../config/internal-api/TestConfiguration"
 import { App } from "@budibase/types"
 import InternalAPIClient from "../../../config/internal-api/TestConfiguration/InternalAPIClient"
 import generateApp from "../../../config/internal-api/fixtures/applications"
-import { Screen } from  "@budibase/types"
+import { Screen } from "@budibase/types"
 import generateScreen from "../../../config/internal-api/fixtures/screens"
 
-
 describe("Internal API - /screens endpoints", () => {
   const api = new InternalAPIClient()
   const config = new TestConfiguration<Screen>(api)
@@ -21,13 +20,17 @@ describe("Internal API - /screens endpoints", () => {
 
   it("POST - Create a screen with each role type", async () => {
     // Create app
-    const [appResponse, app] = await appConfig.applications.create(generateApp())
-    
+    const [appResponse, app] = await appConfig.applications.create(
+      generateApp()
+    )
+
     // Create Screen
     const roleArray = ["BASIC", "POWER", "ADMIN", "PUBLIC"]
     appConfig.applications.api.appId = app.appId
     for (let role in roleArray) {
-      const [response, screen] = await config.screen.create(generateScreen(roleArray[role]))
+      const [response, screen] = await config.screen.create(
+        generateScreen(roleArray[role])
+      )
       expect(response).toHaveStatusCode(200)
       expect(screen.routing.roleId).toEqual(roleArray[role])
     }
@@ -35,11 +38,15 @@ describe("Internal API - /screens endpoints", () => {
 
   it("GET - Fetch screens", async () => {
     // Create app
-    const [appResponse, app] = await appConfig.applications.create(generateApp())
-    
+    const [appResponse, app] = await appConfig.applications.create(
+      generateApp()
+    )
+
     // Create Screen
     appConfig.applications.api.appId = app.appId
-    const [response, screen] = await config.screen.create(generateScreen("BASIC"))
+    const [response, screen] = await config.screen.create(
+      generateScreen("BASIC")
+    )
 
     // Check screen exists
     const [routesResponse, routes] = await appConfig.applications.getRoutes()
@@ -49,11 +56,15 @@ describe("Internal API - /screens endpoints", () => {
 
   it("DELETE - Delete a screen", async () => {
     // Create app
-    const [appResponse, app] = await appConfig.applications.create(generateApp())
-    
+    const [appResponse, app] = await appConfig.applications.create(
+      generateApp()
+    )
+
     // Create Screen
     appConfig.applications.api.appId = app.appId
-    const [screenResponse, screen] = await config.screen.create(generateScreen("BASIC"))
+    const [screenResponse, screen] = await config.screen.create(
+      generateScreen("BASIC")
+    )
 
     // Delete Screen
     const [response] = await config.screen.delete(screen._id!, screen._rev!)

From 4daf9c26155b4e47c00514580323968aa1ec2407 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 24 Oct 2022 10:04:14 +0100
Subject: [PATCH 39/46] Addressing majority of PR comments.

---
 packages/backend-core/package.json            |  2 +-
 .../src/events/publishers/backup.ts           |  4 ++--
 packages/backend-core/src/queue/listeners.ts  | 12 +++++-----
 packages/backend-core/src/queue/queue.ts      |  4 ++--
 packages/backend-core/yarn.lock               | 23 ++++---------------
 packages/server/package.json                  |  2 +-
 packages/server/src/api/controllers/cloud.js  |  6 ++---
 packages/server/src/sdk/index.ts              |  6 ++---
 packages/server/yarn.lock                     | 16 +------------
 packages/types/src/core/index.ts              |  1 -
 packages/types/src/{core => sdk}/db.ts        |  0
 packages/types/src/sdk/events/backup.ts       |  2 +-
 packages/types/src/sdk/index.ts               |  1 +
 packages/worker/src/sdk/users/users.ts        |  3 ---
 14 files changed, 26 insertions(+), 56 deletions(-)
 delete mode 100644 packages/types/src/core/index.ts
 rename packages/types/src/{core => sdk}/db.ts (100%)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index e9c9261f62..3ca2281bd2 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -64,7 +64,7 @@
   },
   "devDependencies": {
     "@types/chance": "1.1.3",
-    "@types/ioredis": "4.28.10",
+    "@types/ioredis": "4.28.0",
     "@types/jest": "27.5.1",
     "@types/koa": "2.0.52",
     "@types/lodash": "4.14.180",
diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts
index bd346cad64..0fc81da259 100644
--- a/packages/backend-core/src/events/publishers/backup.ts
+++ b/packages/backend-core/src/events/publishers/backup.ts
@@ -1,8 +1,8 @@
-import { AppBackup, AppBackupRevertEvent, Event } from "@budibase/types"
+import { AppBackup, AppBackupRestoreEvent, Event } from "@budibase/types"
 import { publishEvent } from "../events"
 
 export async function appBackupRestored(backup: AppBackup) {
-  const properties: AppBackupRevertEvent = {
+  const properties: AppBackupRestoreEvent = {
     appId: backup.appId,
     backupName: backup.name!,
     backupCreatedAt: backup.timestamp,
diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts
index f264c3a84c..e1975b5d06 100644
--- a/packages/backend-core/src/queue/listeners.ts
+++ b/packages/backend-core/src/queue/listeners.ts
@@ -6,18 +6,18 @@ export type StalledFn = (job: Job) => Promise<void>
 export function addListeners(
   queue: Queue,
   jobQueue: JobQueue,
-  removeStalled?: StalledFn
+  removeStalledCb?: StalledFn
 ) {
   logging(queue, jobQueue)
-  if (removeStalled) {
-    handleStalled(queue, removeStalled)
+  if (removeStalledCb) {
+    handleStalled(queue, removeStalledCb)
   }
 }
 
-function handleStalled(queue: Queue, removeStalled?: StalledFn) {
+function handleStalled(queue: Queue, removeStalledCb?: StalledFn) {
   queue.on("stalled", async (job: Job) => {
-    if (removeStalled) {
-      await removeStalled(job)
+    if (removeStalledCb) {
+      await removeStalledCb(job)
     } else if (job.opts.repeat) {
       const jobId = job.id
       const repeatJobs = await queue.getRepeatableJobs()
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index 7eaafe38a8..e2f9ea9d94 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -18,7 +18,7 @@ async function cleanup() {
 
 export function createQueue<T>(
   jobQueue: JobQueue,
-  removeStalled?: StalledFn
+  opts: { removeStalledCb?: StalledFn }
 ): BullQueue.Queue<T> {
   const queueConfig: any = redisProtocolUrl || { redis: opts }
   let queue: any
@@ -27,7 +27,7 @@ export function createQueue<T>(
   } else {
     queue = new InMemoryQueue(jobQueue, queueConfig)
   }
-  addListeners(queue, jobQueue, removeStalled)
+  addListeners(queue, jobQueue, opts?.removeStalledCb)
   QUEUES.push(queue)
   if (!cleanupInterval) {
     cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS)
diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock
index d2831ca8fe..d301526ba1 100644
--- a/packages/backend-core/yarn.lock
+++ b/packages/backend-core/yarn.lock
@@ -291,11 +291,6 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/types@2.0.30-alpha.3":
-  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2"
-  integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg==
-
 "@hapi/hoek@^9.0.0":
   version "9.3.0"
   resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb"
@@ -698,14 +693,6 @@
     "@types/connect" "*"
     "@types/node" "*"
 
-"@types/bull@^3.15.9":
-  version "3.15.9"
-  resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.9.tgz#e10e0901ec3762bff85716b3c580277960751c93"
-  integrity sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==
-  dependencies:
-    "@types/ioredis" "*"
-    "@types/redis" "^2.8.0"
-
 "@types/chance@1.1.3":
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea"
@@ -776,10 +763,10 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
   integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
 
-"@types/ioredis@*", "@types/ioredis@^4.28.10":
-  version "4.28.10"
-  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
-  integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
+"@types/ioredis@4.28.0":
+  version "4.28.0"
+  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.0.tgz#609b2ea0d91231df2dd7f67dd77436bc72584911"
+  integrity sha512-HSA/JQivJgV0e+353gvgu6WVoWvGRe0HyHOnAN2AvbVIhUlJBhNnnkP8gEEokrDWrxywrBkwo8NuDZ6TVPL9XA==
   dependencies:
     "@types/node" "*"
 
@@ -1547,7 +1534,7 @@ buffer@^5.5.0, buffer@^5.6.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
-bull@^4.10.1:
+bull@4.10.1:
   version "4.10.1"
   resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f"
   integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g==
diff --git a/packages/server/package.json b/packages/server/package.json
index 80d72723fb..88ee16dab3 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -163,7 +163,7 @@
     "@types/google-spreadsheet": "3.1.5",
     "@types/ioredis": "4.28.10",
     "@types/jest": "27.5.1",
-    "@types/koa": "2.13.5",
+    "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.11",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js
index 27b576f181..0a111eae83 100644
--- a/packages/server/src/api/controllers/cloud.js
+++ b/packages/server/src/api/controllers/cloud.js
@@ -35,9 +35,9 @@ async function getAllDocType(db, docType) {
 }
 
 exports.exportApps = async ctx => {
-  // if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
-  //   ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
-  // }
+  if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
+    ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
+  }
   const apps = await getAllApps({ all: true })
   const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentType.USER),
diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts
index 85c01cdb44..8bdc4f8e77 100644
--- a/packages/server/src/sdk/index.ts
+++ b/packages/server/src/sdk/index.ts
@@ -1,13 +1,13 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 
-const toExport = {
+const sdk = {
   backups,
   tables,
 }
 
 // default export for TS
-export default toExport
+export default sdk
 
 // default export for JS
-module.exports = toExport
+module.exports = sdk
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index 4d07fc6450..5fa18745e7 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -2831,7 +2831,7 @@
   dependencies:
     "@types/koa" "*"
 
-"@types/koa@*":
+"@types/koa@*", "@types/koa@2.13.4":
   version "2.13.4"
   resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b"
   integrity sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw==
@@ -2845,20 +2845,6 @@
     "@types/koa-compose" "*"
     "@types/node" "*"
 
-"@types/koa@2.13.5":
-  version "2.13.5"
-  resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.5.tgz#64b3ca4d54e08c0062e89ec666c9f45443b21a61"
-  integrity sha512-HSUOdzKz3by4fnqagwthW/1w/yJspTgppyyalPVbgZf8jQWvdIXcVW5h2DGtw4zYntOaeRGx49r1hxoPWrD4aA==
-  dependencies:
-    "@types/accepts" "*"
-    "@types/content-disposition" "*"
-    "@types/cookies" "*"
-    "@types/http-assert" "*"
-    "@types/http-errors" "*"
-    "@types/keygrip" "*"
-    "@types/koa-compose" "*"
-    "@types/node" "*"
-
 "@types/koa__router@8.0.11":
   version "8.0.11"
   resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.11.tgz#d7b37e6db934fc072ea1baa2ab92bc8ac4564f3e"
diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts
deleted file mode 100644
index 9071393365..0000000000
--- a/packages/types/src/core/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from "./db"
diff --git a/packages/types/src/core/db.ts b/packages/types/src/sdk/db.ts
similarity index 100%
rename from packages/types/src/core/db.ts
rename to packages/types/src/sdk/db.ts
diff --git a/packages/types/src/sdk/events/backup.ts b/packages/types/src/sdk/events/backup.ts
index f3ddafcafc..8d3e2b1afa 100644
--- a/packages/types/src/sdk/events/backup.ts
+++ b/packages/types/src/sdk/events/backup.ts
@@ -1,6 +1,6 @@
 import { BaseEvent } from "./event"
 
-export interface AppBackupRevertEvent extends BaseEvent {
+export interface AppBackupRestoreEvent extends BaseEvent {
   appId: string
   backupName: string
   backupCreatedAt: string
diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts
index 0c374dd105..724b152303 100644
--- a/packages/types/src/sdk/index.ts
+++ b/packages/types/src/sdk/index.ts
@@ -8,3 +8,4 @@ export * from "./search"
 export * from "./koa"
 export * from "./auth"
 export * from "./locks"
+export * from "./db"
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index ce03a12587..b0290507fc 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -79,9 +79,6 @@ export const paginatedUsers = async ({
   } else if (email) {
     userList = await usersCore.searchGlobalUsersByEmail(email, opts)
     property = "email"
-  }
-  if (userIds) {
-    // TODO: search users by userIds
   } else {
     // no search, query allDocs
     const response = await db.allDocs(dbUtils.getGlobalUserParams(null, opts))

From fbed8923dc4ca87e4a56185f7bd4068e9e237a3c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 24 Oct 2022 10:14:35 +0100
Subject: [PATCH 40/46] Moving statistics under backups sdk subfolder.

---
 packages/server/src/sdk/app/backups/backup.ts                   | 2 +-
 packages/server/src/sdk/app/backups/index.ts                    | 2 ++
 .../src/sdk/app/{statistics/index.ts => backups/statistics.ts}  | 0
 3 files changed, 3 insertions(+), 1 deletion(-)
 rename packages/server/src/sdk/app/{statistics/index.ts => backups/statistics.ts} (100%)

diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
index abb2272a99..ac85b2940f 100644
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ b/packages/server/src/sdk/app/backups/backup.ts
@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { exportApp } from "./exports"
 import { importApp } from "./imports"
-import { calculateBackupStats } from "../statistics"
+import { calculateBackupStats } from "./statistics"
 import { Job } from "bull"
 import fs from "fs"
 import env from "../../../environment"
diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts
index 94210c2c4c..c2dd7a7b71 100644
--- a/packages/server/src/sdk/app/backups/index.ts
+++ b/packages/server/src/sdk/app/backups/index.ts
@@ -1,9 +1,11 @@
 import * as exportApps from "./exports"
 import * as importApps from "./imports"
 import * as backup from "./backup"
+import * as statistics from "./statistics"
 
 export default {
   ...exportApps,
   ...importApps,
   ...backup,
+  ...statistics,
 }
diff --git a/packages/server/src/sdk/app/statistics/index.ts b/packages/server/src/sdk/app/backups/statistics.ts
similarity index 100%
rename from packages/server/src/sdk/app/statistics/index.ts
rename to packages/server/src/sdk/app/backups/statistics.ts

From 5340b49d6b15480562b75ff705f56192a68c810d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 24 Oct 2022 12:06:50 +0100
Subject: [PATCH 41/46] Some fixes based on PR comments.

---
 packages/backend-core/src/queue/queue.ts |  6 +++---
 packages/types/src/index.ts              |  1 -
 packages/types/src/sdk/koa.ts            | 20 ++++++++------------
 3 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
index e2f9ea9d94..b4eeeb31aa 100644
--- a/packages/backend-core/src/queue/queue.ts
+++ b/packages/backend-core/src/queue/queue.ts
@@ -4,7 +4,7 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
-const { opts, redisProtocolUrl } = getRedisOptions()
+const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -18,9 +18,9 @@ async function cleanup() {
 
 export function createQueue<T>(
   jobQueue: JobQueue,
-  opts: { removeStalledCb?: StalledFn }
+  opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
-  const queueConfig: any = redisProtocolUrl || { redis: opts }
+  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index 92d2ceb050..4adb2fda97 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -1,4 +1,3 @@
 export * from "./documents"
 export * from "./sdk"
 export * from "./api"
-export * from "./core"
diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts
index 56666f7e2a..8004ba72ae 100644
--- a/packages/types/src/sdk/koa.ts
+++ b/packages/types/src/sdk/koa.ts
@@ -1,4 +1,4 @@
-import { Context } from "koa"
+import { Context, Request } from "koa"
 import { User } from "../documents"
 import { License } from "../sdk"
 
@@ -7,15 +7,11 @@ export interface ContextUser extends User {
   license: License
 }
 
-export interface BBContext {
-  user?: ContextUser
-  status?: number
-  request: {
-    body: any
-  }
-  params: any
-  body?: any
-  redirect?: any
-  attachment: any
-  throw: any
+export interface BBRequest extends Request {
+  body: any
+}
+
+export interface BBContext extends Context {
+  request: BBRequest
+  user?: ContextUser
 }

From 5614c5db6173345986de8a8ac554bbb47921b4b9 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 24 Oct 2022 13:05:40 +0100
Subject: [PATCH 42/46] Replace isTest check with mock for triggering app
 backup

---
 packages/server/specs/openapi.json            |  3 ++
 packages/server/specs/openapi.yaml            |  3 ++
 .../src/api/controllers/deploy/index.ts       | 16 ++------
 .../src/api/routes/tests/deployment.spec.js   | 25 ------------
 .../src/api/routes/tests/deployment.spec.ts   | 40 +++++++++++++++++++
 .../src/tests/utilities/TestConfiguration.js  |  7 +++-
 6 files changed, 55 insertions(+), 39 deletions(-)
 delete mode 100644 packages/server/src/api/routes/tests/deployment.spec.js
 create mode 100644 packages/server/src/api/routes/tests/deployment.spec.ts

diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json
index ce410823ec..62f59e9113 100644
--- a/packages/server/specs/openapi.json
+++ b/packages/server/specs/openapi.json
@@ -783,6 +783,7 @@
                       "type": "string",
                       "enum": [
                         "string",
+                        "barcodeqr",
                         "longform",
                         "options",
                         "number",
@@ -986,6 +987,7 @@
                           "type": "string",
                           "enum": [
                             "string",
+                            "barcodeqr",
                             "longform",
                             "options",
                             "number",
@@ -1200,6 +1202,7 @@
                             "type": "string",
                             "enum": [
                               "string",
+                              "barcodeqr",
                               "longform",
                               "options",
                               "number",
diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml
index ed13ac01f4..25069f40a4 100644
--- a/packages/server/specs/openapi.yaml
+++ b/packages/server/specs/openapi.yaml
@@ -579,6 +579,7 @@ components:
                     type: string
                     enum:
                       - string
+                      - barcodeqr
                       - longform
                       - options
                       - number
@@ -741,6 +742,7 @@ components:
                         type: string
                         enum:
                           - string
+                          - barcodeqr
                           - longform
                           - options
                           - number
@@ -910,6 +912,7 @@ components:
                           type: string
                           enum:
                             - string
+                            - barcodeqr
                             - longform
                             - options
                             - number
diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts
index b3b875e397..a1cb905930 100644
--- a/packages/server/src/api/controllers/deploy/index.ts
+++ b/packages/server/src/api/controllers/deploy/index.ts
@@ -20,7 +20,6 @@ import {
 import { events } from "@budibase/backend-core"
 import { backups } from "@budibase/pro"
 import { AppBackupTrigger } from "@budibase/types"
-import env from "../../../environment"
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -108,17 +107,10 @@ async function deployApp(deployment: any, userId: string) {
     const devAppId = getDevelopmentAppID(appId)
     const productionAppId = getProdAppID(appId)
 
-    // can't do this in test
-    if (!env.isTest()) {
-      // trigger backup initially
-      await backups.triggerAppBackup(
-        productionAppId,
-        AppBackupTrigger.PUBLISH,
-        {
-          createdBy: userId,
-        }
-      )
-    }
+    // trigger backup initially
+    await backups.triggerAppBackup(productionAppId, AppBackupTrigger.PUBLISH, {
+      createdBy: userId,
+    })
 
     const config: any = {
       source: devAppId,
diff --git a/packages/server/src/api/routes/tests/deployment.spec.js b/packages/server/src/api/routes/tests/deployment.spec.js
deleted file mode 100644
index be126fa239..0000000000
--- a/packages/server/src/api/routes/tests/deployment.spec.js
+++ /dev/null
@@ -1,25 +0,0 @@
-const setup = require("./utilities")
-const { events } = require("@budibase/backend-core")
-
-describe("/deployments", () => {
-  let request = setup.getRequest()
-  let config = setup.getConfig()
-
-  afterAll(setup.afterAll)
-
-  beforeEach(async () => {
-    await config.init()
-    jest.clearAllMocks()
-  })
-
-  describe("deploy", () => {
-    it("should deploy the application", async () => {
-      await request
-        .post(`/api/deploy`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(events.app.published.mock.calls.length).toBe(1)
-    })
-  })
-})
\ No newline at end of file
diff --git a/packages/server/src/api/routes/tests/deployment.spec.ts b/packages/server/src/api/routes/tests/deployment.spec.ts
new file mode 100644
index 0000000000..e659ff1a69
--- /dev/null
+++ b/packages/server/src/api/routes/tests/deployment.spec.ts
@@ -0,0 +1,40 @@
+const triggerAppBackupMock = jest.fn()
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  backups: {
+    triggerAppBackup: triggerAppBackupMock,
+    addAppBackupProcessors: jest.fn(),
+  },
+}))
+import setup from "./utilities"
+import { events } from "@budibase/backend-core"
+import { AppBackupTrigger } from "@budibase/types"
+
+describe("/deployments", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    await config.init()
+    jest.clearAllMocks()
+  })
+
+  describe("deploy", () => {
+    it("should deploy the application", async () => {
+      await request
+        .post(`/api/deploy`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(triggerAppBackupMock).toBeCalledTimes(1)
+      expect(triggerAppBackupMock).toBeCalledWith(
+        config.prodAppId,
+        AppBackupTrigger.PUBLISH,
+        { createdBy: config.userMetadataId }
+      )
+      expect((events.app.published as jest.Mock).mock.calls.length).toBe(1)
+    })
+  })
+})
diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js
index 38aa84be2d..097b2eabaf 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.js
+++ b/packages/server/src/tests/utilities/TestConfiguration.js
@@ -25,7 +25,7 @@ const newid = require("../../db/newid")
 const context = require("@budibase/backend-core/context")
 const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
 const { encrypt } = require("@budibase/backend-core/encryption")
-const { DocumentType } = require("../../db/utils")
+const { DocumentType, generateUserMetadataID } = require("../../db/utils")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -95,7 +95,10 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
-    await this.globalUser()
+    this.user = await this.globalUser()
+    this.globalUserId = this.user._id
+    this.userMetadataId = generateUserMetadataID(this.globalUserId)
+
     return this.createApp(appName)
   }
 

From bc2d499625b2cf5ffa101934fd1f93ba104a769e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 24 Oct 2022 13:33:36 +0100
Subject: [PATCH 43/46] Adding a check to see if app backups feature is
 enabled.

---
 .../src/api/controllers/deploy/index.ts       | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts
index b3b875e397..f1700aef87 100644
--- a/packages/server/src/api/controllers/deploy/index.ts
+++ b/packages/server/src/api/controllers/deploy/index.ts
@@ -110,14 +110,17 @@ async function deployApp(deployment: any, userId: string) {
 
     // can't do this in test
     if (!env.isTest()) {
-      // trigger backup initially
-      await backups.triggerAppBackup(
-        productionAppId,
-        AppBackupTrigger.PUBLISH,
-        {
-          createdBy: userId,
-        }
-      )
+      // don't try this if feature isn't allowed, will error
+      if (!(await backups.isEnabled())) {
+        // trigger backup initially
+        await backups.triggerAppBackup(
+          productionAppId,
+          AppBackupTrigger.PUBLISH,
+          {
+            createdBy: userId,
+          }
+        )
+      }
     }
 
     const config: any = {

From d746c425034a4d23ac05f2b256cd7ba9c89e32d1 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 24 Oct 2022 14:23:16 +0100
Subject: [PATCH 44/46] Remove pro mock and invert if condition on backups
 enabled

---
 .../server/src/api/controllers/deploy/index.ts    |  2 +-
 .../src/api/routes/tests/deployment.spec.ts       | 15 ---------------
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts
index 4a7cc5a754..cb4534a1a5 100644
--- a/packages/server/src/api/controllers/deploy/index.ts
+++ b/packages/server/src/api/controllers/deploy/index.ts
@@ -108,7 +108,7 @@ async function deployApp(deployment: any, userId: string) {
     const productionAppId = getProdAppID(appId)
 
     // don't try this if feature isn't allowed, will error
-    if (!(await backups.isEnabled())) {
+    if (await backups.isEnabled()) {
       // trigger backup initially
       await backups.triggerAppBackup(
         productionAppId,
diff --git a/packages/server/src/api/routes/tests/deployment.spec.ts b/packages/server/src/api/routes/tests/deployment.spec.ts
index e659ff1a69..0219e3f2b4 100644
--- a/packages/server/src/api/routes/tests/deployment.spec.ts
+++ b/packages/server/src/api/routes/tests/deployment.spec.ts
@@ -1,14 +1,5 @@
-const triggerAppBackupMock = jest.fn()
-jest.mock("@budibase/pro", () => ({
-  ...jest.requireActual("@budibase/pro"),
-  backups: {
-    triggerAppBackup: triggerAppBackupMock,
-    addAppBackupProcessors: jest.fn(),
-  },
-}))
 import setup from "./utilities"
 import { events } from "@budibase/backend-core"
-import { AppBackupTrigger } from "@budibase/types"
 
 describe("/deployments", () => {
   let request = setup.getRequest()
@@ -28,12 +19,6 @@ describe("/deployments", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(triggerAppBackupMock).toBeCalledTimes(1)
-      expect(triggerAppBackupMock).toBeCalledWith(
-        config.prodAppId,
-        AppBackupTrigger.PUBLISH,
-        { createdBy: config.userMetadataId }
-      )
       expect((events.app.published as jest.Mock).mock.calls.length).toBe(1)
     })
   })

From 39c83d8fb13dc26d2712e5f44cc0fbd16e151802 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 24 Oct 2022 15:28:43 +0100
Subject: [PATCH 45/46] Move backup processing into pro

---
 packages/server/src/app.ts                    |  15 +-
 packages/server/src/sdk/app/backups/backup.ts | 145 ------------------
 packages/server/src/sdk/app/backups/index.ts  |   2 -
 3 files changed, 14 insertions(+), 148 deletions(-)
 delete mode 100644 packages/server/src/sdk/app/backups/backup.ts

diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts
index 776adb602d..330adadd3d 100644
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@@ -38,6 +38,7 @@ import {
 import { watch } from "./watch"
 import { initialise as initialiseWebsockets } from "./websocket"
 import sdk from "./sdk"
+import * as pro from "@budibase/pro"
 
 const app = new Koa()
 
@@ -103,13 +104,25 @@ server.on("close", async () => {
   }
 })
 
+const initPro = async () => {
+  await pro.init({
+    backups: {
+      processing: {
+        exportAppFn: sdk.backups.exportApp,
+        importAppFn: sdk.backups.importApp,
+        statsFn: sdk.backups.calculateBackupStats,
+      },
+    },
+  })
+}
+
 module.exports = server.listen(env.PORT || 0, async () => {
   console.log(`Budibase running on ${JSON.stringify(server.address())}`)
   env._set("PORT", server.address().port)
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
-  await sdk.backups.init()
+  await initPro()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment
diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts
deleted file mode 100644
index ac85b2940f..0000000000
--- a/packages/server/src/sdk/app/backups/backup.ts
+++ /dev/null
@@ -1,145 +0,0 @@
-import { backups } from "@budibase/pro"
-import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core"
-import {
-  AppBackupQueueData,
-  AppBackupStatus,
-  AppBackupTrigger,
-  AppBackupType,
-} from "@budibase/types"
-import { exportApp } from "./exports"
-import { importApp } from "./imports"
-import { calculateBackupStats } from "./statistics"
-import { Job } from "bull"
-import fs from "fs"
-import env from "../../../environment"
-
-type BackupOpts = {
-  doc?: { id: string; rev: string }
-  createdBy?: string
-}
-
-async function removeExistingApp(devId: string) {
-  const devDb = dbCore.dangerousGetDB(devId, { skip_setup: true })
-  await devDb.destroy()
-}
-
-async function runBackup(
-  name: string,
-  trigger: AppBackupTrigger,
-  tenantId: string,
-  appId: string,
-  opts?: BackupOpts
-) {
-  const devAppId = dbCore.getDevAppID(appId),
-    prodAppId = dbCore.getProdAppID(appId)
-  const timestamp = new Date().toISOString()
-  const tarPath = await exportApp(devAppId, { tar: true })
-  const contents = await calculateBackupStats(devAppId)
-  let filename = `${prodAppId}/backup-${timestamp}.tar.gz`
-  // add the tenant to the bucket path if backing up within a multi-tenant environment
-  if (env.MULTI_TENANCY) {
-    filename = `${tenantId}/${filename}`
-  }
-  const bucket = objectStore.ObjectStoreBuckets.BACKUPS
-  await objectStore.upload({
-    path: tarPath,
-    type: "application/gzip",
-    bucket,
-    filename,
-    metadata: {
-      name,
-      trigger,
-      timestamp,
-      appId: prodAppId,
-    },
-  })
-  if (opts?.doc) {
-    await backups.updateBackupStatus(
-      opts.doc.id,
-      opts.doc.rev,
-      AppBackupStatus.COMPLETE,
-      contents,
-      filename
-    )
-  } else {
-    await backups.storeAppBackupMetadata(
-      {
-        appId: prodAppId,
-        timestamp,
-        name,
-        trigger,
-        type: AppBackupType.BACKUP,
-        status: AppBackupStatus.COMPLETE,
-        contents,
-        createdBy: opts?.createdBy,
-      },
-      { filename }
-    )
-  }
-  // clear up the tarball after uploading it
-  fs.rmSync(tarPath)
-}
-
-async function importProcessor(job: Job) {
-  const data: AppBackupQueueData = job.data
-  const appId = data.appId,
-    backupId = data.import!.backupId,
-    nameForBackup = data.import!.nameForBackup,
-    createdBy = data.import!.createdBy
-  const tenantId = tenancy.getTenantIDFromAppID(appId) as string
-  tenancy.doInTenant(tenantId, async () => {
-    const devAppId = dbCore.getDevAppID(appId)
-    const { rev } = await backups.updateRestoreStatus(
-      data.docId,
-      data.docRev,
-      AppBackupStatus.PENDING
-    )
-    // initially export the current state to disk - incase something goes wrong
-    await runBackup(
-      nameForBackup,
-      AppBackupTrigger.RESTORING,
-      tenantId,
-      appId,
-      { createdBy }
-    )
-    // get the backup ready on disk
-    const { path } = await backups.downloadAppBackup(backupId)
-    // start by removing app database and contents of bucket - which will be updated
-    await removeExistingApp(devAppId)
-    let status = AppBackupStatus.COMPLETE
-    try {
-      await importApp(devAppId, dbCore.dangerousGetDB(devAppId), {
-        file: {
-          type: "application/gzip",
-          path,
-        },
-        key: path,
-      })
-    } catch (err) {
-      status = AppBackupStatus.FAILED
-    }
-    await backups.updateRestoreStatus(data.docId, rev, status)
-  })
-}
-
-async function exportProcessor(job: Job) {
-  const data: AppBackupQueueData = job.data
-  const appId = data.appId,
-    trigger = data.export!.trigger,
-    name = data.export!.name || `${trigger} - backup`
-  const tenantId = tenancy.getTenantIDFromAppID(appId) as string
-  await tenancy.doInTenant(tenantId, async () => {
-    const { rev } = await backups.updateBackupStatus(
-      data.docId,
-      data.docRev,
-      AppBackupStatus.PENDING
-    )
-    return runBackup(name, trigger, tenantId, appId, {
-      doc: { id: data.docId, rev },
-    })
-  })
-}
-
-export async function init() {
-  await backups.addAppBackupProcessors(importProcessor, exportProcessor)
-}
diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts
index c2dd7a7b71..8e5697c53c 100644
--- a/packages/server/src/sdk/app/backups/index.ts
+++ b/packages/server/src/sdk/app/backups/index.ts
@@ -1,11 +1,9 @@
 import * as exportApps from "./exports"
 import * as importApps from "./imports"
-import * as backup from "./backup"
 import * as statistics from "./statistics"
 
 export default {
   ...exportApps,
   ...importApps,
-  ...backup,
   ...statistics,
 }

From ae8145baecf114d9b96244ba03668fda32f486c8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 24 Oct 2022 15:43:48 +0100
Subject: [PATCH 46/46] Adding in retention days for app backups.

---
 packages/types/src/sdk/licensing/quota.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 74777d4590..86a092c6f5 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -25,6 +25,7 @@ export enum MonthlyQuotaName {
 
 export enum ConstantQuotaName {
   AUTOMATION_LOG_RETENTION_DAYS = "automationLogRetentionDays",
+  APP_BACKUPS_RETENTION_DAYS = "appBackupRetentionDays",
 }
 
 export type MeteredQuotaName = StaticQuotaName | MonthlyQuotaName
@@ -76,6 +77,7 @@ export type StaticQuotas = {
 
 export type ConstantQuotas = {
   [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota
+  [ConstantQuotaName.APP_BACKUPS_RETENTION_DAYS]: Quota
 }
 
 export type Quotas = {