From 0bd2a18e46f09c51a75d19f6161a1a3c0bfb358a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Oct 2022 19:24:03 +0100 Subject: [PATCH] Main types and work for the CRUD operations of app backup backend in pro + the listeners to handle exporting apps from the server. --- packages/backend-core/package.json | 1 + packages/backend-core/src/db/constants.ts | 2 + .../backend-core/src/objectStore/index.ts | 3 +- packages/backend-core/src/queue/constants.ts | 4 +- packages/backend-core/src/queue/listeners.ts | 4 +- packages/backend-core/src/queue/queue.ts | 4 +- packages/backend-core/yarn.lock | 10 ++++- packages/server/src/app.ts | 2 + packages/server/src/automations/bullboard.js | 2 +- packages/server/src/sdk/app/backups/backup.ts | 39 +++++++++++++++++++ packages/server/src/sdk/app/backups/index.ts | 2 + packages/types/src/api/web/app/backup.ts | 1 + packages/types/src/documents/app/backup.ts | 37 +++++++++++++----- 13 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 packages/server/src/sdk/app/backups/backup.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index eb90fa4159..17977decba 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -63,6 +63,7 @@ ] }, "devDependencies": { + "@types/bull": "^3.15.9", "@types/chance": "1.1.3", "@types/ioredis": "^4.28.10", "@types/jest": "27.5.1", diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index a61e8a2af2..8f8f45638b 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -21,6 +21,7 @@ export enum ViewName { ACCOUNT_BY_EMAIL = "account_by_email", PLATFORM_USERS_LOWERCASE = "platform_users_lowercase", USER_BY_GROUP = "by_group_user", + APP_BACKUP_BY_TRIGGER = "by_trigger", } export const DeprecatedViews = { @@ -49,6 +50,7 @@ export enum DocumentType { TABLE = "ta", DATASOURCE = "datasource", DATASOURCE_PLUS = "datasource_plus", + APP_BACKUP = "backup", } export const StaticDatabases = { diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index e210f9dccc..8453c9aee6 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -27,6 +27,7 @@ const CONTENT_TYPE_MAP: any = { css: "text/css", js: "application/javascript", json: "application/json", + gz: "application/gzip", } const STRING_CONTENT_TYPES = [ CONTENT_TYPE_MAP.html, @@ -149,7 +150,7 @@ export const upload = async ({ type, metadata, }: any) => { - const extension = [...filename.split(".")].pop() + const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) const objectStore = ObjectStore(bucketName) diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index d8fb3121a3..e8323dacb8 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -1,4 +1,4 @@ export enum JobQueue { - AUTOMATIONS = "automationQueue", - APP_BACKUPS = "appBackupQueue", + AUTOMATION = "automationQueue", + APP_BACKUP = "appBackupQueue", } diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 0e06fb5ef0..f264c3a84c 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -34,10 +34,10 @@ function handleStalled(queue: Queue, removeStalled?: StalledFn) { function logging(queue: Queue, jobQueue: JobQueue) { let eventType: string switch (jobQueue) { - case JobQueue.AUTOMATIONS: + case JobQueue.AUTOMATION: eventType = "automation-event" break - case JobQueue.APP_BACKUPS: + case JobQueue.APP_BACKUP: eventType = "app-backup-event" break } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index d83e421456..6dd10ee091 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -16,10 +16,10 @@ async function cleanup() { } } -export function createQueue( +export function createQueue( jobQueue: JobQueue, removeStalled?: StalledFn -): BullQueue.Queue { +): BullQueue.Queue { const queueConfig: any = redisProtocolUrl || { redis: opts } let queue: any if (env.isTest()) { diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 22b93d7b3e..d2831ca8fe 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -698,6 +698,14 @@ "@types/connect" "*" "@types/node" "*" +"@types/bull@^3.15.9": + version "3.15.9" + resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.9.tgz#e10e0901ec3762bff85716b3c580277960751c93" + integrity sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ== + dependencies: + "@types/ioredis" "*" + "@types/redis" "^2.8.0" + "@types/chance@1.1.3": version "1.1.3" resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" @@ -768,7 +776,7 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== -"@types/ioredis@^4.28.10": +"@types/ioredis@*", "@types/ioredis@^4.28.10": version "4.28.10" resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 3bfe8976f7..776adb602d 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -37,6 +37,7 @@ import { } from "./utilities/workerRequests" import { watch } from "./watch" import { initialise as initialiseWebsockets } from "./websocket" +import sdk from "./sdk" const app = new Koa() @@ -108,6 +109,7 @@ module.exports = server.listen(env.PORT || 0, async () => { eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() + await sdk.backups.init() // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index af2243f13d..c4f33e07a9 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -5,7 +5,7 @@ const { queue } = require("@budibase/backend-core") const automation = require("../threads/automation") let automationQueue = queue.createQueue( - queue.JobQueue.AUTOMATIONS, + queue.JobQueue.AUTOMATION, automation.removeStalled ) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts new file mode 100644 index 0000000000..6a0d370653 --- /dev/null +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -0,0 +1,39 @@ +import { backups } from "@budibase/pro" +import { objectStore, tenancy } from "@budibase/backend-core" +import { exportApp } from "./exports" +import { Job } from "bull" +import fs from "fs" +import env from "../../../environment" + +export async function init() { + await backups.addAppBackupProcessor(async (job: Job) => { + const appId = job.data.appId, + trigger = job.data.trigger, + name = job.data.name + const createdAt = new Date().toISOString() + const tarPath = await exportApp(appId, { tar: true }) + let filename = `${appId}/backup-${createdAt}.tar.gz` + // add the tenant to the bucket path if backing up within a multi-tenant environment + if (env.MULTI_TENANCY) { + const tenantId = tenancy.getTenantIDFromAppID(appId) + filename = `${tenantId}/${filename}` + } + const bucket = objectStore.ObjectStoreBuckets.BACKUPS + const metadata = { + appId, + createdAt, + trigger, + name, + } + await objectStore.upload({ + path: tarPath, + type: "application/gzip", + bucket, + filename, + metadata, + }) + await backups.storeAppBackupMetadata(filename, metadata) + // clear up the tarball after uploading it + fs.rmSync(tarPath) + }) +} diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts index fe7e4e1049..94210c2c4c 100644 --- a/packages/server/src/sdk/app/backups/index.ts +++ b/packages/server/src/sdk/app/backups/index.ts @@ -1,7 +1,9 @@ import * as exportApps from "./exports" import * as importApps from "./imports" +import * as backup from "./backup" export default { ...exportApps, ...importApps, + ...backup, } diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index 57ffba0d70..092cf714b5 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -4,6 +4,7 @@ export interface SearchAppBackupsRequest { trigger: AppBackupTrigger startDate: string endDate: string + page?: string } export interface CreateAppBackupRequest { diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index a935ed5ba1..fb8ef61e3a 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -6,16 +6,35 @@ export enum AppBackupTrigger { SCHEDULED = "scheduled", } -export interface AppBackupContents { - datasources: string[] - screens: string[] - automations: string[] -} - export interface AppBackup extends Document { trigger: AppBackupTrigger name: string - date: string - userId: string - contents: AppBackupContents + createdAt: string + filename: string + appId: string + userId?: string + contents?: { + datasources: string[] + screens: string[] + automations: string[] + } +} + +export type AppBackupFetchOpts = { + trigger?: AppBackupTrigger + limit?: number + page?: string + paginate?: boolean + startDate?: string + endDate?: string +} + +export interface AppBackupQueueData { + trigger: AppBackupTrigger + name?: string + appId: string +} + +export interface AppBackupMetadata extends AppBackupQueueData { + createdAt: string }