From ef52bde670d5b73e5a742216de6573014d9d0af5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 6 Oct 2022 19:10:45 +0100 Subject: [PATCH] Building out initial SDK work - converting some existing exporting work to typescript. --- .../api/controllers/{backup.js => backup.ts} | 12 +- packages/server/src/api/controllers/cloud.js | 11 +- packages/server/src/api/routes/backup.js | 10 -- packages/server/src/api/routes/backup.ts | 10 ++ packages/server/src/sdk/app/export.ts | 113 ++++++++++++++++++ packages/server/src/sdk/app/index.ts | 1 + packages/server/src/sdk/index.ts | 5 + .../server/src/utilities/fileSystem/index.js | 106 ---------------- 8 files changed, 139 insertions(+), 129 deletions(-) rename packages/server/src/api/controllers/{backup.js => backup.ts} (56%) delete mode 100644 packages/server/src/api/routes/backup.js create mode 100644 packages/server/src/api/routes/backup.ts create mode 100644 packages/server/src/sdk/app/export.ts create mode 100644 packages/server/src/sdk/app/index.ts create mode 100644 packages/server/src/sdk/index.ts diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.ts similarity index 56% rename from packages/server/src/api/controllers/backup.js rename to packages/server/src/api/controllers/backup.ts index 5e08c823ed..cd2ea415a4 100644 --- a/packages/server/src/api/controllers/backup.js +++ b/packages/server/src/api/controllers/backup.ts @@ -1,15 +1,15 @@ -const { streamBackup } = require("../../utilities/fileSystem") -const { events, context } = require("@budibase/backend-core") -const { DocumentType } = require("../../db/utils") -const { isQsTrue } = require("../../utilities") +import sdk from "../../sdk" +import { events, context } from "@budibase/backend-core" +import { DocumentType } from "../../db/utils" +import { isQsTrue } from "../../utilities" -exports.exportAppDump = async function (ctx) { +export async function exportAppDump(ctx: any) { let { appId, excludeRows } = ctx.query const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` ctx.attachment(backupIdentifier) - ctx.body = await streamBackup(appId, excludeRows) + ctx.body = await sdk.apps.exports.streamBackup(appId, excludeRows) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 1e6abb1d3b..55aa6bb548 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -1,14 +1,11 @@ const env = require("../../environment") const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db") -const { - exportDB, - sendTempFile, - readFileSync, -} = require("../../utilities/fileSystem") +const { sendTempFile, readFileSync } = require("../../utilities/fileSystem") const { stringToReadStream } = require("../../utilities") const { getGlobalDB } = require("@budibase/backend-core/tenancy") const { create } = require("./application") const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils") +const sdk = require("../../sdk") async function createApp(appName, appImport) { const ctx = { @@ -27,7 +24,7 @@ exports.exportApps = async ctx => { ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") } const apps = await getAllApps({ all: true }) - const globalDBString = await exportDB(getGlobalDBName(), { + const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), { filter: doc => !doc._id.startsWith(DocumentType.USER), }) let allDBs = { @@ -38,7 +35,7 @@ exports.exportApps = async ctx => { // only export the dev apps as they will be the latest, the user can republish the apps // in their self hosted environment if (isDevAppID(appId)) { - allDBs[app.name] = await exportDB(appId) + allDBs[app.name] = await sdk.apps.exports.exportDB(appId) } } const filename = `cloud-export-${new Date().getTime()}.txt` diff --git a/packages/server/src/api/routes/backup.js b/packages/server/src/api/routes/backup.js deleted file mode 100644 index 9f3b27e95a..0000000000 --- a/packages/server/src/api/routes/backup.js +++ /dev/null @@ -1,10 +0,0 @@ -const Router = require("@koa/router") -const controller = require("../controllers/backup") -const authorized = require("../../middleware/authorized") -const { BUILDER } = require("@budibase/backend-core/permissions") - -const router = new Router() - -router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump) - -module.exports = router diff --git a/packages/server/src/api/routes/backup.ts b/packages/server/src/api/routes/backup.ts new file mode 100644 index 0000000000..2473fa9f67 --- /dev/null +++ b/packages/server/src/api/routes/backup.ts @@ -0,0 +1,10 @@ +import Router from "@koa/router" +import * as controller from "../controllers/backup" +import authorized from "../../middleware/authorized" +import { BUILDER } from "@budibase/backend-core/permissions" + +const router = new Router() + +router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump) + +export default router diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts new file mode 100644 index 0000000000..70c4446273 --- /dev/null +++ b/packages/server/src/sdk/app/export.ts @@ -0,0 +1,113 @@ +import { closeDB, dangerousGetDB, doWithDB } from "@budibase/backend-core/db" +import { budibaseTempDir } from "../../utilities/budibaseDir" +import { streamUpload } from "../../utilities/fileSystem/utilities" +import { ObjectStoreBuckets } from "../../constants" +import { + LINK_USER_METADATA_PREFIX, + TABLE_ROW_PREFIX, + USER_METDATA_PREFIX, +} from "../../db/utils" +import fs from "fs" +import env from "../../environment" +import { join } from "path" +const MemoryStream = require("memorystream") + +/** + * Exports a DB to either file or a variable (memory). + * @param {string} dbName the DB which is to be exported. + * @param {object} opts various options for the export, e.g. whether to stream, + * a filter function or the name of the export. + * @return {*} either a readable stream or a string + */ +export async function exportDB( + dbName: string, + opts: { stream?: boolean; filter?: any; exportName?: string } = {} +) { + // streaming a DB dump is a bit more complicated, can't close DB + if (opts?.stream) { + const db = dangerousGetDB(dbName) + const memStream = new MemoryStream() + memStream.on("end", async () => { + await closeDB(db) + }) + db.dump(memStream, { filter: opts?.filter }) + return memStream + } + + return doWithDB(dbName, async (db: any) => { + // Write the dump to file if required + if (opts?.exportName) { + const path = join(budibaseTempDir(), opts?.exportName) + const writeStream = fs.createWriteStream(path) + await db.dump(writeStream, { filter: opts?.filter }) + + // Upload the dump to the object store if self-hosted + if (env.SELF_HOSTED) { + await streamUpload( + ObjectStoreBuckets.BACKUPS, + join(dbName, opts?.exportName), + fs.createReadStream(path) + ) + } + + return fs.createReadStream(path) + } + + // Stringify the dump in memory if required + const memStream = new MemoryStream() + let appString = "" + memStream.on("data", (chunk: any) => { + appString += chunk.toString() + }) + await db.dump(memStream, { filter: opts?.filter }) + return appString + }) +} + +function defineFilter(excludeRows?: boolean) { + const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] + if (excludeRows) { + ids.push(TABLE_ROW_PREFIX) + } + return (doc: any) => + !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) +} + +/** + * Local utility to back up the database state for an app, excluding global user + * data or user relationships. + * @param {string} appId The app to back up + * @param {object} config Config to send to export DB + * @param {boolean} excludeRows Flag to state whether the export should include data. + * @returns {*} either a string or a stream of the backup + */ +async function backupAppData( + appId: string, + config: any, + excludeRows?: boolean +) { + return await exportDB(appId, { + ...config, + filter: defineFilter(excludeRows), + }) +} + +/** + * Streams a backup of the database state for an app + * @param {string} appId The ID of the app which is to be backed up. + * @param {boolean} excludeRows Flag to state whether the export should include data. + * @returns {*} a readable stream of the backup which is written in real time + */ +export async function streamBackup(appId: string, excludeRows: boolean) { + return await backupAppData(appId, { stream: true }, excludeRows) +} + +/** + * Takes a copy of the database state for an app to the object store. + * @param {string} appId The ID of the app which is to be backed up. + * @param {string} backupName The name of the backup located in the object store. + * @return {*} a readable stream to the completed backup file + */ +export async function performBackup(appId: string, backupName: string) { + return await backupAppData(appId, { exportName: backupName }) +} diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts new file mode 100644 index 0000000000..3927539bc8 --- /dev/null +++ b/packages/server/src/sdk/app/index.ts @@ -0,0 +1 @@ +export * as exports from "./export" diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts new file mode 100644 index 0000000000..9199c1cc72 --- /dev/null +++ b/packages/server/src/sdk/index.ts @@ -0,0 +1,5 @@ +import * as apps from "./app" + +export default { + apps, +} diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 4e9b13cca0..656ba4816c 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -2,17 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") const uuid = require("uuid/v4") -const { - doWithDB, - dangerousGetDB, - closeDB, -} = require("@budibase/backend-core/db") const { ObjectStoreBuckets } = require("../../constants") const { upload, retrieve, retrieveToTmp, - streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, @@ -21,12 +15,6 @@ const { const { updateClientLibrary } = require("./clientLibrary") const { checkSlashesInUrl } = require("../") const env = require("../../environment") -const { - USER_METDATA_PREFIX, - LINK_USER_METADATA_PREFIX, - TABLE_ROW_PREFIX, -} = require("../../db/utils") -const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") @@ -124,100 +112,6 @@ exports.apiFileReturn = contents => { return fs.createReadStream(path) } -exports.defineFilter = excludeRows => { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] - if (excludeRows) { - ids.push(TABLE_ROW_PREFIX) - } - return doc => - !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) -} - -/** - * Local utility to back up the database state for an app, excluding global user - * data or user relationships. - * @param {string} appId The app to backup - * @param {object} config Config to send to export DB - * @param {boolean} excludeRows Flag to state whether the export should include data. - * @returns {*} either a string or a stream of the backup - */ -const backupAppData = async (appId, config, excludeRows) => { - return await exports.exportDB(appId, { - ...config, - filter: exports.defineFilter(excludeRows), - }) -} - -/** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return {*} a readable stream to the completed backup file - */ -exports.performBackup = async (appId, backupName) => { - return await backupAppData(appId, { exportName: backupName }) -} - -/** - * Streams a backup of the database state for an app - * @param {string} appId The ID of the app which is to be backed up. - * @param {boolean} excludeRows Flag to state whether the export should include data. - * @returns {*} a readable stream of the backup which is written in real time - */ -exports.streamBackup = async (appId, excludeRows) => { - return await backupAppData(appId, { stream: true }, excludeRows) -} - -/** - * Exports a DB to either file or a variable (memory). - * @param {string} dbName the DB which is to be exported. - * @param {string} exportName optional - provide a filename to write the backup to a file - * @param {boolean} stream optional - whether to perform a full backup - * @param {function} filter optional - a filter function to clear out any un-wanted docs. - * @return {*} either a readable stream or a string - */ -exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { - // streaming a DB dump is a bit more complicated, can't close DB - if (stream) { - const db = dangerousGetDB(dbName) - const memStream = new MemoryStream() - memStream.on("end", async () => { - await closeDB(db) - }) - db.dump(memStream, { filter }) - return memStream - } - - return doWithDB(dbName, async db => { - // Write the dump to file if required - if (exportName) { - const path = join(budibaseTempDir(), exportName) - const writeStream = fs.createWriteStream(path) - await db.dump(writeStream, { filter }) - - // Upload the dump to the object store if self hosted - if (env.SELF_HOSTED) { - await streamUpload( - ObjectStoreBuckets.BACKUPS, - join(dbName, exportName), - fs.createReadStream(path) - ) - } - - return fs.createReadStream(path) - } - - // Stringify the dump in memory if required - const memStream = new MemoryStream() - let appString = "" - memStream.on("data", chunk => { - appString += chunk.toString() - }) - await db.dump(memStream, { filter }) - return appString - }) -} - /** * Writes the provided contents to a temporary file, which can be used briefly. * @param {string} fileContents contents which will be written to a temp file.