From bb65cf8ef90b23f9bed7a1cb840da5af4c743454 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Oct 2022 20:08:59 +0100 Subject: [PATCH] Some updates towards supporting attachments in app exports. --- .../backend-core/src/objectStore/index.ts | 64 +++++++++++++++++-- packages/builder/src/stores/portal/apps.js | 13 ++++ packages/server/src/api/controllers/backup.ts | 2 +- packages/server/src/api/controllers/cloud.js | 2 +- .../src/api/controllers/static/index.ts | 4 +- packages/server/src/constants/index.js | 6 +- packages/server/src/sdk/app/export.ts | 59 ++++++++--------- .../src/utilities/fileSystem/utilities.js | 2 + packages/worker/src/sdk/users/users.ts | 11 +--- 9 files changed, 110 insertions(+), 53 deletions(-) diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index 17e002cc49..903ba28ed1 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -18,6 +18,10 @@ const STATE = { bucketCreationPromises: {}, } +type ListParams = { + ContinuationToken?: string +} + const CONTENT_TYPE_MAP: any = { html: "text/html", css: "text/css", @@ -93,7 +97,7 @@ export const ObjectStore = (bucket: any) => { * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: any) => { +export const makeSureBucketExists = async (client: any, bucketName: string) => { bucketName = sanitizeBucket(bucketName) try { await client @@ -168,8 +172,8 @@ export const upload = async ({ * through to the object store. */ export const streamUpload = async ( - bucketName: any, - filename: any, + bucketName: string, + filename: string, stream: any, extra = {} ) => { @@ -202,7 +206,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: any, filepath: any) => { +export const retrieve = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -217,10 +221,38 @@ export const retrieve = async (bucketName: any, filepath: any) => { } } +export const listAllObjects = async (bucketName: string, path: string) => { + const objectStore = ObjectStore(bucketName) + const list = (params: ListParams = {}) => { + return objectStore + .listObjectsV2({ + ...params, + Bucket: sanitizeBucket(bucketName), + Prefix: sanitizeKey(path), + }) + .promise() + } + let isTruncated = false, + token, + objects: AWS.S3.Types.Object[] = [] + do { + let params: ListParams = {} + if (token) { + params.ContinuationToken = token + } + const response = await list(params) + if (response.Contents) { + objects = objects.concat(response.Contents) + } + isTruncated = !!response.IsTruncated + } while (isTruncated) + return objects +} + /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: any, filepath: any) => { +export const retrieveToTmp = async (bucketName: string, filepath: string) => { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -229,10 +261,30 @@ export const retrieveToTmp = async (bucketName: any, filepath: any) => { return outputPath } +export const retrieveDirectory = async (bucketName: string, path: string) => { + let writePath = join(budibaseTempDir(), v4()) + const objects = await listAllObjects(bucketName, path) + let fullObjects = await Promise.all( + objects.map(obj => retrieve(bucketName, obj.Key!)) + ) + let count = 0 + for (let obj of objects) { + const filename = obj.Key! + const data = fullObjects[count++] + const possiblePath = filename.split("/") + if (possiblePath.length > 1) { + const dirs = possiblePath.slice(0, possiblePath.length - 1) + fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) + } + fs.writeFileSync(join(writePath, ...possiblePath), data) + } + return writePath +} + /** * Delete a single file. */ -export const deleteFile = async (bucketName: any, filepath: any) => { +export const deleteFile = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { diff --git a/packages/builder/src/stores/portal/apps.js b/packages/builder/src/stores/portal/apps.js index a83e35e941..f84e0c973c 100644 --- a/packages/builder/src/stores/portal/apps.js +++ b/packages/builder/src/stores/portal/apps.js @@ -2,6 +2,9 @@ import { writable } from "svelte/store" import { AppStatus } from "../../constants" import { API } from "api" +// properties that should always come from the dev app, not the deployed +const DEV_PROPS = ["updatedBy", "updatedAt"] + const extractAppId = id => { const split = id?.split("_") || [] return split.length ? split[split.length - 1] : null @@ -57,9 +60,19 @@ export function createAppStore() { return } + let devProps = {} + if (appMap[id]) { + const entries = Object.entries(appMap[id]).filter( + ([key]) => DEV_PROPS.indexOf(key) !== -1 + ) + entries.forEach(entry => { + devProps[entry[0]] = entry[1] + }) + } appMap[id] = { ...appMap[id], ...app, + ...devProps, prodId: app.appId, prodRev: app._rev, } diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index cd2ea415a4..878a81e110 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -9,7 +9,7 @@ export async function exportAppDump(ctx: any) { excludeRows = isQsTrue(excludeRows) const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` ctx.attachment(backupIdentifier) - ctx.body = await sdk.apps.exports.streamBackup(appId, excludeRows) + ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 55aa6bb548..5de5141d74 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -35,7 +35,7 @@ exports.exportApps = async ctx => { // only export the dev apps as they will be the latest, the user can republish the apps // in their self hosted environment if (isDevAppID(appId)) { - allDBs[app.name] = await sdk.apps.exports.exportDB(appId) + allDBs[app.name] = await sdk.apps.exports.exportApp(appId) } } const filename = `cloud-export-${new Date().getTime()}.txt` diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 08213c2cf8..f60dc12971 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -5,7 +5,7 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") const uuid = require("uuid") -const { ObjectStoreBuckets } = require("../../../constants") +const { ObjectStoreBuckets, ATTACHMENT_PATH } = require("../../../constants") const { processString } = require("@budibase/string-templates") const { loadHandlebarsFile, @@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) { return prepareUpload({ file, - s3Key: `${ctx.appId}/attachments/${processedFileName}`, + s3Key: `${ctx.appId}/${ATTACHMENT_PATH}/${processedFileName}`, bucket: ObjectStoreBuckets.APPS, }) }) diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index c002c10f7b..c1a992f70e 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -1,6 +1,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") const { UserStatus } = require("@budibase/backend-core/constants") -const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore") +const { objectStore } = require("@budibase/backend-core") exports.JobQueues = { AUTOMATIONS: "automationQueue", @@ -209,6 +209,8 @@ exports.AutomationErrors = { } // pass through the list from the auth/core lib -exports.ObjectStoreBuckets = ObjectStoreBuckets +exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets + +exports.ATTACHMENT_PATH = "attachments" exports.MAX_AUTOMATION_RECURRING_ERRORS = 5 diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts index 70c4446273..bf7ea81cbc 100644 --- a/packages/server/src/sdk/app/export.ts +++ b/packages/server/src/sdk/app/export.ts @@ -1,7 +1,10 @@ -import { closeDB, dangerousGetDB, doWithDB } from "@budibase/backend-core/db" +import { db as dbCore } from "@budibase/backend-core" import { budibaseTempDir } from "../../utilities/budibaseDir" -import { streamUpload } from "../../utilities/fileSystem/utilities" -import { ObjectStoreBuckets } from "../../constants" +import { + streamUpload, + retrieveDirectory, +} from "../../utilities/fileSystem/utilities" +import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants" import { LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, @@ -25,16 +28,16 @@ export async function exportDB( ) { // streaming a DB dump is a bit more complicated, can't close DB if (opts?.stream) { - const db = dangerousGetDB(dbName) + const db = dbCore.dangerousGetDB(dbName) const memStream = new MemoryStream() memStream.on("end", async () => { - await closeDB(db) + await dbCore.closeDB(db) }) db.dump(memStream, { filter: opts?.filter }) return memStream } - return doWithDB(dbName, async (db: any) => { + return dbCore.doWithDB(dbName, async (db: any) => { // Write the dump to file if required if (opts?.exportName) { const path = join(budibaseTempDir(), opts?.exportName) @@ -49,18 +52,17 @@ export async function exportDB( fs.createReadStream(path) ) } - return fs.createReadStream(path) + } else { + // Stringify the dump in memory if required + const memStream = new MemoryStream() + let appString = "" + memStream.on("data", (chunk: any) => { + appString += chunk.toString() + }) + await db.dump(memStream, { filter: opts?.filter }) + return appString } - - // Stringify the dump in memory if required - const memStream = new MemoryStream() - let appString = "" - memStream.on("data", (chunk: any) => { - appString += chunk.toString() - }) - await db.dump(memStream, { filter: opts?.filter }) - return appString }) } @@ -81,12 +83,17 @@ function defineFilter(excludeRows?: boolean) { * @param {boolean} excludeRows Flag to state whether the export should include data. * @returns {*} either a string or a stream of the backup */ -async function backupAppData( +export async function exportApp( appId: string, - config: any, + config?: any, excludeRows?: boolean ) { - return await exportDB(appId, { + const attachmentsPath = `${dbCore.getProdAppID(appId)}/${ATTACHMENT_PATH}` + const tmpPath = await retrieveDirectory( + ObjectStoreBuckets.APPS, + attachmentsPath + ) + await exportDB(appId, { ...config, filter: defineFilter(excludeRows), }) @@ -98,16 +105,6 @@ async function backupAppData( * @param {boolean} excludeRows Flag to state whether the export should include data. * @returns {*} a readable stream of the backup which is written in real time */ -export async function streamBackup(appId: string, excludeRows: boolean) { - return await backupAppData(appId, { stream: true }, excludeRows) -} - -/** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return {*} a readable stream to the completed backup file - */ -export async function performBackup(appId: string, backupName: string) { - return await backupAppData(appId, { exportName: backupName }) +export async function streamExportApp(appId: string, excludeRows: boolean) { + return await exportApp(appId, { stream: true }, excludeRows) } diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index 1c804c0142..01ba58f5bc 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -6,6 +6,7 @@ const { streamUpload, retrieve, retrieveToTmp, + retrieveDirectory, deleteFolder, uploadDirectory, downloadTarball, @@ -27,6 +28,7 @@ exports.upload = upload exports.streamUpload = streamUpload exports.retrieve = retrieve exports.retrieveToTmp = retrieveToTmp +exports.retrieveDirectory = retrieveDirectory exports.deleteFolder = deleteFolder exports.uploadDirectory = uploadDirectory exports.downloadTarball = downloadTarball diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 3b98c8ef52..ce03a12587 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -99,16 +99,7 @@ export const paginatedUsers = async ({ */ export const getUser = async (userId: string) => { const db = tenancy.getGlobalDB() - let user - try { - user = await db.get(userId) - } catch (err: any) { - // no user found, just return nothing - if (err.status === 404) { - return {} - } - throw err - } + let user = await db.get(userId) if (user) { delete user.password }