From e21e1d10e158ef286b9f9fa3981caace30f9b32e Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 18 Oct 2021 12:06:09 +0100 Subject: [PATCH 1/3] Increase envoy timeout for API routes to 120s rather than 15s, to match Koa and allw longer connections for downloads --- hosting/envoy.dev.yaml.hbs | 1 + hosting/envoy.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/hosting/envoy.dev.yaml.hbs b/hosting/envoy.dev.yaml.hbs index 01d5a09efa..59363fab5e 100644 --- a/hosting/envoy.dev.yaml.hbs +++ b/hosting/envoy.dev.yaml.hbs @@ -41,6 +41,7 @@ static_resources: - match: { prefix: "/api/" } route: cluster: server-dev + timeout: 120s - match: { prefix: "/app_" } route: diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index d5f9ebee28..d9f8384688 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -58,6 +58,7 @@ static_resources: - match: { prefix: "/api/" } route: cluster: app-service + timeout: 120s - match: { prefix: "/worker/" } route: From 75c2f45cf16832a8f0e2bea565814266a8c23dbf Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 18 Oct 2021 12:07:44 +0100 Subject: [PATCH 2/3] Improve app export UX by immediately starting a download stream of the app export --- .../src/pages/builder/portal/apps/index.svelte | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index 01eaa8281e..1b0732ab01 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -112,16 +112,8 @@ const exportApp = app => { const id = app.deployed ? app.prodId : app.devId - try { - download( - `/api/backups/export?appId=${id}&appname=${encodeURIComponent( - app.name - )}` - ) - notifications.success("App exported successfully") - } catch (err) { - notifications.error(`Error exporting app: ${err}`) - } + const appName = encodeURIComponent(app.name) + window.location = `/api/backups/export?appId=${id}&appname=${appName}` } const unpublishApp = app => { From 8817230bf1643b5bb4f53bc54bd509ed332cfa82 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 18 Oct 2021 12:08:12 +0100 Subject: [PATCH 3/3] Refactor app exports to allow a streaming realtime backup, and tidy up --- packages/server/src/api/controllers/backup.js | 9 +- .../server/src/utilities/fileSystem/index.js | 114 +++++++++++------- 2 files changed, 72 insertions(+), 51 deletions(-) diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.js index 0be19eb7bd..daaa59b341 100644 --- a/packages/server/src/api/controllers/backup.js +++ b/packages/server/src/api/controllers/backup.js @@ -1,10 +1,9 @@ -const { performBackup } = require("../../utilities/fileSystem") +const { streamBackup } = require("../../utilities/fileSystem") exports.exportAppDump = async function (ctx) { const { appId } = ctx.query - const appname = decodeURI(ctx.query.appname) - const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` - + const appName = decodeURI(ctx.query.appname) + const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` ctx.attachment(backupIdentifier) - ctx.body = await performBackup(appId, backupIdentifier) + ctx.body = await streamBackup(appId) } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 0c2da56b58..b8ddb1a356 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -106,64 +106,86 @@ exports.apiFileReturn = contents => { } /** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return The backup has been completed when this promise completes and returns a file stream - * to the temporary backup file (to return via API if required). + * Local utility to back up the database state for an app, excluding global user + * data or user relationships. + * @param {string} appId The app to backup + * @param {object} config Config to send to export DB + * @returns {*} either a string or a stream of the backup */ -exports.performBackup = async (appId, backupName) => { - return exports.exportDB(appId, { - exportName: backupName, +const backupAppData = async (appId, config) => { + return await exports.exportDB(appId, { + ...config, filter: doc => !( doc._id.includes(USER_METDATA_PREFIX) || - doc.includes(LINK_USER_METADATA_PREFIX) + doc._id.includes(LINK_USER_METADATA_PREFIX) ), }) } /** - * exports a DB to either file or a variable (memory). - * @param {string} dbName the DB which is to be exported. - * @param {string} exportName optional - the file name to export to, if not in memory. - * @param {function} filter optional - a filter function to clear out any un-wanted docs. - * @return Either the file stream or the variable (if no export name provided). + * Takes a copy of the database state for an app to the object store. + * @param {string} appId The ID of the app which is to be backed up. + * @param {string} backupName The name of the backup located in the object store. + * @return {*} a readable stream to the completed backup file */ -exports.exportDB = async ( - dbName, - { exportName, filter } = { exportName: undefined, filter: undefined } -) => { - let stream, - appString = "", - path = null - if (exportName) { - path = join(budibaseTempDir(), exportName) - stream = fs.createWriteStream(path) - } else { - stream = new MemoryStream() - stream.on("data", chunk => { - appString += chunk.toString() - }) - } - // perform couch dump +exports.performBackup = async (appId, backupName) => { + return await backupAppData(appId, { exportName: backupName }) +} + +/** + * Streams a backup of the database state for an app + * @param {string} appId The ID of the app which is to be backed up. + * @returns {*} a readable stream of the backup which is written in real time + */ +exports.streamBackup = async appId => { + return await backupAppData(appId, { stream: true }) +} + +/** + * Exports a DB to either file or a variable (memory). + * @param {string} dbName the DB which is to be exported. + * @param {string} exportName optional - provide a filename to write the backup to a file + * @param {boolean} stream optional - whether to perform a full backup + * @param {function} filter optional - a filter function to clear out any un-wanted docs. + * @return {*} either a readable stream or a string + */ +exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { const instanceDb = new CouchDB(dbName) - await instanceDb.dump(stream, { - filter, + + // Stream the dump if required + if (stream) { + const memStream = new MemoryStream() + instanceDb.dump(memStream, { filter }) + return memStream + } + + // Write the dump to file if required + if (exportName) { + const path = join(budibaseTempDir(), exportName) + const writeStream = fs.createWriteStream(path) + await instanceDb.dump(writeStream, { filter }) + + // Upload the dump to the object store if self hosted + if (env.SELF_HOSTED) { + await streamUpload( + ObjectStoreBuckets.BACKUPS, + join(dbName, exportName), + fs.createReadStream(path) + ) + } + + return fs.createReadStream(path) + } + + // Stringify the dump in memory if required + const memStream = new MemoryStream() + let appString = "" + memStream.on("data", chunk => { + appString += chunk.toString() }) - // just in memory, return the final string - if (!exportName) { - return appString - } - // write the file to the object store - if (env.SELF_HOSTED) { - await streamUpload( - ObjectStoreBuckets.BACKUPS, - join(dbName, exportName), - fs.createReadStream(path) - ) - } - return fs.createReadStream(path) + await instanceDb.dump(memStream, { filter }) + return appString } /**