From 4acfc623b4fba8248c5288a84976258e735854ac Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Jun 2023 14:52:19 +0100 Subject: [PATCH 01/13] Use import for tar --- packages/server/src/sdk/app/backups/exports.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 57342e7462..a31ade6d13 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -18,7 +18,8 @@ import { join } from "path" import env from "../../../environment" const uuid = require("uuid/v4") -const tar = require("tar") +import tar from "tar" + const MemoryStream = require("memorystream") interface DBDumpOpts { @@ -33,13 +34,14 @@ interface ExportOpts extends DBDumpOpts { } function tarFilesToTmp(tmpDir: string, files: string[]) { - const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`) + const fileName = `${uuid()}.tar.gz` + const exportFile = join(budibaseTempDir(), fileName) tar.create( { sync: true, gzip: true, file: exportFile, - recursive: true, + noDirRecurse: false, cwd: tmpDir, }, files @@ -124,6 +126,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { ) } } + const downloadedPath = join(tmpPath, appPath) if (fs.existsSync(downloadedPath)) { const allFiles = fs.readdirSync(downloadedPath) From 978591e2ba83678d72f9b9766feb707f3e008d7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Jun 2023 16:26:48 +0100 Subject: [PATCH 02/13] Enable encrypting --- .../backend-core/src/security/encryption.ts | 23 ++++++++++++++++++ packages/server/src/api/controllers/backup.ts | 11 ++++++--- .../server/src/sdk/app/backups/exports.ts | 24 ++++++++++++++++--- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index f9adb68955..ac8589ddf3 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -1,4 +1,6 @@ import crypto from "crypto" +import fs from "fs" +import zlib from "zlib" import env from "../environment" const ALGO = "aes-256-ctr" @@ -60,3 +62,24 @@ export function decrypt( const final = decipher.final() return Buffer.concat([base, final]).toString() } + +export async function encryptFile(filePath: string, secret: string) { + const outputFilePath = `${filePath}.enc` + + const inputFile = fs.createReadStream(filePath) + const outputFile = fs.createWriteStream(outputFilePath) + + const salt = crypto.randomBytes(RANDOM_BYTES) + const stretched = stretchString(secret, salt) + const cipher = crypto.createCipheriv(ALGO, stretched, salt) + + const encrypted = inputFile.pipe(cipher).pipe(zlib.createGzip()) + + encrypted.pipe(outputFile) + + return new Promise(r => { + outputFile.on("finish", () => { + r(outputFilePath) + }) + }) +} diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 53e1bd1792..716f5298dd 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -4,14 +4,19 @@ import { DocumentType } from "../../db/utils" import { isQsTrue } from "../../utilities" export async function exportAppDump(ctx: any) { - let { appId, excludeRows } = ctx.query + let { appId, excludeRows = false, encryptPassword } = ctx.query // remove the 120 second limit for the request ctx.req.setTimeout(0) const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) - const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz` + const extension = encryptPassword ? "data" : "tar.gz" + const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}` ctx.attachment(backupIdentifier) - ctx.body = await sdk.backups.streamExportApp(appId, excludeRows) + ctx.body = await sdk.backups.streamExportApp({ + appId, + excludeRows, + encryptPassword, + }) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index a31ade6d13..6bce7eed5c 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -1,4 +1,4 @@ -import { db as dbCore, objectStore } from "@budibase/backend-core" +import { db as dbCore, encryption, objectStore } from "@budibase/backend-core" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { streamFile, createTempFolder } from "../../../utilities/fileSystem" import { ObjectStoreBuckets } from "../../../constants" @@ -31,6 +31,7 @@ interface ExportOpts extends DBDumpOpts { tar?: boolean excludeRows?: boolean excludeLogs?: boolean + encryptPassword?: string } function tarFilesToTmp(tmpDir: string, files: string[]) { @@ -150,7 +151,15 @@ export async function exportApp(appId: string, config?: ExportOpts) { const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) // cleanup the tmp export files as tarball returned fs.rmSync(tmpPath, { recursive: true, force: true }) - return tarPath + if (!config.encryptPassword) { + return tarPath + } + + const encryptedTarPath = await encryption.encryptFile( + tarPath, + config.encryptPassword + ) + return encryptedTarPath } // tar not requested, turn the directory where export is else { @@ -164,11 +173,20 @@ export async function exportApp(appId: string, config?: ExportOpts) { * @param {boolean} excludeRows Flag to state whether the export should include data. * @returns {*} a readable stream of the backup which is written in real time */ -export async function streamExportApp(appId: string, excludeRows: boolean) { +export async function streamExportApp({ + appId, + excludeRows, + encryptPassword, +}: { + appId: string + excludeRows: boolean + encryptPassword: string +}) { const tmpPath = await exportApp(appId, { excludeRows, excludeLogs: true, tar: true, + encryptPassword, }) return streamFile(tmpPath) } From d7f64fe6a40cedf0b90f86a70e457b076ed04736 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 11:12:22 +0100 Subject: [PATCH 03/13] Update tar libs --- packages/cli/package.json | 2 +- packages/server/package.json | 4 ++-- yarn.lock | 24 ++++++++++++++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 3bd5ba279c..37c6f68b3f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -49,7 +49,7 @@ "pouchdb": "7.3.0", "pouchdb-replication-stream": "1.2.9", "randomstring": "1.1.5", - "tar": "6.1.11", + "tar": "6.1.15", "yaml": "^2.1.1" }, "devDependencies": { diff --git a/packages/server/package.json b/packages/server/package.json index 6e74de6afa..5563b3d9ba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -117,7 +117,7 @@ "socket.io": "4.6.1", "svelte": "3.49.0", "swagger-parser": "10.0.3", - "tar": "6.1.11", + "tar": "6.1.15", "to-json-schema": "0.2.5", "uuid": "3.3.2", "validate.js": "0.13.1", @@ -150,7 +150,7 @@ "@types/redis": "4.0.11", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.12", - "@types/tar": "6.1.3", + "@types/tar": "6.1.5", "@typescript-eslint/parser": "5.45.0", "apidoc": "0.50.4", "babel-jest": "29.5.0", diff --git a/yarn.lock b/yarn.lock index ea9823b58b..4d290b5996 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6197,13 +6197,13 @@ dependencies: "@types/node" "*" -"@types/tar@6.1.3": - version "6.1.3" - resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.3.tgz#46a2ce7617950c4852dfd7e9cd41aa8161b9d750" - integrity sha512-YzDOr5kdAeqS8dcO6NTTHTMJ44MUCBDoLEIyPtwEn7PssKqUYL49R1iCVJPeiPzPlKi6DbH33eZkpeJ27e4vHg== +"@types/tar@6.1.5": + version "6.1.5" + resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.5.tgz#90ccb3b6a35430e7427410d50eed564e85feaaff" + integrity sha512-qm2I/RlZij5RofuY7vohTpYNaYcrSQlN2MyjucQc7ZweDwaEWkdN/EeNh6e9zjK6uEm6PwjdMXkcj05BxZdX1Q== dependencies: "@types/node" "*" - minipass "^3.3.5" + minipass "^4.0.0" "@types/tern@*": version "0.23.4" @@ -18036,7 +18036,7 @@ minipass-sized@^1.0.3: dependencies: minipass "^3.0.0" -minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6, minipass@^3.3.5: +minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6: version "3.3.6" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== @@ -24163,6 +24163,18 @@ tar@6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" +tar@6.1.15: + version "6.1.15" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69" + integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^5.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + tar@^6.1.11, tar@^6.1.2: version "6.1.13" resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.13.tgz#46e22529000f612180601a6fe0680e7da508847b" From 1f4cdf348fbeaa3a2edbc9d95dc552c402515ea0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 11:49:38 +0100 Subject: [PATCH 04/13] Encrypt files --- .../backend-core/src/security/encryption.ts | 18 +++++++++++---- packages/server/src/api/controllers/backup.ts | 2 +- .../server/src/sdk/app/backups/exports.ts | 23 ++++++++++++------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index ac8589ddf3..0cc52d6210 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -2,6 +2,7 @@ import crypto from "crypto" import fs from "fs" import zlib from "zlib" import env from "../environment" +import { join } from "path" const ALGO = "aes-256-ctr" const SEPARATOR = "-" @@ -63,11 +64,15 @@ export function decrypt( return Buffer.concat([base, final]).toString() } -export async function encryptFile(filePath: string, secret: string) { - const outputFilePath = `${filePath}.enc` +export async function encryptFile( + { dir, filename }: { dir: string; filename: string }, + secret: string +) { + const outputFileName = `${filename}.enc` + const filePath = join(dir, filename) const inputFile = fs.createReadStream(filePath) - const outputFile = fs.createWriteStream(outputFilePath) + const outputFile = fs.createWriteStream(join(dir, outputFileName)) const salt = crypto.randomBytes(RANDOM_BYTES) const stretched = stretchString(secret, salt) @@ -77,9 +82,12 @@ export async function encryptFile(filePath: string, secret: string) { encrypted.pipe(outputFile) - return new Promise(r => { + return new Promise<{ filename: string; dir: string }>(r => { outputFile.on("finish", () => { - r(outputFilePath) + r({ + filename: outputFileName, + dir, + }) }) }) } diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 716f5298dd..4c267d5c77 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -9,7 +9,7 @@ export async function exportAppDump(ctx: any) { ctx.req.setTimeout(0) const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) - const extension = encryptPassword ? "data" : "tar.gz" + const extension = encryptPassword ? "enc.tar.gz" : "tar.gz" const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}` ctx.attachment(backupIdentifier) ctx.body = await sdk.backups.streamExportApp({ diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 6bce7eed5c..0f629630e2 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -145,21 +145,28 @@ export async function exportApp(appId: string, config?: ExportOpts) { filter: defineFilter(config?.excludeRows, config?.excludeLogs), exportPath: dbPath, }) + + if (config?.encryptPassword) { + for (let file of fs.readdirSync(tmpPath)) { + const path = join(tmpPath, file) + + await encryption.encryptFile( + { dir: tmpPath, filename: file }, + config.encryptPassword + ) + + fs.rmSync(path) + } + } + // if tar requested, return where the tarball is if (config?.tar) { // now the tmpPath contains both the DB export and attachments, tar this const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) // cleanup the tmp export files as tarball returned fs.rmSync(tmpPath, { recursive: true, force: true }) - if (!config.encryptPassword) { - return tarPath - } - const encryptedTarPath = await encryption.encryptFile( - tarPath, - config.encryptPassword - ) - return encryptedTarPath + return tarPath } // tar not requested, turn the directory where export is else { From 551ca404b45d88afc067a8bc4e956e7d5906ebb2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 16:27:19 +0100 Subject: [PATCH 05/13] Decrypt file --- .../backend-core/src/security/encryption.ts | 67 +++++++++++++++++-- packages/server/src/api/controllers/backup.ts | 7 +- .../server/src/sdk/app/backups/imports.ts | 15 ++++- 3 files changed, 79 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 0cc52d6210..3b614e1c7d 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -7,9 +7,11 @@ import { join } from "path" const ALGO = "aes-256-ctr" const SEPARATOR = "-" const ITERATIONS = 10000 -const RANDOM_BYTES = 16 const STRETCH_LENGTH = 32 +const SALT_LENGTH = 16 +const IV_LENGTH = 16 + export enum SecretOption { API = "api", ENCRYPTION = "encryption", @@ -42,7 +44,7 @@ export function encrypt( input: string, secretOption: SecretOption = SecretOption.API ) { - const salt = crypto.randomBytes(RANDOM_BYTES) + const salt = crypto.randomBytes(SALT_LENGTH) const stretched = stretchString(getSecret(secretOption), salt) const cipher = crypto.createCipheriv(ALGO, stretched, salt) const base = cipher.update(input) @@ -74,13 +76,15 @@ export async function encryptFile( const inputFile = fs.createReadStream(filePath) const outputFile = fs.createWriteStream(join(dir, outputFileName)) - const salt = crypto.randomBytes(RANDOM_BYTES) + const salt = crypto.randomBytes(SALT_LENGTH) + const iv = crypto.randomBytes(IV_LENGTH) const stretched = stretchString(secret, salt) - const cipher = crypto.createCipheriv(ALGO, stretched, salt) + const cipher = crypto.createCipheriv(ALGO, stretched, iv) - const encrypted = inputFile.pipe(cipher).pipe(zlib.createGzip()) + outputFile.write(salt) + outputFile.write(iv) - encrypted.pipe(outputFile) + inputFile.pipe(cipher).pipe(outputFile) return new Promise<{ filename: string; dir: string }>(r => { outputFile.on("finish", () => { @@ -91,3 +95,54 @@ export async function encryptFile( }) }) } + +export async function decryptFile( + inputPath: string, + outputPath: string, + secret: string +) { + const inputFile = fs.createReadStream(inputPath) + const outputFile = fs.createWriteStream(outputPath) + + const salt = await readBytes(inputFile, SALT_LENGTH) + const iv = await readBytes(inputFile, IV_LENGTH) + + const stretched = stretchString(secret, salt) + const decipher = crypto.createDecipheriv(ALGO, stretched, iv) + + fs.createReadStream(inputPath, { start: SALT_LENGTH + IV_LENGTH }) + .pipe(decipher) + .pipe(outputFile) + + return new Promise(r => { + outputFile.on("finish", () => { + r() + }) + }) +} + +function readBytes(stream: fs.ReadStream, length: number) { + return new Promise((resolve, reject) => { + let bytesRead = 0 + const data: Buffer[] = [] + + stream.on("readable", () => { + let chunk + + while ((chunk = stream.read(length - bytesRead)) !== null) { + data.push(chunk) + bytesRead += chunk.length + } + + resolve(Buffer.concat(data)) + }) + + stream.on("end", () => { + reject(new Error("Insufficient data in the stream.")) + }) + + stream.on("error", (error: any) => { + reject(error) + }) + }) +} diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 4c267d5c77..02dc852003 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -4,13 +4,14 @@ import { DocumentType } from "../../db/utils" import { isQsTrue } from "../../utilities" export async function exportAppDump(ctx: any) { - let { appId, excludeRows = false, encryptPassword } = ctx.query + let { appId, excludeRows = false, encryptPassword = "password" } = ctx.query // remove the 120 second limit for the request ctx.req.setTimeout(0) const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) - const extension = encryptPassword ? "enc.tar.gz" : "tar.gz" - const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}` + const backupIdentifier = `${appName}-export-${new Date().getTime()}${ + encryptPassword ? "-enc" : "" + }.tar.gz` ctx.attachment(backupIdentifier) ctx.body = await sdk.backups.streamExportApp({ appId, diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index b63d4c5a40..08b003a55b 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,4 @@ -import { db as dbCore, objectStore } from "@budibase/backend-core" +import { db as dbCore, encryption, objectStore } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" @@ -20,6 +20,7 @@ type TemplateType = { file?: { type: string path: string + password?: string } key?: string } @@ -123,6 +124,15 @@ export function untarFile(file: { path: string }) { return tmpPath } +async function decryptFiles(path: string) { + for (let file of fs.readdirSync(path)) { + const inputPath = join(path, file) + const outputPath = inputPath.replace(/\.enc$/, "") + await encryption.decryptFile(inputPath, outputPath, "password") + fs.rmSync(inputPath) + } +} + export function getGlobalDBFile(tmpPath: string) { return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8") } @@ -143,6 +153,9 @@ export async function importApp( template.file && fs.lstatSync(template.file.path).isDirectory() if (template.file && (isTar || isDirectory)) { const tmpPath = isTar ? untarFile(template.file) : template.file.path + if (isTar && template.file.password) { + await decryptFiles(tmpPath) + } const contents = fs.readdirSync(tmpPath) // have to handle object import if (contents.length) { From 2971dfba9d038272dca791d159307aeb634fecea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 17:31:08 +0100 Subject: [PATCH 06/13] Renames --- packages/backend-core/src/security/encryption.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 3b614e1c7d..29a8274c54 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -36,8 +36,8 @@ export function getSecret(secretOption: SecretOption): string { return secret } -function stretchString(string: string, salt: Buffer) { - return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512") +function stretchString(secret: string, salt: Buffer) { + return crypto.pbkdf2Sync(secret, salt, ITERATIONS, STRETCH_LENGTH, "sha512") } export function encrypt( From 92a8c97aba082877d788a7f8e3f60ce296c210b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 17:34:12 +0100 Subject: [PATCH 07/13] Close streams --- packages/backend-core/src/security/encryption.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 29a8274c54..e4d4048590 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -116,6 +116,8 @@ export async function decryptFile( return new Promise(r => { outputFile.on("finish", () => { + inputFile.close() + outputFile.close() r() }) }) From ded738a56607b0c2135894152231db70ac1fdd54 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 17:49:12 +0100 Subject: [PATCH 08/13] Clean code --- .../backend-core/src/security/encryption.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index e4d4048590..22a747ac30 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -96,27 +96,34 @@ export async function encryptFile( }) } +async function getSaltAndIV(path: string) { + const fileStream = fs.createReadStream(path) + + const salt = await readBytes(fileStream, SALT_LENGTH) + const iv = await readBytes(fileStream, IV_LENGTH) + fileStream.close() + return { salt, iv } +} + export async function decryptFile( inputPath: string, outputPath: string, secret: string ) { - const inputFile = fs.createReadStream(inputPath) - const outputFile = fs.createWriteStream(outputPath) + const { salt, iv } = await getSaltAndIV(inputPath) + const inputFile = fs.createReadStream(inputPath, { + start: SALT_LENGTH + IV_LENGTH, + }) - const salt = await readBytes(inputFile, SALT_LENGTH) - const iv = await readBytes(inputFile, IV_LENGTH) + const outputFile = fs.createWriteStream(outputPath) const stretched = stretchString(secret, salt) const decipher = crypto.createDecipheriv(ALGO, stretched, iv) - fs.createReadStream(inputPath, { start: SALT_LENGTH + IV_LENGTH }) - .pipe(decipher) - .pipe(outputFile) + inputFile.pipe(decipher).pipe(outputFile) return new Promise(r => { outputFile.on("finish", () => { - inputFile.close() outputFile.close() r() }) From 70798a6b937e08af44711ed63506d27a7b9e0879 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 17:50:07 +0100 Subject: [PATCH 09/13] Clean --- packages/backend-core/src/security/encryption.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 22a747ac30..8a2657bb40 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -150,7 +150,7 @@ function readBytes(stream: fs.ReadStream, length: number) { reject(new Error("Insufficient data in the stream.")) }) - stream.on("error", (error: any) => { + stream.on("error", error => { reject(error) }) }) From 81522d0784b24cf29ba9b8a0ae44a70cec5e7100 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 17:51:21 +0100 Subject: [PATCH 10/13] Clean defaults --- packages/server/src/api/controllers/backup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 02dc852003..c599641ef0 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -4,7 +4,7 @@ import { DocumentType } from "../../db/utils" import { isQsTrue } from "../../utilities" export async function exportAppDump(ctx: any) { - let { appId, excludeRows = false, encryptPassword = "password" } = ctx.query + let { appId, excludeRows = false, encryptPassword } = ctx.query // remove the 120 second limit for the request ctx.req.setTimeout(0) const appName = decodeURI(ctx.query.appname) From 57c5facc6e81695f0cd2d4c14f22020e0632cc01 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Jun 2023 17:54:09 +0100 Subject: [PATCH 11/13] Zip exports --- packages/backend-core/src/security/encryption.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 8a2657bb40..0147b45c6c 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -84,7 +84,7 @@ export async function encryptFile( outputFile.write(salt) outputFile.write(iv) - inputFile.pipe(cipher).pipe(outputFile) + inputFile.pipe(zlib.createGzip()).pipe(cipher).pipe(outputFile) return new Promise<{ filename: string; dir: string }>(r => { outputFile.on("finish", () => { @@ -120,7 +120,7 @@ export async function decryptFile( const stretched = stretchString(secret, salt) const decipher = crypto.createDecipheriv(ALGO, stretched, iv) - inputFile.pipe(decipher).pipe(outputFile) + inputFile.pipe(decipher).pipe(zlib.createGunzip()).pipe(outputFile) return new Promise(r => { outputFile.on("finish", () => { From a4f0b45d5e1f0b4677745fcbd42c04bdd192780a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Jun 2023 11:17:32 +0100 Subject: [PATCH 12/13] Fix merge --- packages/server/src/sdk/app/backups/exports.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 0f629630e2..3be8c64159 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -187,7 +187,7 @@ export async function streamExportApp({ }: { appId: string excludeRows: boolean - encryptPassword: string + encryptPassword?: string }) { const tmpPath = await exportApp(appId, { excludeRows, From 4b065dda8bf9abfbb37bd45f6c623ebecd2a295a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Jun 2023 12:32:26 +0100 Subject: [PATCH 13/13] Fix exports/imports --- .../backend-core/src/security/encryption.ts | 28 +++++++++++++++++-- .../server/src/sdk/app/backups/imports.ts | 21 +++++++++----- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 0147b45c6c..7a8cfaf04a 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -120,12 +120,34 @@ export async function decryptFile( const stretched = stretchString(secret, salt) const decipher = crypto.createDecipheriv(ALGO, stretched, iv) - inputFile.pipe(decipher).pipe(zlib.createGunzip()).pipe(outputFile) + const unzip = zlib.createGunzip() - return new Promise(r => { + inputFile.pipe(decipher).pipe(unzip).pipe(outputFile) + + return new Promise((res, rej) => { outputFile.on("finish", () => { outputFile.close() - r() + res() + }) + + inputFile.on("error", e => { + outputFile.close() + rej(e) + }) + + decipher.on("error", e => { + outputFile.close() + rej(e) + }) + + unzip.on("error", e => { + outputFile.close() + rej(e) + }) + + outputFile.on("error", e => { + outputFile.close() + rej(e) }) }) } diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 08b003a55b..619f888329 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -124,12 +124,19 @@ export function untarFile(file: { path: string }) { return tmpPath } -async function decryptFiles(path: string) { - for (let file of fs.readdirSync(path)) { - const inputPath = join(path, file) - const outputPath = inputPath.replace(/\.enc$/, "") - await encryption.decryptFile(inputPath, outputPath, "password") - fs.rmSync(inputPath) +async function decryptFiles(path: string, password: string) { + try { + for (let file of fs.readdirSync(path)) { + const inputPath = join(path, file) + const outputPath = inputPath.replace(/\.enc$/, "") + await encryption.decryptFile(inputPath, outputPath, password) + fs.rmSync(inputPath) + } + } catch (err: any) { + if (err.message === "incorrect header check") { + throw new Error("File cannot be imported") + } + throw err } } @@ -154,7 +161,7 @@ export async function importApp( if (template.file && (isTar || isDirectory)) { const tmpPath = isTar ? untarFile(template.file) : template.file.path if (isTar && template.file.password) { - await decryptFiles(tmpPath) + await decryptFiles(tmpPath, template.file.password) } const contents = fs.readdirSync(tmpPath) // have to handle object import