From 85f7d66a9997ccac2688db48aa0c7b2071ab0d50 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 31 Jan 2024 17:16:51 +0000 Subject: [PATCH 1/4] Handling very large exports/backup downloads. --- packages/backend-core/src/objectStore/objectStore.ts | 4 ++-- packages/pro | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 57ead0e809..3a3b9cdaab 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -7,7 +7,7 @@ import tar from "tar-fs" import zlib from "zlib" import { promisify } from "util" import { join } from "path" -import fs from "fs" +import fs, { ReadStream } from "fs" import env from "../environment" import { budibaseTempDir } from "./utils" import { v4 } from "uuid" @@ -184,7 +184,7 @@ export async function upload({ export async function streamUpload( bucketName: string, filename: string, - stream: any, + stream: ReadStream | ReadableStream, extra = {} ) { const objectStore = ObjectStore(bucketName) diff --git a/packages/pro b/packages/pro index eb9565f568..e4dab7e4cc 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit eb9565f568cfef14b336b14eee753119acfdd43b +Subproject commit e4dab7e4ccb4a1e2864a35dd2a851ca1a898e6c6 From 010f160fa54089eef6a8d75ae289bdd93ee5151e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 31 Jan 2024 18:22:18 +0000 Subject: [PATCH 2/4] Cleaning up files after import is finished (with large files this saves on tmp space) as well as fixing an issue where menu would appear next to a restore despite having no options. --- .../backups/_components/ActionsRenderer.svelte | 16 ++++++++-------- packages/pro | 2 +- packages/server/src/sdk/app/backups/imports.ts | 7 ++++++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte b/packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte index 93f0f518ad..ffdf14fdce 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte @@ -13,6 +13,7 @@ import CreateRestoreModal from "./CreateRestoreModal.svelte" import { createEventDispatcher } from "svelte" import { isOnlyUser } from "builderStore" + import { BackupType } from "constants/backend/backups" export let row @@ -42,12 +43,11 @@
- -
- -
- - {#if row.type !== "restore"} + {#if row.type !== BackupType.RESTORE} + +
+ +
Delete Download - {/if} - + + {/if}
diff --git a/packages/pro b/packages/pro index e4dab7e4cc..f8ebf7e3e1 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e4dab7e4ccb4a1e2864a35dd2a851ca1a898e6c6 +Subproject commit f8ebf7e3e14c894cf7ab831fabd2706e94cfb830 diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 7f76945107..4c5c1e659d 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -165,8 +165,9 @@ export async function importApp( const isTar = template.file && template?.file?.type?.endsWith("gzip") const isDirectory = template.file && fs.lstatSync(template.file.path).isDirectory() + let tmpPath: string | undefined = undefined if (template.file && (isTar || isDirectory)) { - const tmpPath = isTar ? await untarFile(template.file) : template.file.path + tmpPath = isTar ? await untarFile(template.file) : template.file.path if (isTar && template.file.password) { await decryptFiles(tmpPath, template.file.password) } @@ -208,5 +209,9 @@ export async function importApp( } await updateAttachmentColumns(prodAppId, db) await updateAutomations(prodAppId, db) + // clear up afterward + if (tmpPath) { + fs.rmSync(tmpPath, { recursive: true, force: true }) + } return ok } From e61d52ee8ba7b741aa4c3060c4085b7faf7cfd5c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Feb 2024 10:18:14 +0000 Subject: [PATCH 3/4] Updating pro ref to master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f8ebf7e3e1..4f9616f163 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f8ebf7e3e14c894cf7ab831fabd2706e94cfb830 +Subproject commit 4f9616f163039a0eea81319d8e2288340a2ebc79 From c4eae33b6718a722da67c65c96f9817c2c755db0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Feb 2024 11:09:28 +0000 Subject: [PATCH 4/4] Fixing mock to cover getReadStream in object store. --- packages/server/__mocks__/aws-sdk.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index fa6d099f56..e6a624c83e 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -1,7 +1,13 @@ +import fs from "fs" +import { join } from "path" + module AwsMock { const aws: any = {} - const response = (body: any) => () => ({ promise: () => body }) + const response = (body: any, extra?: any) => () => ({ + promise: () => body, + ...extra, + }) function DocumentClient() { // @ts-ignore @@ -73,9 +79,18 @@ module AwsMock { // @ts-ignore this.getObject = jest.fn( - response({ - Body: "", - }) + response( + { + Body: "", + }, + { + createReadStream: jest + .fn() + .mockReturnValue( + fs.createReadStream(join(__dirname, "aws-sdk.ts")) + ), + } + ) ) }