Merge pull request #12923 from Budibase/fix/very-large-backups

Handling very large backup upload/downloads

packages/backend-core/src/objectStore/objectStore.ts

@@ -7,7 +7,7 @@ import tar from "tar-fs"
 import zlib from "zlib"
 import { promisify } from "util"
 import { join } from "path"
-import fs from "fs"
+import fs, { ReadStream } from "fs"
 import env from "../environment"
 import { budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
@@ -184,7 +184,7 @@ export async function upload({
 export async function streamUpload(
   bucketName: string,
   filename: string,
-  stream: any,
+  stream: ReadStream | ReadableStream,
   extra = {}
 ) {
   const objectStore = ObjectStore(bucketName)

packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte

@@ -13,6 +13,7 @@
   import CreateRestoreModal from "./CreateRestoreModal.svelte"
   import { createEventDispatcher } from "svelte"
   import { isOnlyUser } from "builderStore"
+  import { BackupType } from "constants/backend/backups"
 
   export let row
 
@@ -42,12 +43,11 @@
 </script>
 
 <div class="cell">
-  <ActionMenu align="right">
-    <div slot="control">
-      <Icon size="M" hoverable name="MoreSmallList" />
-    </div>
-
-    {#if row.type !== "restore"}
+  {#if row.type !== BackupType.RESTORE}
+    <ActionMenu align="right">
+      <div slot="control">
+        <Icon size="M" hoverable name="MoreSmallList" />
+      </div>
       <AbsTooltip
         position={TooltipPosition.Left}
         text="Unavailable - another user is editing this app"
@@ -62,8 +62,8 @@
       </AbsTooltip>
       <MenuItem on:click={deleteDialog.show} icon="Delete">Delete</MenuItem>
       <MenuItem on:click={downloadExport} icon="Download">Download</MenuItem>
-    {/if}
-  </ActionMenu>
+    </ActionMenu>
+  {/if}
 </div>
 
 <Modal bind:this={restoreBackupModal}>

packages/pro

@@ -1 +1 @@
-Subproject commit eb9565f568cfef14b336b14eee753119acfdd43b
+Subproject commit 4f9616f163039a0eea81319d8e2288340a2ebc79

packages/server/__mocks__/aws-sdk.ts

@@ -1,7 +1,13 @@
+import fs from "fs"
+import { join } from "path"
+
 module AwsMock {
   const aws: any = {}
 
-  const response = (body: any) => () => ({ promise: () => body })
+  const response = (body: any, extra?: any) => () => ({
+    promise: () => body,
+    ...extra,
+  })
 
   function DocumentClient() {
     // @ts-ignore
@@ -73,9 +79,18 @@ module AwsMock {
 
     // @ts-ignore
     this.getObject = jest.fn(
-      response({
-        Body: "",
-      })
+      response(
+        {
+          Body: "",
+        },
+        {
+          createReadStream: jest
+            .fn()
+            .mockReturnValue(
+              fs.createReadStream(join(__dirname, "aws-sdk.ts"))
+            ),
+        }
+      )
     )
   }
 

packages/server/src/sdk/app/backups/imports.ts

@@ -165,8 +165,9 @@ export async function importApp(
   const isTar = template.file && template?.file?.type?.endsWith("gzip")
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
+  let tmpPath: string | undefined = undefined
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
+    tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }
@@ -208,5 +209,9 @@ export async function importApp(
   }
   await updateAttachmentColumns(prodAppId, db)
   await updateAutomations(prodAppId, db)
+  // clear up afterward
+  if (tmpPath) {
+    fs.rmSync(tmpPath, { recursive: true, force: true })
+  }
   return ok
 }