Fixing importing - still some work to get links correct.

packages/server/src/environment.ts

@@ -59,6 +59,7 @@ const environment = {
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/koa.ts

@@ -1,5 +1,5 @@
 import env from "./environment"
-import Koa, { ExtendableContext } from "koa"
+import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
 import * as api from "./api"
@@ -27,6 +27,9 @@ export default function createKoaApp() {
       // @ts-ignore
       enableTypes: ["json", "form", "text"],
       parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
+      formidable: {
+        maxFileSize: parseInt(env.MAX_IMPORT_SIZE_MB || "100") * 1024 * 1024,
+      },
     })
   )
 

packages/server/src/sdk/app/backups/exports.ts

@@ -30,12 +30,11 @@ export interface ExportOpts extends DBDumpOpts {
   encryptPassword?: string
 }
 
-function tarFilesToTmp(tmpDir: string, files: string[]) {
+async function tarFilesToTmp(tmpDir: string, files: string[]) {
   const fileName = `${uuid()}.tar.gz`
   const exportFile = join(budibaseTempDir(), fileName)
-  tar.create(
+  await tar.create(
     {
-      sync: true,
       gzip: true,
       file: exportFile,
       noDirRecurse: false,
@@ -150,19 +149,21 @@ export async function exportApp(appId: string, config?: ExportOpts) {
     for (let file of fs.readdirSync(tmpPath)) {
       const path = join(tmpPath, file)
 
+      // skip the attachments - too big to encrypt
+      if (file !== "attachments") {
         await encryption.encryptFile(
           { dir: tmpPath, filename: file },
           config.encryptPassword
         )
-
         fs.rmSync(path)
       }
     }
+  }
 
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
 

packages/server/src/sdk/app/backups/imports.ts

@@ -6,7 +6,7 @@ import {
   AutomationTriggerStepId,
   RowAttachment,
 } from "@budibase/types"
-import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
@@ -114,12 +114,11 @@ async function getTemplateStream(template: TemplateType) {
   }
 }
 
-export function untarFile(file: { path: string }) {
+export async function untarFile(file: { path: string }) {
   const tmpPath = join(budibaseTempDir(), uuid())
   fs.mkdirSync(tmpPath)
   // extract the tarball
-  tar.extract({
+  await tar.extract({
-    sync: true,
     cwd: tmpPath,
     file: file.path,
   })
@@ -130,10 +129,12 @@ async function decryptFiles(path: string, password: string) {
   try {
     for (let file of fs.readdirSync(path)) {
       const inputPath = join(path, file)
+      if (!inputPath.endsWith("attachments")) {
         const outputPath = inputPath.replace(/\.enc$/, "")
         await encryption.decryptFile(inputPath, outputPath, password)
         fs.rmSync(inputPath)
       }
+    }
   } catch (err: any) {
     if (err.message === "incorrect header check") {
       throw new Error("File cannot be imported")
@@ -162,7 +163,7 @@ export async function importApp(
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }