Merge pull request #12510 from Budibase/fix/massive-attachment-export

Export/backup of large number of attachments

hosting/nginx.dev.conf

@@ -42,7 +42,7 @@ http {
   server { 
     listen       10000 default_server;
     server_name  _;
-    client_max_body_size 1000m;
+    client_max_body_size 50000m;
     ignore_invalid_headers off;
     proxy_buffering off;
 

packages/backend-core/src/auth/auth.ts

@@ -19,6 +19,7 @@ import {
   GoogleInnerConfig,
   OIDCInnerConfig,
   PlatformLogoutOpts,
+  SessionCookie,
   SSOProviderType,
 } from "@budibase/types"
 import * as events from "../events"
@@ -44,7 +45,6 @@ export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
 export const passport = _passport
-export const jwt = require("jsonwebtoken")
 
 // Strategies
 _passport.use(new LocalStrategy(local.options, local.authenticate))
@@ -191,10 +191,10 @@ export async function platformLogout(opts: PlatformLogoutOpts) {
 
   if (!ctx) throw new Error("Koa context must be supplied to logout.")
 
-  const currentSession = getCookie(ctx, Cookie.Auth)
+  const currentSession = getCookie<SessionCookie>(ctx, Cookie.Auth)
   let sessions = await getSessionsForUser(userId)
 
-  if (keepActiveSession) {
+  if (currentSession && keepActiveSession) {
     sessions = sessions.filter(
       session => session.sessionId !== currentSession.sessionId
     )

packages/backend-core/src/middleware/authenticated.ts

@@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { Ctx, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types"
 import { InvalidAPIKeyError, ErrorCode } from "../errors"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
@@ -98,7 +98,9 @@ export default function (
       // check the actual user is authenticated first, try header or cookie
       let headerToken = ctx.request.headers[Header.TOKEN]
 
-      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
+      const authCookie =
+        getCookie<SessionCookie>(ctx, Cookie.Auth) ||
+        openJwt<SessionCookie>(headerToken)
       let apiKey = ctx.request.headers[Header.API_KEY]
 
       if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -3,7 +3,7 @@ import { Cookie } from "../../../constants"
 import * as configs from "../../../configs"
 import * as cache from "../../../cache"
 import * as utils from "../../../utils"
-import { UserCtx, SSOProfile } from "@budibase/types"
+import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types"
 import { ssoSaveUserNoOp } from "../sso/sso"
 
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
@@ -58,7 +58,14 @@ export async function postAuth(
   const platformUrl = await configs.getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = utils.getCookie<{ appId: string }>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+
+  if (!authStateCookie) {
+    throw new Error("Unable to fetch datasource auth cookie")
+  }
 
   return passport.authenticate(
     new GoogleStrategy(

packages/backend-core/src/objectStore/objectStore.ts

@@ -305,20 +305,33 @@ export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
-  let fullObjects = await Promise.all(
-    objects.map(obj => retrieve(bucketName, obj.Key!))
+  let streams = await Promise.all(
+    objects.map(obj => getReadStream(bucketName, obj.Key!))
   )
   let count = 0
+  const writePromises: Promise<Error>[] = []
   for (let obj of objects) {
     const filename = obj.Key!
-    const data = fullObjects[count++]
+    const stream = streams[count++]
     const possiblePath = filename.split("/")
-    if (possiblePath.length > 1) {
-      const dirs = possiblePath.slice(0, possiblePath.length - 1)
-      fs.mkdirSync(join(writePath, ...dirs), { recursive: true })
+    const dirs = possiblePath.slice(0, possiblePath.length - 1)
+    const possibleDir = join(writePath, ...dirs)
+    if (possiblePath.length > 1 && !fs.existsSync(possibleDir)) {
+      fs.mkdirSync(possibleDir, { recursive: true })
     }
-    fs.writeFileSync(join(writePath, ...possiblePath), data)
+    const writeStream = fs.createWriteStream(join(writePath, ...possiblePath), {
+      mode: 0o644,
+    })
+    stream.pipe(writeStream)
+    writePromises.push(
+      new Promise((resolve, reject) => {
+        stream.on("finish", resolve)
+        stream.on("error", reject)
+        writeStream.on("error", reject)
+      })
+    )
   }
+  await Promise.all(writePromises)
   return writePath
 }
 

packages/backend-core/src/security/encryption.ts

@@ -73,6 +73,9 @@ export async function encryptFile(
   const outputFileName = `${filename}.enc`
 
   const filePath = join(dir, filename)
+  if (fs.lstatSync(filePath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const inputFile = fs.createReadStream(filePath)
   const outputFile = fs.createWriteStream(join(dir, outputFileName))
 
@@ -110,6 +113,9 @@ export async function decryptFile(
   outputPath: string,
   secret: string
 ) {
+  if (fs.lstatSync(inputPath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const { salt, iv } = await getSaltAndIV(inputPath)
   const inputFile = fs.createReadStream(inputPath, {
     start: SALT_LENGTH + IV_LENGTH,

packages/backend-core/src/utils/utils.ts

@@ -11,8 +11,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 import type { SetOption } from "cookies"
-
-const jwt = require("jsonwebtoken")
+import jwt, { Secret } from "jsonwebtoken"
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
@@ -60,10 +59,7 @@ export function isServingApp(ctx: Ctx) {
     return true
   }
   // prod app
-  if (ctx.path.startsWith(PROD_APP_PREFIX)) {
-    return true
-  }
-  return false
+  return ctx.path.startsWith(PROD_APP_PREFIX)
 }
 
 export function isServingBuilder(ctx: Ctx): boolean {
@@ -138,16 +134,16 @@ function parseAppIdFromUrl(url?: string) {
  * opens the contents of the specified encrypted JWT.
  * @return the contents of the token.
  */
-export function openJwt(token: string) {
+export function openJwt<T>(token?: string): T | undefined {
   if (!token) {
-    return token
+    return undefined
   }
   try {
-    return jwt.verify(token, env.JWT_SECRET)
+    return jwt.verify(token, env.JWT_SECRET as Secret) as T
   } catch (e) {
     if (env.JWT_SECRET_FALLBACK) {
       // fallback to enable rotation
-      return jwt.verify(token, env.JWT_SECRET_FALLBACK)
+      return jwt.verify(token, env.JWT_SECRET_FALLBACK) as T
     } else {
       throw e
     }
@@ -159,13 +155,9 @@ export function isValidInternalAPIKey(apiKey: string) {
     return true
   }
   // fallback to enable rotation
-  if (
-    env.INTERNAL_API_KEY_FALLBACK &&
-    env.INTERNAL_API_KEY_FALLBACK === apiKey
-  ) {
-    return true
-  }
-  return false
+  return !!(
+    env.INTERNAL_API_KEY_FALLBACK && env.INTERNAL_API_KEY_FALLBACK === apiKey
+  )
 }
 
 /**
@@ -173,14 +165,14 @@ export function isValidInternalAPIKey(apiKey: string) {
  * @param ctx The request which is to be manipulated.
  * @param name The name of the cookie to get.
  */
-export function getCookie(ctx: Ctx, name: string) {
+export function getCookie<T>(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
-    return cookie
+    return undefined
   }
 
-  return openJwt(cookie)
+  return openJwt<T>(cookie)
 }
 
 /**
@@ -197,7 +189,7 @@ export function setCookie(
   opts = { sign: true }
 ) {
   if (value && opts && opts.sign) {
-    value = jwt.sign(value, env.JWT_SECRET)
+    value = jwt.sign(value, env.JWT_SECRET as Secret)
   }
 
   const config: SetOption = {

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -53,7 +53,7 @@
   $: {
     if (selectedImage?.url) {
       selectedUrl = selectedImage?.url
-    } else if (selectedImage) {
+    } else if (selectedImage && isImage) {
       try {
         let reader = new FileReader()
         reader.readAsDataURL(selectedImage)

packages/builder/src/components/start/ExportAppModal.svelte

@@ -13,7 +13,7 @@
   export let app
   export let published
   let includeInternalTablesRows = true
-  let encypt = true
+  let encrypt = true
 
   let password = null
   const validation = createValidationStore()
@@ -27,9 +27,9 @@
   $: stepConfig = {
     [Step.CONFIG]: {
       title: published ? "Export published app" : "Export latest app",
-      confirmText: encypt ? "Continue" : exportButtonText,
+      confirmText: encrypt ? "Continue" : exportButtonText,
       onConfirm: () => {
-        if (!encypt) {
+        if (!encrypt) {
           exportApp()
         } else {
           currentStep = Step.SET_PASSWORD
@@ -46,7 +46,7 @@
         if (!$validation.valid) {
           return keepOpen
         }
-        exportApp(password)
+        await exportApp(password)
       },
       isValid: $validation.valid,
     },
@@ -109,13 +109,13 @@
         text="Export rows from internal tables"
         bind:value={includeInternalTablesRows}
       />
-      <Toggle text="Encrypt my export" bind:value={encypt} />
+      <Toggle text="Encrypt my export" bind:value={encrypt} />
     </Body>
-    {#if !encypt}
-      <InlineAlert
-        header="Do not share your budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."
-      />
-    {/if}
+    <InlineAlert
+      header={encrypt
+        ? "Please note Budibase does not encrypt attachments during the export process to ensure efficient export of large attachments."
+        : "Do not share your Budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."}
+    />
   {/if}
   {#if currentStep === Step.SET_PASSWORD}
     <Input

packages/server/src/api/controllers/query/index.ts

@@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { ConfigType, Query, UserCtx } from "@budibase/types"
+import { ConfigType, Query, UserCtx, SessionCookie } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
@@ -113,7 +113,7 @@ function getOAuthConfigCookieId(ctx: UserCtx) {
 }
 
 function getAuthConfig(ctx: UserCtx) {
-  const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
+  const authCookie = utils.getCookie<SessionCookie>(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
   authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null

packages/server/src/environment.ts

@@ -59,6 +59,7 @@ const environment = {
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/koa.ts

@@ -1,5 +1,5 @@
 import env from "./environment"
-import Koa, { ExtendableContext } from "koa"
+import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
 import * as api from "./api"
@@ -27,6 +27,9 @@ export default function createKoaApp() {
       // @ts-ignore
       enableTypes: ["json", "form", "text"],
       parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
+      formidable: {
+        maxFileSize: parseInt(env.MAX_IMPORT_SIZE_MB || "100") * 1024 * 1024,
+      },
     })
   )
 

packages/server/src/sdk/app/backups/constants.ts

@@ -1,3 +1,4 @@
 export const DB_EXPORT_FILE = "db.txt"
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
 export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"]
+export const ATTACHMENT_DIRECTORY = "attachments"

packages/server/src/sdk/app/backups/exports.ts

@@ -8,13 +8,15 @@ import {
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../../db/utils"
-import { DB_EXPORT_FILE, STATIC_APP_FILES } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  STATIC_APP_FILES,
+  ATTACHMENT_DIRECTORY,
+} from "./constants"
 import fs from "fs"
 import { join } from "path"
 import env from "../../../environment"
-
-const uuid = require("uuid/v4")
-
+import { v4 as uuid } from "uuid"
 import tar from "tar"
 
 const MemoryStream = require("memorystream")
@@ -30,12 +32,11 @@ export interface ExportOpts extends DBDumpOpts {
   encryptPassword?: string
 }
 
-function tarFilesToTmp(tmpDir: string, files: string[]) {
+async function tarFilesToTmp(tmpDir: string, files: string[]) {
   const fileName = `${uuid()}.tar.gz`
   const exportFile = join(budibaseTempDir(), fileName)
-  tar.create(
+  await tar.create(
     {
-      sync: true,
       gzip: true,
       file: exportFile,
       noDirRecurse: false,
@@ -150,19 +151,21 @@ export async function exportApp(appId: string, config?: ExportOpts) {
     for (let file of fs.readdirSync(tmpPath)) {
       const path = join(tmpPath, file)
 
-      await encryption.encryptFile(
-        { dir: tmpPath, filename: file },
-        config.encryptPassword
-      )
-
-      fs.rmSync(path)
+      // skip the attachments - too big to encrypt
+      if (file !== ATTACHMENT_DIRECTORY) {
+        await encryption.encryptFile(
+          { dir: tmpPath, filename: file },
+          config.encryptPassword
+        )
+        fs.rmSync(path)
+      }
     }
   }
 
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
 

packages/server/src/sdk/app/backups/imports.ts

@@ -6,17 +6,20 @@ import {
   AutomationTriggerStepId,
   RowAttachment,
 } from "@budibase/types"
-import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  GLOBAL_DB_EXPORT_FILE,
+  ATTACHMENT_DIRECTORY,
+} from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-
-const uuid = require("uuid/v4")
-const tar = require("tar")
+import { v4 as uuid } from "uuid"
+import tar from "tar"
 
 type TemplateType = {
   file?: {
@@ -114,12 +117,11 @@ async function getTemplateStream(template: TemplateType) {
   }
 }
 
-export function untarFile(file: { path: string }) {
+export async function untarFile(file: { path: string }) {
   const tmpPath = join(budibaseTempDir(), uuid())
   fs.mkdirSync(tmpPath)
   // extract the tarball
-  tar.extract({
-    sync: true,
+  await tar.extract({
     cwd: tmpPath,
     file: file.path,
   })
@@ -130,9 +132,11 @@ async function decryptFiles(path: string, password: string) {
   try {
     for (let file of fs.readdirSync(path)) {
       const inputPath = join(path, file)
-      const outputPath = inputPath.replace(/\.enc$/, "")
-      await encryption.decryptFile(inputPath, outputPath, password)
-      fs.rmSync(inputPath)
+      if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) {
+        const outputPath = inputPath.replace(/\.enc$/, "")
+        await encryption.decryptFile(inputPath, outputPath, password)
+        fs.rmSync(inputPath)
+      }
     }
   } catch (err: any) {
     if (err.message === "incorrect header check") {
@@ -162,7 +166,7 @@ export async function importApp(
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -56,6 +56,7 @@ import {
 
 import API from "./api"
 import { cloneDeep } from "lodash"
+import jwt, { Secret } from "jsonwebtoken"
 
 mocks.licenses.init(pro)
 
@@ -391,7 +392,7 @@ class TestConfiguration {
         sessionId: "sessionid",
         tenantId: this.getTenantId(),
       }
-      const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
+      const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
 
       // returning necessary request headers
       await cache.user.invalidateUser(userId)
@@ -412,7 +413,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId,
     }
-    const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
+    const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
 
     const headers: any = {
       Accept: "application/json",

packages/server/src/utilities/rowProcessor/index.ts

@@ -249,7 +249,9 @@ export async function outputProcessing<T extends Row[] | Row>(
           continue
         }
         row[property].forEach((attachment: RowAttachment) => {
-          attachment.url ??= objectStore.getAppFileUrl(attachment.key)
+          if (!attachment.url) {
+            attachment.url = objectStore.getAppFileUrl(attachment.key)
+          }
         })
       }
     } else if (

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -3,6 +3,7 @@ import {
   FieldType,
   FieldTypeSubtypes,
   INTERNAL_TABLE_SOURCE_ID,
+  RowAttachment,
   Table,
   TableSourceType,
 } from "@budibase/types"
@@ -70,6 +71,49 @@ describe("rowProcessor - outputProcessing", () => {
     )
   })
 
+  it("should handle attachments correctly", async () => {
+    const table: Table = {
+      _id: generator.guid(),
+      name: "TestTable",
+      type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        attach: {
+          type: FieldType.ATTACHMENT,
+          name: "attach",
+          constraints: {},
+        },
+      },
+    }
+
+    const row: { attach: RowAttachment[] } = {
+      attach: [
+        {
+          size: 10,
+          name: "test",
+          extension: "jpg",
+          key: "test.jpg",
+        },
+      ],
+    }
+
+    const output = await outputProcessing(table, row, { squash: false })
+    expect(output.attach[0].url).toBe(
+      "/files/signed/prod-budi-app-assets/test.jpg"
+    )
+
+    row.attach[0].url = ""
+    const output2 = await outputProcessing(table, row, { squash: false })
+    expect(output2.attach[0].url).toBe(
+      "/files/signed/prod-budi-app-assets/test.jpg"
+    )
+
+    row.attach[0].url = "aaaa"
+    const output3 = await outputProcessing(table, row, { squash: false })
+    expect(output3.attach[0].url).toBe("aaaa")
+  })
+
   it("process output even when the field is not empty", async () => {
     const table: Table = {
       _id: generator.guid(),

packages/types/src/api/web/cookies.ts

@@ -0,0 +1,9 @@
+export interface DatasourceAuthCookie {
+  appId: string
+  provider: string
+}
+
+export interface SessionCookie {
+  sessionId: string
+  userId: string
+}

packages/types/src/api/web/index.ts

@@ -9,3 +9,4 @@ export * from "./app"
 export * from "./global"
 export * from "./pagination"
 export * from "./searchFilter"
+export * from "./cookies"

packages/worker/src/api/controllers/global/auth.ts

@@ -15,6 +15,7 @@ import {
   PasswordResetRequest,
   PasswordResetUpdateRequest,
   GoogleInnerConfig,
+  DatasourceAuthCookie,
 } from "@budibase/types"
 import env from "../../../environment"
 
@@ -148,7 +149,13 @@ export const datasourcePreAuth = async (ctx: any, next: any) => {
 }
 
 export const datasourceAuth = async (ctx: any, next: any) => {
-  const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = getCookie<DatasourceAuthCookie>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+  if (!authStateCookie) {
+    throw new Error("Unable to retrieve datasource authentication cookie")
+  }
   const provider = authStateCookie.provider
   const { middleware } = require(`@budibase/backend-core`)
   const handler = middleware.datasource[provider]

packages/worker/src/tests/TestConfiguration.ts

@@ -35,6 +35,7 @@ import {
   ConfigType,
 } from "@budibase/types"
 import API from "./api"
+import jwt, { Secret } from "jsonwebtoken"
 
 class TestConfiguration {
   server: any
@@ -209,7 +210,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId: user.tenantId,
     }
-    const authCookie = auth.jwt.sign(authToken, coreEnv.JWT_SECRET)
+    const authCookie = jwt.sign(authToken, coreEnv.JWT_SECRET as Secret)
     return {
       Accept: "application/json",
       ...this.cookieHeader([`${constants.Cookie.Auth}=${authCookie}`]),
@@ -327,7 +328,7 @@ class TestConfiguration {
   // CONFIGS - OIDC
 
   getOIDConfigCookie(configId: string) {
-    const token = auth.jwt.sign(configId, coreEnv.JWT_SECRET)
+    const token = jwt.sign(configId, coreEnv.JWT_SECRET as Secret)
     return this.cookieHeader([[`${constants.Cookie.OIDC_CONFIG}=${token}`]])
   }