Merge branch 'master' of github.com:Budibase/budibase into feature/multistep-form-block

hosting/couchdb/runner.sh

@@ -30,6 +30,13 @@ elif [[ "${TARGETBUILD}" = "single" ]]; then
     # mount, so we use that for all persistent data.
     sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
     sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in docker-compose because it will default to /opt/couchdb/data which is what
+    # our docker-compose was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
 elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
     # In Kubernetes the directory /opt/couchdb/data has a persistent volume
     # mount for storing database data.

hosting/nginx.dev.conf

@@ -42,7 +42,7 @@ http {
   server { 
     listen       10000 default_server;
     server_name  _;
-    client_max_body_size 1000m;
+    client_max_body_size 50000m;
     ignore_invalid_headers off;
     proxy_buffering off;
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.32",
+  "version": "2.13.35",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/auth/auth.ts

@@ -19,6 +19,7 @@ import {
   GoogleInnerConfig,
   OIDCInnerConfig,
   PlatformLogoutOpts,
+  SessionCookie,
   SSOProviderType,
 } from "@budibase/types"
 import * as events from "../events"
@@ -44,7 +45,6 @@ export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
 export const passport = _passport
-export const jwt = require("jsonwebtoken")
 
 // Strategies
 _passport.use(new LocalStrategy(local.options, local.authenticate))
@@ -191,10 +191,10 @@ export async function platformLogout(opts: PlatformLogoutOpts) {
 
   if (!ctx) throw new Error("Koa context must be supplied to logout.")
 
-  const currentSession = getCookie(ctx, Cookie.Auth)
+  const currentSession = getCookie<SessionCookie>(ctx, Cookie.Auth)
   let sessions = await getSessionsForUser(userId)
 
-  if (keepActiveSession) {
+  if (currentSession && keepActiveSession) {
     sessions = sessions.filter(
       session => session.sessionId !== currentSession.sessionId
     )

packages/backend-core/src/middleware/authenticated.ts

@@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { Ctx, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types"
 import { InvalidAPIKeyError, ErrorCode } from "../errors"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
@@ -98,7 +98,9 @@ export default function (
       // check the actual user is authenticated first, try header or cookie
       let headerToken = ctx.request.headers[Header.TOKEN]
 
-      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
+      const authCookie =
+        getCookie<SessionCookie>(ctx, Cookie.Auth) ||
+        openJwt<SessionCookie>(headerToken)
       let apiKey = ctx.request.headers[Header.API_KEY]
 
       if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -3,7 +3,7 @@ import { Cookie } from "../../../constants"
 import * as configs from "../../../configs"
 import * as cache from "../../../cache"
 import * as utils from "../../../utils"
-import { UserCtx, SSOProfile } from "@budibase/types"
+import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types"
 import { ssoSaveUserNoOp } from "../sso/sso"
 
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
@@ -58,7 +58,14 @@ export async function postAuth(
   const platformUrl = await configs.getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = utils.getCookie<{ appId: string }>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+
+  if (!authStateCookie) {
+    throw new Error("Unable to fetch datasource auth cookie")
+  }
 
   return passport.authenticate(
     new GoogleStrategy(

packages/backend-core/src/objectStore/objectStore.ts

@@ -305,20 +305,33 @@ export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
-  let fullObjects = await Promise.all(
+  let streams = await Promise.all(
-    objects.map(obj => retrieve(bucketName, obj.Key!))
+    objects.map(obj => getReadStream(bucketName, obj.Key!))
   )
   let count = 0
+  const writePromises: Promise<Error>[] = []
   for (let obj of objects) {
     const filename = obj.Key!
-    const data = fullObjects[count++]
+    const stream = streams[count++]
     const possiblePath = filename.split("/")
-    if (possiblePath.length > 1) {
+    const dirs = possiblePath.slice(0, possiblePath.length - 1)
-      const dirs = possiblePath.slice(0, possiblePath.length - 1)
+    const possibleDir = join(writePath, ...dirs)
-      fs.mkdirSync(join(writePath, ...dirs), { recursive: true })
+    if (possiblePath.length > 1 && !fs.existsSync(possibleDir)) {
+      fs.mkdirSync(possibleDir, { recursive: true })
     }
-    fs.writeFileSync(join(writePath, ...possiblePath), data)
+    const writeStream = fs.createWriteStream(join(writePath, ...possiblePath), {
+      mode: 0o644,
+    })
+    stream.pipe(writeStream)
+    writePromises.push(
+      new Promise((resolve, reject) => {
+        stream.on("finish", resolve)
+        stream.on("error", reject)
+        writeStream.on("error", reject)
+      })
+    )
   }
+  await Promise.all(writePromises)
   return writePath
 }
 

packages/backend-core/src/security/encryption.ts

@@ -73,6 +73,9 @@ export async function encryptFile(
   const outputFileName = `${filename}.enc`
 
   const filePath = join(dir, filename)
+  if (fs.lstatSync(filePath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const inputFile = fs.createReadStream(filePath)
   const outputFile = fs.createWriteStream(join(dir, outputFileName))
 
@@ -110,6 +113,9 @@ export async function decryptFile(
   outputPath: string,
   secret: string
 ) {
+  if (fs.lstatSync(inputPath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const { salt, iv } = await getSaltAndIV(inputPath)
   const inputFile = fs.createReadStream(inputPath, {
     start: SALT_LENGTH + IV_LENGTH,

packages/backend-core/src/utils/utils.ts

@@ -11,8 +11,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 import type { SetOption } from "cookies"
-
+import jwt, { Secret } from "jsonwebtoken"
-const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
@@ -60,10 +59,7 @@ export function isServingApp(ctx: Ctx) {
     return true
   }
   // prod app
-  if (ctx.path.startsWith(PROD_APP_PREFIX)) {
+  return ctx.path.startsWith(PROD_APP_PREFIX)
-    return true
-  }
-  return false
 }
 
 export function isServingBuilder(ctx: Ctx): boolean {
@@ -138,16 +134,16 @@ function parseAppIdFromUrl(url?: string) {
  * opens the contents of the specified encrypted JWT.
  * @return the contents of the token.
  */
-export function openJwt(token: string) {
+export function openJwt<T>(token?: string): T | undefined {
   if (!token) {
-    return token
+    return undefined
   }
   try {
-    return jwt.verify(token, env.JWT_SECRET)
+    return jwt.verify(token, env.JWT_SECRET as Secret) as T
   } catch (e) {
     if (env.JWT_SECRET_FALLBACK) {
       // fallback to enable rotation
-      return jwt.verify(token, env.JWT_SECRET_FALLBACK)
+      return jwt.verify(token, env.JWT_SECRET_FALLBACK) as T
     } else {
       throw e
     }
@@ -159,13 +155,9 @@ export function isValidInternalAPIKey(apiKey: string) {
     return true
   }
   // fallback to enable rotation
-  if (
+  return !!(
-    env.INTERNAL_API_KEY_FALLBACK &&
+    env.INTERNAL_API_KEY_FALLBACK && env.INTERNAL_API_KEY_FALLBACK === apiKey
-    env.INTERNAL_API_KEY_FALLBACK === apiKey
+  )
-  ) {
-    return true
-  }
-  return false
 }
 
 /**
@@ -173,14 +165,14 @@ export function isValidInternalAPIKey(apiKey: string) {
  * @param ctx The request which is to be manipulated.
  * @param name The name of the cookie to get.
  */
-export function getCookie(ctx: Ctx, name: string) {
+export function getCookie<T>(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
-    return cookie
+    return undefined
   }
 
-  return openJwt(cookie)
+  return openJwt<T>(cookie)
 }
 
 /**
@@ -197,7 +189,7 @@ export function setCookie(
   opts = { sign: true }
 ) {
   if (value && opts && opts.sign) {
-    value = jwt.sign(value, env.JWT_SECRET)
+    value = jwt.sign(value, env.JWT_SECRET as Secret)
   }
 
   const config: SetOption = {

packages/bbui/src/DetailSummary/DetailSummary.svelte

@@ -1,21 +1,18 @@
 <script>
   import Icon from "../Icon/Icon.svelte"
-  import { createEventDispatcher } from "svelte"
 
   export let name
-  export let show = false
+  export let initiallyShow = false
   export let collapsible = true
   export let noPadding = false
 
-  const dispatch = createEventDispatcher()
+  let show = initiallyShow
+
   const onHeaderClick = () => {
     if (!collapsible) {
       return
     }
     show = !show
-    if (show) {
-      dispatch("open")
-    }
   }
 </script>
 

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -53,7 +53,7 @@
   $: {
     if (selectedImage?.url) {
       selectedUrl = selectedImage?.url
-    } else if (selectedImage) {
+    } else if (selectedImage && isImage) {
       try {
         let reader = new FileReader()
         reader.readAsDataURL(selectedImage)

packages/builder/src/builderStore/store/frontend.js

@@ -85,6 +85,7 @@ const INITIAL_FRONTEND_STATE = {
   selectedScreenId: null,
   selectedComponentId: null,
   selectedLayoutId: null,
+  hoverComponentId: null,
 
   // Client state
   selectedComponentInstance: null,
@@ -112,7 +113,7 @@ export const getFrontendStore = () => {
     }
     let clone = cloneDeep(screen)
     const result = patchFn(clone)
-
+    // An explicit false result means skip this change
     if (result === false) {
       return
     }
@@ -879,11 +880,14 @@ export const getFrontendStore = () => {
           }
 
           // Mutates the fetched component with updates
-          const updated = patchFn(component, screen)
+          const patchResult = patchFn(component, screen)
+
           // Mutates the component with any required settings updates
           const migrated = store.actions.components.migrateSettings(component)
 
-          return updated || migrated
+          // Returning an explicit false signifies that we should skip this
+          // update. If we migrated something, ensure we never skip.
+          return migrated ? null : patchResult
         }
         await store.actions.screens.patch(patchScreen, screenId)
       },

packages/builder/src/components/common/NavItem.svelte

@@ -23,6 +23,7 @@
   export let showTooltip = false
   export let selectedBy = null
   export let compact = false
+  export let hovering = false
 
   const scrollApi = getContext("scroll")
   const dispatch = createEventDispatcher()
@@ -61,6 +62,7 @@
 
 <div
   class="nav-item"
+  class:hovering
   class:border
   class:selected
   class:withActions
@@ -71,6 +73,8 @@
   on:dragstart
   on:dragover
   on:drop
+  on:mouseenter
+  on:mouseleave
   on:click={onClick}
   ondragover="return false"
   ondragenter="return false"
@@ -152,15 +156,17 @@
     --avatars-background: var(--spectrum-global-color-gray-200);
   }
   .nav-item.selected {
-    background-color: var(--spectrum-global-color-gray-300);
+    background-color: var(--spectrum-global-color-gray-300) !important;
     --avatars-background: var(--spectrum-global-color-gray-300);
     color: var(--ink);
   }
-  .nav-item:hover {
+  .nav-item:hover,
-    background-color: var(--spectrum-global-color-gray-300);
+  .hovering {
+    background-color: var(--spectrum-global-color-gray-200);
     --avatars-background: var(--spectrum-global-color-gray-300);
   }
-  .nav-item:hover .actions {
+  .nav-item:hover .actions,
+  .hovering .actions {
     visibility: visible;
   }
   .nav-item-content {

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte

@@ -49,7 +49,15 @@
     <div class="field-label">{item.label || item.field}</div>
   </div>
   <div class="list-item-right">
-    <Toggle on:change={onToggle(item)} text="" value={item.active} thin />
+    <Toggle
+      on:click={e => {
+        e.stopPropagation()
+      }}
+      on:change={onToggle(item)}
+      text=""
+      value={item.active}
+      thin
+    />
   </div>
 </div>
 

packages/builder/src/components/start/ExportAppModal.svelte

@@ -13,7 +13,7 @@
   export let app
   export let published
   let includeInternalTablesRows = true
-  let encypt = true
+  let encrypt = true
 
   let password = null
   const validation = createValidationStore()
@@ -27,9 +27,9 @@
   $: stepConfig = {
     [Step.CONFIG]: {
       title: published ? "Export published app" : "Export latest app",
-      confirmText: encypt ? "Continue" : exportButtonText,
+      confirmText: encrypt ? "Continue" : exportButtonText,
       onConfirm: () => {
-        if (!encypt) {
+        if (!encrypt) {
           exportApp()
         } else {
           currentStep = Step.SET_PASSWORD
@@ -46,7 +46,7 @@
         if (!$validation.valid) {
           return keepOpen
         }
-        exportApp(password)
+        await exportApp(password)
       },
       isValid: $validation.valid,
     },
@@ -109,13 +109,13 @@
         text="Export rows from internal tables"
         bind:value={includeInternalTablesRows}
       />
-      <Toggle text="Encrypt my export" bind:value={encypt} />
+      <Toggle text="Encrypt my export" bind:value={encrypt} />
     </Body>
-    {#if !encypt}
+    <InlineAlert
-      <InlineAlert
+      header={encrypt
-        header="Do not share your budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."
+        ? "Please note Budibase does not encrypt attachments during the export process to ensure efficient export of large attachments."
-      />
+        : "Do not share your Budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."}
-    {/if}
+    />
   {/if}
   {#if currentStep === Step.SET_PASSWORD}
     <Input

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -35,6 +35,7 @@
     const generalSettings = settings.filter(
       setting => !setting.section && setting.tag === tag
     )
+
     const customSections = settings.filter(
       setting => setting.section && setting.tag === tag
     )
@@ -173,6 +174,7 @@
       name={showSectionTitle ? section.name : ""}
       show={section.collapsed !== true}
       {noPadding}
+      initiallyShow={section.collapsed !== true}
     >
       {#if section.info}
         <div class="section-info">

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -36,12 +36,14 @@
 
   // Determine selected component ID
   $: selectedComponentId = $store.selectedComponentId
+  $: hoverComponentId = $store.hoverComponentId
 
   $: previewData = {
     appId: $store.appId,
     layout,
     screen,
     selectedComponentId,
+    hoverComponentId,
     theme: $store.theme,
     customTheme: $store.customTheme,
     previewDevice: $store.previewDevice,
@@ -117,6 +119,8 @@
       error = event.error || "An unknown error occurred"
     } else if (type === "select-component" && data.id) {
       $store.selectedComponentId = data.id
+    } else if (type === "hover-component" && data.id) {
+      $store.hoverComponentId = data.id
     } else if (type === "update-prop") {
       await store.actions.components.updateSetting(data.prop, data.value)
     } else if (type === "update-styles") {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -89,6 +89,17 @@
     }
     return findComponentPath($selectedComponent, component._id)?.length > 0
   }
+
+  const handleMouseover = componentId => {
+    if ($store.hoverComponentId !== componentId) {
+      $store.hoverComponentId = componentId
+    }
+  }
+  const handleMouseout = componentId => {
+    if ($store.hoverComponentId === componentId) {
+      $store.hoverComponentId = null
+    }
+  }
 </script>
 
 <ul>
@@ -109,6 +120,9 @@
         on:dragover={dragover(component, index)}
         on:iconClick={() => toggleNodeOpen(component._id)}
         on:drop={onDrop}
+        hovering={$store.hoverComponentId === component._id}
+        on:mouseenter={() => handleMouseover(component._id)}
+        on:mouseleave={() => handleMouseout(component._id)}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
         iconTooltip={getComponentName(component)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte

@@ -32,6 +32,17 @@
   const handleScroll = e => {
     scrolling = e.target.scrollTop !== 0
   }
+
+  const handleMouseover = componentId => {
+    if ($store.hoverComponentId !== componentId) {
+      $store.hoverComponentId = componentId
+    }
+  }
+  const handleMouseout = componentId => {
+    if ($store.hoverComponentId === componentId) {
+      $store.hoverComponentId = null
+    }
+  }
 </script>
 
 <div class="components">
@@ -57,6 +68,12 @@
             on:click={() => {
               $store.selectedComponentId = `${$store.selectedScreenId}-screen`
             }}
+            hovering={$store.hoverComponentId ===
+              `${$store.selectedScreenId}-screen`}
+            on:mouseenter={() =>
+              handleMouseover(`${$store.selectedScreenId}-screen`)}
+            on:mouseleave={() =>
+              handleMouseout(`${$store.selectedScreenId}-screen`)}
             id={`component-screen`}
             selectedBy={$userSelectedResourceMap[
               `${$store.selectedScreenId}-screen`
@@ -78,6 +95,12 @@
             on:click={() => {
               $store.selectedComponentId = `${$store.selectedScreenId}-navigation`
             }}
+            hovering={$store.hoverComponentId ===
+              `${$store.selectedScreenId}-navigation`}
+            on:mouseenter={() =>
+              handleMouseover(`${$store.selectedScreenId}-navigation`)}
+            on:mouseleave={() =>
+              handleMouseout(`${$store.selectedScreenId}-navigation`)}
             id={`component-nav`}
             selectedBy={$userSelectedResourceMap[
               `${$store.selectedScreenId}-navigation`

packages/cli/package.json

@@ -15,7 +15,8 @@
     "pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip",
     "build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
-    "postbuild": "rm -rf prebuilds 2> /dev/null"
+    "postbuild": "rm -rf prebuilds 2> /dev/null",
+    "start": "ts-node ./src/index.ts"
   },
   "pkg": {
     "targets": [

packages/client/src/components/preview/HoverIndicator.svelte

@@ -3,8 +3,7 @@
   import IndicatorSet from "./IndicatorSet.svelte"
   import { builderStore, dndIsDragging } from "stores"
 
-  let componentId
+  $: componentId = $builderStore.hoverComponentId
-
   $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
 
   const onMouseOver = e => {
@@ -24,12 +23,12 @@
     }
 
     if (newId !== componentId) {
-      componentId = newId
+      builderStore.actions.hoverComponent(newId)
     }
   }
 
   const onMouseLeave = () => {
-    componentId = null
+    builderStore.actions.hoverComponent(null)
   }
 
   onMount(() => {

packages/client/src/index.js

@@ -32,6 +32,7 @@ const loadBudibase = async () => {
     layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
+    hoverComponentId: window["##BUDIBASE_HOVER_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
     theme: window["##BUDIBASE_PREVIEW_THEME##"],
     customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],

packages/client/src/stores/builder.js

@@ -8,6 +8,7 @@ const createBuilderStore = () => {
     inBuilder: false,
     screen: null,
     selectedComponentId: null,
+    hoverComponentId: null,
     editMode: false,
     previewId: null,
     theme: null,
@@ -23,6 +24,16 @@ const createBuilderStore = () => {
   }
   const store = writable(initialState)
   const actions = {
+    hoverComponent: id => {
+      if (id === get(store).hoverComponentId) {
+        return
+      }
+      store.update(state => ({
+        ...state,
+        hoverComponentId: id,
+      }))
+      eventStore.actions.dispatchEvent("hover-component", { id })
+    },
     selectComponent: id => {
       if (id === get(store).selectedComponentId) {
         return

packages/server/src/api/controllers/query/index.ts

@@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { ConfigType, Query, UserCtx } from "@budibase/types"
+import { ConfigType, Query, UserCtx, SessionCookie } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
@@ -113,7 +113,7 @@ function getOAuthConfigCookieId(ctx: UserCtx) {
 }
 
 function getAuthConfig(ctx: UserCtx) {
-  const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
+  const authCookie = utils.getCookie<SessionCookie>(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
   authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -63,6 +63,7 @@
       // Extract data from message
       const {
         selectedComponentId,
+        hoverComponentId,
         layout,
         screen,
         appId,
@@ -81,6 +82,7 @@
       window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
       window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
       window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
+      window["##BUDIBASE_HOVER_COMPONENT_ID##"] = hoverComponentId
       window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
       window["##BUDIBASE_PREVIEW_THEME##"] = theme
       window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme

packages/server/src/environment.ts

@@ -59,6 +59,7 @@ const environment = {
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/koa.ts

@@ -1,5 +1,5 @@
 import env from "./environment"
-import Koa, { ExtendableContext } from "koa"
+import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
 import * as api from "./api"
@@ -27,6 +27,9 @@ export default function createKoaApp() {
       // @ts-ignore
       enableTypes: ["json", "form", "text"],
       parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
+      formidable: {
+        maxFileSize: parseInt(env.MAX_IMPORT_SIZE_MB || "100") * 1024 * 1024,
+      },
     })
   )
 

packages/server/src/sdk/app/backups/constants.ts

@@ -1,3 +1,4 @@
 export const DB_EXPORT_FILE = "db.txt"
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
 export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"]
+export const ATTACHMENT_DIRECTORY = "attachments"

packages/server/src/sdk/app/backups/exports.ts

@@ -8,13 +8,15 @@ import {
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../../db/utils"
-import { DB_EXPORT_FILE, STATIC_APP_FILES } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  STATIC_APP_FILES,
+  ATTACHMENT_DIRECTORY,
+} from "./constants"
 import fs from "fs"
 import { join } from "path"
 import env from "../../../environment"
-
+import { v4 as uuid } from "uuid"
-const uuid = require("uuid/v4")
-
 import tar from "tar"
 
 const MemoryStream = require("memorystream")
@@ -30,12 +32,11 @@ export interface ExportOpts extends DBDumpOpts {
   encryptPassword?: string
 }
 
-function tarFilesToTmp(tmpDir: string, files: string[]) {
+async function tarFilesToTmp(tmpDir: string, files: string[]) {
   const fileName = `${uuid()}.tar.gz`
   const exportFile = join(budibaseTempDir(), fileName)
-  tar.create(
+  await tar.create(
     {
-      sync: true,
       gzip: true,
       file: exportFile,
       noDirRecurse: false,
@@ -150,19 +151,21 @@ export async function exportApp(appId: string, config?: ExportOpts) {
     for (let file of fs.readdirSync(tmpPath)) {
       const path = join(tmpPath, file)
 
-      await encryption.encryptFile(
+      // skip the attachments - too big to encrypt
-        { dir: tmpPath, filename: file },
+      if (file !== ATTACHMENT_DIRECTORY) {
-        config.encryptPassword
+        await encryption.encryptFile(
-      )
+          { dir: tmpPath, filename: file },
-
+          config.encryptPassword
-      fs.rmSync(path)
+        )
+        fs.rmSync(path)
+      }
     }
   }
 
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
 

packages/server/src/sdk/app/backups/imports.ts

@@ -6,17 +6,20 @@ import {
   AutomationTriggerStepId,
   RowAttachment,
 } from "@budibase/types"
-import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
-import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  GLOBAL_DB_EXPORT_FILE,
+  ATTACHMENT_DIRECTORY,
+} from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-
+import { v4 as uuid } from "uuid"
-const uuid = require("uuid/v4")
+import tar from "tar"
-const tar = require("tar")
 
 type TemplateType = {
   file?: {
@@ -114,12 +117,11 @@ async function getTemplateStream(template: TemplateType) {
   }
 }
 
-export function untarFile(file: { path: string }) {
+export async function untarFile(file: { path: string }) {
   const tmpPath = join(budibaseTempDir(), uuid())
   fs.mkdirSync(tmpPath)
   // extract the tarball
-  tar.extract({
+  await tar.extract({
-    sync: true,
     cwd: tmpPath,
     file: file.path,
   })
@@ -130,9 +132,11 @@ async function decryptFiles(path: string, password: string) {
   try {
     for (let file of fs.readdirSync(path)) {
       const inputPath = join(path, file)
-      const outputPath = inputPath.replace(/\.enc$/, "")
+      if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) {
-      await encryption.decryptFile(inputPath, outputPath, password)
+        const outputPath = inputPath.replace(/\.enc$/, "")
-      fs.rmSync(inputPath)
+        await encryption.decryptFile(inputPath, outputPath, password)
+        fs.rmSync(inputPath)
+      }
     }
   } catch (err: any) {
     if (err.message === "incorrect header check") {
@@ -162,7 +166,7 @@ export async function importApp(
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -56,6 +56,7 @@ import {
 
 import API from "./api"
 import { cloneDeep } from "lodash"
+import jwt, { Secret } from "jsonwebtoken"
 
 mocks.licenses.init(pro)
 
@@ -391,7 +392,7 @@ class TestConfiguration {
         sessionId: "sessionid",
         tenantId: this.getTenantId(),
       }
-      const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
+      const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
 
       // returning necessary request headers
       await cache.user.invalidateUser(userId)
@@ -412,7 +413,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId,
     }
-    const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
+    const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret)
 
     const headers: any = {
       Accept: "application/json",

packages/server/src/utilities/rowProcessor/index.ts

@@ -249,7 +249,9 @@ export async function outputProcessing<T extends Row[] | Row>(
           continue
         }
         row[property].forEach((attachment: RowAttachment) => {
-          attachment.url ??= objectStore.getAppFileUrl(attachment.key)
+          if (!attachment.url) {
+            attachment.url = objectStore.getAppFileUrl(attachment.key)
+          }
         })
       }
     } else if (

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -3,6 +3,7 @@ import {
   FieldType,
   FieldTypeSubtypes,
   INTERNAL_TABLE_SOURCE_ID,
+  RowAttachment,
   Table,
   TableSourceType,
 } from "@budibase/types"
@@ -70,6 +71,49 @@ describe("rowProcessor - outputProcessing", () => {
     )
   })
 
+  it("should handle attachments correctly", async () => {
+    const table: Table = {
+      _id: generator.guid(),
+      name: "TestTable",
+      type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        attach: {
+          type: FieldType.ATTACHMENT,
+          name: "attach",
+          constraints: {},
+        },
+      },
+    }
+
+    const row: { attach: RowAttachment[] } = {
+      attach: [
+        {
+          size: 10,
+          name: "test",
+          extension: "jpg",
+          key: "test.jpg",
+        },
+      ],
+    }
+
+    const output = await outputProcessing(table, row, { squash: false })
+    expect(output.attach[0].url).toBe(
+      "/files/signed/prod-budi-app-assets/test.jpg"
+    )
+
+    row.attach[0].url = ""
+    const output2 = await outputProcessing(table, row, { squash: false })
+    expect(output2.attach[0].url).toBe(
+      "/files/signed/prod-budi-app-assets/test.jpg"
+    )
+
+    row.attach[0].url = "aaaa"
+    const output3 = await outputProcessing(table, row, { squash: false })
+    expect(output3.attach[0].url).toBe("aaaa")
+  })
+
   it("process output even when the field is not empty", async () => {
     const table: Table = {
       _id: generator.guid(),

packages/types/src/api/web/cookies.ts

@@ -0,0 +1,9 @@
+export interface DatasourceAuthCookie {
+  appId: string
+  provider: string
+}
+
+export interface SessionCookie {
+  sessionId: string
+  userId: string
+}

packages/types/src/api/web/index.ts

@@ -9,3 +9,4 @@ export * from "./app"
 export * from "./global"
 export * from "./pagination"
 export * from "./searchFilter"
+export * from "./cookies"

packages/worker/src/api/controllers/global/auth.ts

@@ -15,6 +15,7 @@ import {
   PasswordResetRequest,
   PasswordResetUpdateRequest,
   GoogleInnerConfig,
+  DatasourceAuthCookie,
 } from "@budibase/types"
 import env from "../../../environment"
 
@@ -148,7 +149,13 @@ export const datasourcePreAuth = async (ctx: any, next: any) => {
 }
 
 export const datasourceAuth = async (ctx: any, next: any) => {
-  const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = getCookie<DatasourceAuthCookie>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+  if (!authStateCookie) {
+    throw new Error("Unable to retrieve datasource authentication cookie")
+  }
   const provider = authStateCookie.provider
   const { middleware } = require(`@budibase/backend-core`)
   const handler = middleware.datasource[provider]

packages/worker/src/tests/TestConfiguration.ts

@@ -35,6 +35,7 @@ import {
   ConfigType,
 } from "@budibase/types"
 import API from "./api"
+import jwt, { Secret } from "jsonwebtoken"
 
 class TestConfiguration {
   server: any
@@ -209,7 +210,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId: user.tenantId,
     }
-    const authCookie = auth.jwt.sign(authToken, coreEnv.JWT_SECRET)
+    const authCookie = jwt.sign(authToken, coreEnv.JWT_SECRET as Secret)
     return {
       Accept: "application/json",
       ...this.cookieHeader([`${constants.Cookie.Auth}=${authCookie}`]),
@@ -327,7 +328,7 @@ class TestConfiguration {
   // CONFIGS - OIDC
 
   getOIDConfigCookie(configId: string) {
-    const token = auth.jwt.sign(configId, coreEnv.JWT_SECRET)
+    const token = jwt.sign(configId, coreEnv.JWT_SECRET as Secret)
     return this.cookieHeader([[`${constants.Cookie.OIDC_CONFIG}=${token}`]])
   }