diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index b576c886c2..19c612a17e 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -30,6 +30,13 @@ elif [[ "${TARGETBUILD}" = "single" ]]; then # mount, so we use that for all persistent data. sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini +elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then + # We remove the database_dir and view_index_dir settings from the local.ini + # in docker-compose because it will default to /opt/couchdb/data which is what + # our docker-compose was using prior to us switching to using our own CouchDB + # image. + sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini + sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then # In Kubernetes the directory /opt/couchdb/data has a persistent volume # mount for storing database data. diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index 915125cbce..f0a58a9a98 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -42,7 +42,7 @@ http { server { listen 10000 default_server; server_name _; - client_max_body_size 1000m; + client_max_body_size 50000m; ignore_invalid_headers off; proxy_buffering off; diff --git a/lerna.json b/lerna.json index 67ffc34bff..d6a5f41281 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.32", + "version": "2.13.35", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index e31bc81eed..1951c7986c 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -19,6 +19,7 @@ import { GoogleInnerConfig, OIDCInnerConfig, PlatformLogoutOpts, + SessionCookie, SSOProviderType, } from "@budibase/types" import * as events from "../events" @@ -44,7 +45,6 @@ export const buildAuthMiddleware = authenticated export const buildTenancyMiddleware = tenancy export const buildCsrfMiddleware = csrf export const passport = _passport -export const jwt = require("jsonwebtoken") // Strategies _passport.use(new LocalStrategy(local.options, local.authenticate)) @@ -191,10 +191,10 @@ export async function platformLogout(opts: PlatformLogoutOpts) { if (!ctx) throw new Error("Koa context must be supplied to logout.") - const currentSession = getCookie(ctx, Cookie.Auth) + const currentSession = getCookie(ctx, Cookie.Auth) let sessions = await getSessionsForUser(userId) - if (keepActiveSession) { + if (currentSession && keepActiveSession) { sessions = sessions.filter( session => session.sessionId !== currentSession.sessionId ) diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 8bd6591d05..16f658b90a 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { Ctx, EndpointMatcher } from "@budibase/types" +import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types" import { InvalidAPIKeyError, ErrorCode } from "../errors" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD @@ -98,7 +98,9 @@ export default function ( // check the actual user is authenticated first, try header or cookie let headerToken = ctx.request.headers[Header.TOKEN] - const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken) + const authCookie = + getCookie(ctx, Cookie.Auth) || + openJwt(headerToken) let apiKey = ctx.request.headers[Header.API_KEY] if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) { diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index ae6b3b4913..ab4ffee9d2 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -3,7 +3,7 @@ import { Cookie } from "../../../constants" import * as configs from "../../../configs" import * as cache from "../../../cache" import * as utils from "../../../utils" -import { UserCtx, SSOProfile } from "@budibase/types" +import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy @@ -58,7 +58,14 @@ export async function postAuth( const platformUrl = await configs.getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` - const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth) + const authStateCookie = utils.getCookie<{ appId: string }>( + ctx, + Cookie.DatasourceAuth + ) + + if (!authStateCookie) { + throw new Error("Unable to fetch datasource auth cookie") + } return passport.authenticate( new GoogleStrategy( diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 1971c09e9d..9b44eace49 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -305,20 +305,33 @@ export async function retrieveDirectory(bucketName: string, path: string) { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) - let fullObjects = await Promise.all( - objects.map(obj => retrieve(bucketName, obj.Key!)) + let streams = await Promise.all( + objects.map(obj => getReadStream(bucketName, obj.Key!)) ) let count = 0 + const writePromises: Promise[] = [] for (let obj of objects) { const filename = obj.Key! - const data = fullObjects[count++] + const stream = streams[count++] const possiblePath = filename.split("/") - if (possiblePath.length > 1) { - const dirs = possiblePath.slice(0, possiblePath.length - 1) - fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) + const dirs = possiblePath.slice(0, possiblePath.length - 1) + const possibleDir = join(writePath, ...dirs) + if (possiblePath.length > 1 && !fs.existsSync(possibleDir)) { + fs.mkdirSync(possibleDir, { recursive: true }) } - fs.writeFileSync(join(writePath, ...possiblePath), data) + const writeStream = fs.createWriteStream(join(writePath, ...possiblePath), { + mode: 0o644, + }) + stream.pipe(writeStream) + writePromises.push( + new Promise((resolve, reject) => { + stream.on("finish", resolve) + stream.on("error", reject) + writeStream.on("error", reject) + }) + ) } + await Promise.all(writePromises) return writePath } diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 7a8cfaf04a..45ed566a92 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -73,6 +73,9 @@ export async function encryptFile( const outputFileName = `${filename}.enc` const filePath = join(dir, filename) + if (fs.lstatSync(filePath).isDirectory()) { + throw new Error("Unable to encrypt directory") + } const inputFile = fs.createReadStream(filePath) const outputFile = fs.createWriteStream(join(dir, outputFileName)) @@ -110,6 +113,9 @@ export async function decryptFile( outputPath: string, secret: string ) { + if (fs.lstatSync(inputPath).isDirectory()) { + throw new Error("Unable to encrypt directory") + } const { salt, iv } = await getSaltAndIV(inputPath) const inputFile = fs.createReadStream(inputPath, { start: SALT_LENGTH + IV_LENGTH, diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b10d9ebdc0..ee1ef6da0c 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -11,8 +11,7 @@ import { TenantResolutionStrategy, } from "@budibase/types" import type { SetOption } from "cookies" - -const jwt = require("jsonwebtoken") +import jwt, { Secret } from "jsonwebtoken" const APP_PREFIX = DocumentType.APP + SEPARATOR const PROD_APP_PREFIX = "/app/" @@ -60,10 +59,7 @@ export function isServingApp(ctx: Ctx) { return true } // prod app - if (ctx.path.startsWith(PROD_APP_PREFIX)) { - return true - } - return false + return ctx.path.startsWith(PROD_APP_PREFIX) } export function isServingBuilder(ctx: Ctx): boolean { @@ -138,16 +134,16 @@ function parseAppIdFromUrl(url?: string) { * opens the contents of the specified encrypted JWT. * @return the contents of the token. */ -export function openJwt(token: string) { +export function openJwt(token?: string): T | undefined { if (!token) { - return token + return undefined } try { - return jwt.verify(token, env.JWT_SECRET) + return jwt.verify(token, env.JWT_SECRET as Secret) as T } catch (e) { if (env.JWT_SECRET_FALLBACK) { // fallback to enable rotation - return jwt.verify(token, env.JWT_SECRET_FALLBACK) + return jwt.verify(token, env.JWT_SECRET_FALLBACK) as T } else { throw e } @@ -159,13 +155,9 @@ export function isValidInternalAPIKey(apiKey: string) { return true } // fallback to enable rotation - if ( - env.INTERNAL_API_KEY_FALLBACK && - env.INTERNAL_API_KEY_FALLBACK === apiKey - ) { - return true - } - return false + return !!( + env.INTERNAL_API_KEY_FALLBACK && env.INTERNAL_API_KEY_FALLBACK === apiKey + ) } /** @@ -173,14 +165,14 @@ export function isValidInternalAPIKey(apiKey: string) { * @param ctx The request which is to be manipulated. * @param name The name of the cookie to get. */ -export function getCookie(ctx: Ctx, name: string) { +export function getCookie(ctx: Ctx, name: string) { const cookie = ctx.cookies.get(name) if (!cookie) { - return cookie + return undefined } - return openJwt(cookie) + return openJwt(cookie) } /** @@ -197,7 +189,7 @@ export function setCookie( opts = { sign: true } ) { if (value && opts && opts.sign) { - value = jwt.sign(value, env.JWT_SECRET) + value = jwt.sign(value, env.JWT_SECRET as Secret) } const config: SetOption = { diff --git a/packages/bbui/src/DetailSummary/DetailSummary.svelte b/packages/bbui/src/DetailSummary/DetailSummary.svelte index 808e12b6c4..4188422e20 100644 --- a/packages/bbui/src/DetailSummary/DetailSummary.svelte +++ b/packages/bbui/src/DetailSummary/DetailSummary.svelte @@ -1,21 +1,18 @@ diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index dc89476db2..fa0be630ba 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -53,7 +53,7 @@ $: { if (selectedImage?.url) { selectedUrl = selectedImage?.url - } else if (selectedImage) { + } else if (selectedImage && isImage) { try { let reader = new FileReader() reader.readAsDataURL(selectedImage) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 206becedd5..7a72b7a9c3 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -85,6 +85,7 @@ const INITIAL_FRONTEND_STATE = { selectedScreenId: null, selectedComponentId: null, selectedLayoutId: null, + hoverComponentId: null, // Client state selectedComponentInstance: null, @@ -112,7 +113,7 @@ export const getFrontendStore = () => { } let clone = cloneDeep(screen) const result = patchFn(clone) - + // An explicit false result means skip this change if (result === false) { return } @@ -879,11 +880,14 @@ export const getFrontendStore = () => { } // Mutates the fetched component with updates - const updated = patchFn(component, screen) + const patchResult = patchFn(component, screen) + // Mutates the component with any required settings updates const migrated = store.actions.components.migrateSettings(component) - return updated || migrated + // Returning an explicit false signifies that we should skip this + // update. If we migrated something, ensure we never skip. + return migrated ? null : patchResult } await store.actions.screens.patch(patchScreen, screenId) }, diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index 35846525af..02cef82e80 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -23,6 +23,7 @@ export let showTooltip = false export let selectedBy = null export let compact = false + export let hovering = false const scrollApi = getContext("scroll") const dispatch = createEventDispatcher() @@ -61,6 +62,7 @@
{item.label || item.field}
- + { + e.stopPropagation() + }} + on:change={onToggle(item)} + text="" + value={item.active} + thin + />
diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index e492b6be46..734e4448a1 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -13,7 +13,7 @@ export let app export let published let includeInternalTablesRows = true - let encypt = true + let encrypt = true let password = null const validation = createValidationStore() @@ -27,9 +27,9 @@ $: stepConfig = { [Step.CONFIG]: { title: published ? "Export published app" : "Export latest app", - confirmText: encypt ? "Continue" : exportButtonText, + confirmText: encrypt ? "Continue" : exportButtonText, onConfirm: () => { - if (!encypt) { + if (!encrypt) { exportApp() } else { currentStep = Step.SET_PASSWORD @@ -46,7 +46,7 @@ if (!$validation.valid) { return keepOpen } - exportApp(password) + await exportApp(password) }, isValid: $validation.valid, }, @@ -109,13 +109,13 @@ text="Export rows from internal tables" bind:value={includeInternalTablesRows} /> - + - {#if !encypt} - - {/if} + {/if} {#if currentStep === Step.SET_PASSWORD} !setting.section && setting.tag === tag ) + const customSections = settings.filter( setting => setting.section && setting.tag === tag ) @@ -173,6 +174,7 @@ name={showSectionTitle ? section.name : ""} show={section.collapsed !== true} {noPadding} + initiallyShow={section.collapsed !== true} > {#if section.info}
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 45fe005ceb..65ea172012 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -36,12 +36,14 @@ // Determine selected component ID $: selectedComponentId = $store.selectedComponentId + $: hoverComponentId = $store.hoverComponentId $: previewData = { appId: $store.appId, layout, screen, selectedComponentId, + hoverComponentId, theme: $store.theme, customTheme: $store.customTheme, previewDevice: $store.previewDevice, @@ -117,6 +119,8 @@ error = event.error || "An unknown error occurred" } else if (type === "select-component" && data.id) { $store.selectedComponentId = data.id + } else if (type === "hover-component" && data.id) { + $store.hoverComponentId = data.id } else if (type === "update-prop") { await store.actions.components.updateSetting(data.prop, data.value) } else if (type === "update-styles") { diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte index 1ad522cbd4..8d4d64e4be 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte @@ -89,6 +89,17 @@ } return findComponentPath($selectedComponent, component._id)?.length > 0 } + + const handleMouseover = componentId => { + if ($store.hoverComponentId !== componentId) { + $store.hoverComponentId = componentId + } + } + const handleMouseout = componentId => { + if ($store.hoverComponentId === componentId) { + $store.hoverComponentId = null + } + }
    @@ -109,6 +120,9 @@ on:dragover={dragover(component, index)} on:iconClick={() => toggleNodeOpen(component._id)} on:drop={onDrop} + hovering={$store.hoverComponentId === component._id} + on:mouseenter={() => handleMouseover(component._id)} + on:mouseleave={() => handleMouseout(component._id)} text={getComponentText(component)} icon={getComponentIcon(component)} iconTooltip={getComponentName(component)} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte index 67b501e141..1e2ea47e63 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte @@ -32,6 +32,17 @@ const handleScroll = e => { scrolling = e.target.scrollTop !== 0 } + + const handleMouseover = componentId => { + if ($store.hoverComponentId !== componentId) { + $store.hoverComponentId = componentId + } + } + const handleMouseout = componentId => { + if ($store.hoverComponentId === componentId) { + $store.hoverComponentId = null + } + }
    @@ -57,6 +68,12 @@ on:click={() => { $store.selectedComponentId = `${$store.selectedScreenId}-screen` }} + hovering={$store.hoverComponentId === + `${$store.selectedScreenId}-screen`} + on:mouseenter={() => + handleMouseover(`${$store.selectedScreenId}-screen`)} + on:mouseleave={() => + handleMouseout(`${$store.selectedScreenId}-screen`)} id={`component-screen`} selectedBy={$userSelectedResourceMap[ `${$store.selectedScreenId}-screen` @@ -78,6 +95,12 @@ on:click={() => { $store.selectedComponentId = `${$store.selectedScreenId}-navigation` }} + hovering={$store.hoverComponentId === + `${$store.selectedScreenId}-navigation`} + on:mouseenter={() => + handleMouseover(`${$store.selectedScreenId}-navigation`)} + on:mouseleave={() => + handleMouseout(`${$store.selectedScreenId}-navigation`)} id={`component-nav`} selectedBy={$userSelectedResourceMap[ `${$store.selectedScreenId}-navigation` diff --git a/packages/cli/package.json b/packages/cli/package.json index 49384538ef..fc3044fb14 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -15,7 +15,8 @@ "pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip", "build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild", "check:types": "tsc -p tsconfig.json --noEmit --paths null", - "postbuild": "rm -rf prebuilds 2> /dev/null" + "postbuild": "rm -rf prebuilds 2> /dev/null", + "start": "ts-node ./src/index.ts" }, "pkg": { "targets": [ diff --git a/packages/client/src/components/preview/HoverIndicator.svelte b/packages/client/src/components/preview/HoverIndicator.svelte index f976e2cffb..ddef62ab74 100644 --- a/packages/client/src/components/preview/HoverIndicator.svelte +++ b/packages/client/src/components/preview/HoverIndicator.svelte @@ -3,8 +3,7 @@ import IndicatorSet from "./IndicatorSet.svelte" import { builderStore, dndIsDragging } from "stores" - let componentId - + $: componentId = $builderStore.hoverComponentId $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920 const onMouseOver = e => { @@ -24,12 +23,12 @@ } if (newId !== componentId) { - componentId = newId + builderStore.actions.hoverComponent(newId) } } const onMouseLeave = () => { - componentId = null + builderStore.actions.hoverComponent(null) } onMount(() => { diff --git a/packages/client/src/index.js b/packages/client/src/index.js index 730425abfd..36c3f24fc1 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -32,6 +32,7 @@ const loadBudibase = async () => { layout: window["##BUDIBASE_PREVIEW_LAYOUT##"], screen: window["##BUDIBASE_PREVIEW_SCREEN##"], selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"], + hoverComponentId: window["##BUDIBASE_HOVER_COMPONENT_ID##"], previewId: window["##BUDIBASE_PREVIEW_ID##"], theme: window["##BUDIBASE_PREVIEW_THEME##"], customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"], diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 036558e8b2..f92e6b9df9 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -8,6 +8,7 @@ const createBuilderStore = () => { inBuilder: false, screen: null, selectedComponentId: null, + hoverComponentId: null, editMode: false, previewId: null, theme: null, @@ -23,6 +24,16 @@ const createBuilderStore = () => { } const store = writable(initialState) const actions = { + hoverComponent: id => { + if (id === get(store).hoverComponentId) { + return + } + store.update(state => ({ + ...state, + hoverComponentId: id, + })) + eventStore.actions.dispatchEvent("hover-component", { id }) + }, selectComponent: id => { if (id === get(store).selectedComponentId) { return diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 472c0d6272..4d307e9593 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro" import { events, context, utils, constants } from "@budibase/backend-core" import sdk from "../../../sdk" import { QueryEvent } from "../../../threads/definitions" -import { ConfigType, Query, UserCtx } from "@budibase/types" +import { ConfigType, Query, UserCtx, SessionCookie } from "@budibase/types" import { ValidQueryNameRegex } from "@budibase/shared-core" const Runner = new Thread(ThreadType.QUERY, { @@ -113,7 +113,7 @@ function getOAuthConfigCookieId(ctx: UserCtx) { } function getAuthConfig(ctx: UserCtx) { - const authCookie = utils.getCookie(ctx, constants.Cookie.Auth) + const authCookie = utils.getCookie(ctx, constants.Cookie.Auth) let authConfigCtx: any = {} authConfigCtx["configId"] = getOAuthConfigCookieId(ctx) authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null diff --git a/packages/server/src/api/controllers/static/templates/preview.hbs b/packages/server/src/api/controllers/static/templates/preview.hbs index 31bf0762e0..e5b97afd66 100644 --- a/packages/server/src/api/controllers/static/templates/preview.hbs +++ b/packages/server/src/api/controllers/static/templates/preview.hbs @@ -63,6 +63,7 @@ // Extract data from message const { selectedComponentId, + hoverComponentId, layout, screen, appId, @@ -81,6 +82,7 @@ window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout window["##BUDIBASE_PREVIEW_SCREEN##"] = screen window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId + window["##BUDIBASE_HOVER_COMPONENT_ID##"] = hoverComponentId window["##BUDIBASE_PREVIEW_ID##"] = Math.random() window["##BUDIBASE_PREVIEW_THEME##"] = theme window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme @@ -108,4 +110,4 @@ - \ No newline at end of file + diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index e7eea5f0b6..931ddfd443 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -59,6 +59,7 @@ const environment = { BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins", OPENAI_API_KEY: process.env.OPENAI_API_KEY, + MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, diff --git a/packages/server/src/koa.ts b/packages/server/src/koa.ts index b331d87120..9f90c04b50 100644 --- a/packages/server/src/koa.ts +++ b/packages/server/src/koa.ts @@ -1,5 +1,5 @@ import env from "./environment" -import Koa, { ExtendableContext } from "koa" +import Koa from "koa" import koaBody from "koa-body" import http from "http" import * as api from "./api" @@ -27,6 +27,9 @@ export default function createKoaApp() { // @ts-ignore enableTypes: ["json", "form", "text"], parsedMethods: ["POST", "PUT", "PATCH", "DELETE"], + formidable: { + maxFileSize: parseInt(env.MAX_IMPORT_SIZE_MB || "100") * 1024 * 1024, + }, }) ) diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts index 0584fcb3d0..ddb1954f8f 100644 --- a/packages/server/src/sdk/app/backups/constants.ts +++ b/packages/server/src/sdk/app/backups/constants.ts @@ -1,3 +1,4 @@ export const DB_EXPORT_FILE = "db.txt" export const GLOBAL_DB_EXPORT_FILE = "global.txt" export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"] +export const ATTACHMENT_DIRECTORY = "attachments" diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 65708e6ba2..813f813177 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -8,13 +8,15 @@ import { TABLE_ROW_PREFIX, USER_METDATA_PREFIX, } from "../../../db/utils" -import { DB_EXPORT_FILE, STATIC_APP_FILES } from "./constants" +import { + DB_EXPORT_FILE, + STATIC_APP_FILES, + ATTACHMENT_DIRECTORY, +} from "./constants" import fs from "fs" import { join } from "path" import env from "../../../environment" - -const uuid = require("uuid/v4") - +import { v4 as uuid } from "uuid" import tar from "tar" const MemoryStream = require("memorystream") @@ -30,12 +32,11 @@ export interface ExportOpts extends DBDumpOpts { encryptPassword?: string } -function tarFilesToTmp(tmpDir: string, files: string[]) { +async function tarFilesToTmp(tmpDir: string, files: string[]) { const fileName = `${uuid()}.tar.gz` const exportFile = join(budibaseTempDir(), fileName) - tar.create( + await tar.create( { - sync: true, gzip: true, file: exportFile, noDirRecurse: false, @@ -150,19 +151,21 @@ export async function exportApp(appId: string, config?: ExportOpts) { for (let file of fs.readdirSync(tmpPath)) { const path = join(tmpPath, file) - await encryption.encryptFile( - { dir: tmpPath, filename: file }, - config.encryptPassword - ) - - fs.rmSync(path) + // skip the attachments - too big to encrypt + if (file !== ATTACHMENT_DIRECTORY) { + await encryption.encryptFile( + { dir: tmpPath, filename: file }, + config.encryptPassword + ) + fs.rmSync(path) + } } } // if tar requested, return where the tarball is if (config?.tar) { // now the tmpPath contains both the DB export and attachments, tar this - const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) + const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) // cleanup the tmp export files as tarball returned fs.rmSync(tmpPath, { recursive: true, force: true }) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 1e229d283a..7f76945107 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -6,17 +6,20 @@ import { AutomationTriggerStepId, RowAttachment, } from "@budibase/types" -import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" +import { getAutomationParams } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" -import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" +import { + DB_EXPORT_FILE, + GLOBAL_DB_EXPORT_FILE, + ATTACHMENT_DIRECTORY, +} from "./constants" import { downloadTemplate } from "../../../utilities/fileSystem" import { ObjectStoreBuckets } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" - -const uuid = require("uuid/v4") -const tar = require("tar") +import { v4 as uuid } from "uuid" +import tar from "tar" type TemplateType = { file?: { @@ -114,12 +117,11 @@ async function getTemplateStream(template: TemplateType) { } } -export function untarFile(file: { path: string }) { +export async function untarFile(file: { path: string }) { const tmpPath = join(budibaseTempDir(), uuid()) fs.mkdirSync(tmpPath) // extract the tarball - tar.extract({ - sync: true, + await tar.extract({ cwd: tmpPath, file: file.path, }) @@ -130,9 +132,11 @@ async function decryptFiles(path: string, password: string) { try { for (let file of fs.readdirSync(path)) { const inputPath = join(path, file) - const outputPath = inputPath.replace(/\.enc$/, "") - await encryption.decryptFile(inputPath, outputPath, password) - fs.rmSync(inputPath) + if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) { + const outputPath = inputPath.replace(/\.enc$/, "") + await encryption.decryptFile(inputPath, outputPath, password) + fs.rmSync(inputPath) + } } } catch (err: any) { if (err.message === "incorrect header check") { @@ -162,7 +166,7 @@ export async function importApp( const isDirectory = template.file && fs.lstatSync(template.file.path).isDirectory() if (template.file && (isTar || isDirectory)) { - const tmpPath = isTar ? untarFile(template.file) : template.file.path + const tmpPath = isTar ? await untarFile(template.file) : template.file.path if (isTar && template.file.password) { await decryptFiles(tmpPath, template.file.password) } diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index b7886ccea4..afaad64723 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -56,6 +56,7 @@ import { import API from "./api" import { cloneDeep } from "lodash" +import jwt, { Secret } from "jsonwebtoken" mocks.licenses.init(pro) @@ -391,7 +392,7 @@ class TestConfiguration { sessionId: "sessionid", tenantId: this.getTenantId(), } - const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET) + const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret) // returning necessary request headers await cache.user.invalidateUser(userId) @@ -412,7 +413,7 @@ class TestConfiguration { sessionId: "sessionid", tenantId, } - const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET) + const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret) const headers: any = { Accept: "application/json", diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 293e74312e..a6817ddf19 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -249,7 +249,9 @@ export async function outputProcessing( continue } row[property].forEach((attachment: RowAttachment) => { - attachment.url ??= objectStore.getAppFileUrl(attachment.key) + if (!attachment.url) { + attachment.url = objectStore.getAppFileUrl(attachment.key) + } }) } } else if ( diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index 03584ef53b..95ce340910 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -3,6 +3,7 @@ import { FieldType, FieldTypeSubtypes, INTERNAL_TABLE_SOURCE_ID, + RowAttachment, Table, TableSourceType, } from "@budibase/types" @@ -70,6 +71,49 @@ describe("rowProcessor - outputProcessing", () => { ) }) + it("should handle attachments correctly", async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + attach: { + type: FieldType.ATTACHMENT, + name: "attach", + constraints: {}, + }, + }, + } + + const row: { attach: RowAttachment[] } = { + attach: [ + { + size: 10, + name: "test", + extension: "jpg", + key: "test.jpg", + }, + ], + } + + const output = await outputProcessing(table, row, { squash: false }) + expect(output.attach[0].url).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) + + row.attach[0].url = "" + const output2 = await outputProcessing(table, row, { squash: false }) + expect(output2.attach[0].url).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) + + row.attach[0].url = "aaaa" + const output3 = await outputProcessing(table, row, { squash: false }) + expect(output3.attach[0].url).toBe("aaaa") + }) + it("process output even when the field is not empty", async () => { const table: Table = { _id: generator.guid(), diff --git a/packages/types/src/api/web/cookies.ts b/packages/types/src/api/web/cookies.ts new file mode 100644 index 0000000000..27954a36a1 --- /dev/null +++ b/packages/types/src/api/web/cookies.ts @@ -0,0 +1,9 @@ +export interface DatasourceAuthCookie { + appId: string + provider: string +} + +export interface SessionCookie { + sessionId: string + userId: string +} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index cba1e04f9a..75c246ab9b 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -9,3 +9,4 @@ export * from "./app" export * from "./global" export * from "./pagination" export * from "./searchFilter" +export * from "./cookies" diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 279162fb08..a94ed082f7 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -15,6 +15,7 @@ import { PasswordResetRequest, PasswordResetUpdateRequest, GoogleInnerConfig, + DatasourceAuthCookie, } from "@budibase/types" import env from "../../../environment" @@ -148,7 +149,13 @@ export const datasourcePreAuth = async (ctx: any, next: any) => { } export const datasourceAuth = async (ctx: any, next: any) => { - const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) + const authStateCookie = getCookie( + ctx, + Cookie.DatasourceAuth + ) + if (!authStateCookie) { + throw new Error("Unable to retrieve datasource authentication cookie") + } const provider = authStateCookie.provider const { middleware } = require(`@budibase/backend-core`) const handler = middleware.datasource[provider] diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 289f31079a..c43d1b9d13 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -35,6 +35,7 @@ import { ConfigType, } from "@budibase/types" import API from "./api" +import jwt, { Secret } from "jsonwebtoken" class TestConfiguration { server: any @@ -209,7 +210,7 @@ class TestConfiguration { sessionId: "sessionid", tenantId: user.tenantId, } - const authCookie = auth.jwt.sign(authToken, coreEnv.JWT_SECRET) + const authCookie = jwt.sign(authToken, coreEnv.JWT_SECRET as Secret) return { Accept: "application/json", ...this.cookieHeader([`${constants.Cookie.Auth}=${authCookie}`]), @@ -327,7 +328,7 @@ class TestConfiguration { // CONFIGS - OIDC getOIDConfigCookie(configId: string) { - const token = auth.jwt.sign(configId, coreEnv.JWT_SECRET) + const token = jwt.sign(configId, coreEnv.JWT_SECRET as Secret) return this.cookieHeader([[`${constants.Cookie.OIDC_CONFIG}=${token}`]]) }