From 00a3c630ef4438392cf035abea47b11a94e24e6d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Dec 2023 18:36:40 +0000 Subject: [PATCH 01/17] Initial work to get file streaming working when dealing with a large amount of file exports, tested with up to 1.5GB of attachments. --- .../src/objectStore/objectStore.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 1971c09e9d..9b44eace49 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -305,20 +305,33 @@ export async function retrieveDirectory(bucketName: string, path: string) { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) - let fullObjects = await Promise.all( - objects.map(obj => retrieve(bucketName, obj.Key!)) + let streams = await Promise.all( + objects.map(obj => getReadStream(bucketName, obj.Key!)) ) let count = 0 + const writePromises: Promise[] = [] for (let obj of objects) { const filename = obj.Key! - const data = fullObjects[count++] + const stream = streams[count++] const possiblePath = filename.split("/") - if (possiblePath.length > 1) { - const dirs = possiblePath.slice(0, possiblePath.length - 1) - fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) + const dirs = possiblePath.slice(0, possiblePath.length - 1) + const possibleDir = join(writePath, ...dirs) + if (possiblePath.length > 1 && !fs.existsSync(possibleDir)) { + fs.mkdirSync(possibleDir, { recursive: true }) } - fs.writeFileSync(join(writePath, ...possiblePath), data) + const writeStream = fs.createWriteStream(join(writePath, ...possiblePath), { + mode: 0o644, + }) + stream.pipe(writeStream) + writePromises.push( + new Promise((resolve, reject) => { + stream.on("finish", resolve) + stream.on("error", reject) + writeStream.on("error", reject) + }) + ) } + await Promise.all(writePromises) return writePath } From 19c069946a3dc9759540db2f57c3e9f9dbf0e830 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Dec 2023 19:37:43 +0000 Subject: [PATCH 02/17] Fixing importing - still some work to get links correct. --- packages/server/src/environment.ts | 1 + packages/server/src/koa.ts | 5 ++++- .../server/src/sdk/app/backups/exports.ts | 21 ++++++++++--------- .../server/src/sdk/app/backups/imports.ts | 17 ++++++++------- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index e7eea5f0b6..931ddfd443 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -59,6 +59,7 @@ const environment = { BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins", OPENAI_API_KEY: process.env.OPENAI_API_KEY, + MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, diff --git a/packages/server/src/koa.ts b/packages/server/src/koa.ts index b331d87120..9f90c04b50 100644 --- a/packages/server/src/koa.ts +++ b/packages/server/src/koa.ts @@ -1,5 +1,5 @@ import env from "./environment" -import Koa, { ExtendableContext } from "koa" +import Koa from "koa" import koaBody from "koa-body" import http from "http" import * as api from "./api" @@ -27,6 +27,9 @@ export default function createKoaApp() { // @ts-ignore enableTypes: ["json", "form", "text"], parsedMethods: ["POST", "PUT", "PATCH", "DELETE"], + formidable: { + maxFileSize: parseInt(env.MAX_IMPORT_SIZE_MB || "100") * 1024 * 1024, + }, }) ) diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 65708e6ba2..cc0b78e34f 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -30,12 +30,11 @@ export interface ExportOpts extends DBDumpOpts { encryptPassword?: string } -function tarFilesToTmp(tmpDir: string, files: string[]) { +async function tarFilesToTmp(tmpDir: string, files: string[]) { const fileName = `${uuid()}.tar.gz` const exportFile = join(budibaseTempDir(), fileName) - tar.create( + await tar.create( { - sync: true, gzip: true, file: exportFile, noDirRecurse: false, @@ -150,19 +149,21 @@ export async function exportApp(appId: string, config?: ExportOpts) { for (let file of fs.readdirSync(tmpPath)) { const path = join(tmpPath, file) - await encryption.encryptFile( - { dir: tmpPath, filename: file }, - config.encryptPassword - ) - - fs.rmSync(path) + // skip the attachments - too big to encrypt + if (file !== "attachments") { + await encryption.encryptFile( + { dir: tmpPath, filename: file }, + config.encryptPassword + ) + fs.rmSync(path) + } } } // if tar requested, return where the tarball is if (config?.tar) { // now the tmpPath contains both the DB export and attachments, tar this - const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) + const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) // cleanup the tmp export files as tarball returned fs.rmSync(tmpPath, { recursive: true, force: true }) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 1e229d283a..4c0030abf9 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -6,7 +6,7 @@ import { AutomationTriggerStepId, RowAttachment, } from "@budibase/types" -import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" +import { getAutomationParams } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" import { downloadTemplate } from "../../../utilities/fileSystem" @@ -114,12 +114,11 @@ async function getTemplateStream(template: TemplateType) { } } -export function untarFile(file: { path: string }) { +export async function untarFile(file: { path: string }) { const tmpPath = join(budibaseTempDir(), uuid()) fs.mkdirSync(tmpPath) // extract the tarball - tar.extract({ - sync: true, + await tar.extract({ cwd: tmpPath, file: file.path, }) @@ -130,9 +129,11 @@ async function decryptFiles(path: string, password: string) { try { for (let file of fs.readdirSync(path)) { const inputPath = join(path, file) - const outputPath = inputPath.replace(/\.enc$/, "") - await encryption.decryptFile(inputPath, outputPath, password) - fs.rmSync(inputPath) + if (!inputPath.endsWith("attachments")) { + const outputPath = inputPath.replace(/\.enc$/, "") + await encryption.decryptFile(inputPath, outputPath, password) + fs.rmSync(inputPath) + } } } catch (err: any) { if (err.message === "incorrect header check") { @@ -162,7 +163,7 @@ export async function importApp( const isDirectory = template.file && fs.lstatSync(template.file.path).isDirectory() if (template.file && (isTar || isDirectory)) { - const tmpPath = isTar ? untarFile(template.file) : template.file.path + const tmpPath = isTar ? await untarFile(template.file) : template.file.path if (isTar && template.file.password) { await decryptFiles(tmpPath, template.file.password) } From 50270b885428b74beba566e3c0e09f660f83500c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Dec 2023 15:54:33 +0000 Subject: [PATCH 03/17] Quick refactor while in the area. --- packages/backend-core/src/utils/utils.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b10d9ebdc0..ef6f07a16b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -11,8 +11,7 @@ import { TenantResolutionStrategy, } from "@budibase/types" import type { SetOption } from "cookies" - -const jwt = require("jsonwebtoken") +import jwt, { Secret } from "jsonwebtoken" const APP_PREFIX = DocumentType.APP + SEPARATOR const PROD_APP_PREFIX = "/app/" @@ -60,10 +59,7 @@ export function isServingApp(ctx: Ctx) { return true } // prod app - if (ctx.path.startsWith(PROD_APP_PREFIX)) { - return true - } - return false + return ctx.path.startsWith(PROD_APP_PREFIX) } export function isServingBuilder(ctx: Ctx): boolean { @@ -143,7 +139,7 @@ export function openJwt(token: string) { return token } try { - return jwt.verify(token, env.JWT_SECRET) + return jwt.verify(token, env.JWT_SECRET as Secret) } catch (e) { if (env.JWT_SECRET_FALLBACK) { // fallback to enable rotation @@ -197,7 +193,7 @@ export function setCookie( opts = { sign: true } ) { if (value && opts && opts.sign) { - value = jwt.sign(value, env.JWT_SECRET) + value = jwt.sign(value, env.JWT_SECRET as Secret) } const config: SetOption = { From 2bf65601a11febf89942104848c96e9639fffd00 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Dec 2023 15:55:09 +0000 Subject: [PATCH 04/17] Another simplification. --- packages/backend-core/src/utils/utils.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index ef6f07a16b..7d3f682fe4 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -155,13 +155,9 @@ export function isValidInternalAPIKey(apiKey: string) { return true } // fallback to enable rotation - if ( - env.INTERNAL_API_KEY_FALLBACK && - env.INTERNAL_API_KEY_FALLBACK === apiKey - ) { - return true - } - return false + return !!( + env.INTERNAL_API_KEY_FALLBACK && env.INTERNAL_API_KEY_FALLBACK === apiKey + ) } /** From fb6f79618ea2b5579491ac1c81bd11c9103921ae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Dec 2023 16:10:01 +0000 Subject: [PATCH 05/17] Getting export modal to wait. --- packages/builder/src/components/start/ExportAppModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index e492b6be46..c05306c679 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -46,7 +46,7 @@ if (!$validation.valid) { return keepOpen } - exportApp(password) + await exportApp(password) }, isValid: $validation.valid, }, From 33b7e4d5d2b3f4ef800648706ad8892b7643662a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Dec 2023 16:28:19 +0000 Subject: [PATCH 06/17] Some final refactoring, using proper jwt lib rather than requiring (proper typing). --- packages/backend-core/src/auth/auth.ts | 6 +++--- .../backend-core/src/middleware/authenticated.ts | 6 ++++-- .../src/middleware/passport/datasource/google.ts | 9 ++++++++- packages/backend-core/src/utils/utils.ts | 14 +++++++------- .../src/tests/utilities/TestConfiguration.ts | 5 +++-- packages/types/src/api/web/global/index.ts | 1 + packages/types/src/api/web/global/sessions.ts | 4 ++++ packages/worker/src/tests/TestConfiguration.ts | 5 +++-- 8 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 packages/types/src/api/web/global/sessions.ts diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index e31bc81eed..8de8904f3a 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -19,6 +19,7 @@ import { GoogleInnerConfig, OIDCInnerConfig, PlatformLogoutOpts, + SessionInfo, SSOProviderType, } from "@budibase/types" import * as events from "../events" @@ -44,7 +45,6 @@ export const buildAuthMiddleware = authenticated export const buildTenancyMiddleware = tenancy export const buildCsrfMiddleware = csrf export const passport = _passport -export const jwt = require("jsonwebtoken") // Strategies _passport.use(new LocalStrategy(local.options, local.authenticate)) @@ -191,10 +191,10 @@ export async function platformLogout(opts: PlatformLogoutOpts) { if (!ctx) throw new Error("Koa context must be supplied to logout.") - const currentSession = getCookie(ctx, Cookie.Auth) + const currentSession = getCookie(ctx, Cookie.Auth) let sessions = await getSessionsForUser(userId) - if (keepActiveSession) { + if (currentSession && keepActiveSession) { sessions = sessions.filter( session => session.sessionId !== currentSession.sessionId ) diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 8bd6591d05..3e82498bdd 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { Ctx, EndpointMatcher } from "@budibase/types" +import { Ctx, EndpointMatcher, SessionInfo } from "@budibase/types" import { InvalidAPIKeyError, ErrorCode } from "../errors" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD @@ -98,7 +98,9 @@ export default function ( // check the actual user is authenticated first, try header or cookie let headerToken = ctx.request.headers[Header.TOKEN] - const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken) + const authCookie = + getCookie(ctx, Cookie.Auth) || + openJwt(headerToken) let apiKey = ctx.request.headers[Header.API_KEY] if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) { diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index ae6b3b4913..7f768f1623 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -58,7 +58,14 @@ export async function postAuth( const platformUrl = await configs.getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` - const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth) + const authStateCookie = utils.getCookie<{ appId: string }>( + ctx, + Cookie.DatasourceAuth + ) + + if (!authStateCookie) { + throw new Error("Unable to fetch datasource auth cookie") + } return passport.authenticate( new GoogleStrategy( diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 7d3f682fe4..429978fc97 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -134,16 +134,16 @@ function parseAppIdFromUrl(url?: string) { * opens the contents of the specified encrypted JWT. * @return the contents of the token. */ -export function openJwt(token: string) { +export function openJwt(token?: string): T | undefined { if (!token) { - return token + return undefined } try { - return jwt.verify(token, env.JWT_SECRET as Secret) + return jwt.verify(token, env.JWT_SECRET as Secret) as T } catch (e) { if (env.JWT_SECRET_FALLBACK) { // fallback to enable rotation - return jwt.verify(token, env.JWT_SECRET_FALLBACK) + return jwt.verify(token, env.JWT_SECRET_FALLBACK) as T } else { throw e } @@ -165,14 +165,14 @@ export function isValidInternalAPIKey(apiKey: string) { * @param ctx The request which is to be manipulated. * @param name The name of the cookie to get. */ -export function getCookie(ctx: Ctx, name: string) { +export function getCookie(ctx: Ctx, name: string) { const cookie = ctx.cookies.get(name) if (!cookie) { - return cookie + return undefined } - return openJwt(cookie) + return openJwt(cookie) as T } /** diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index b7886ccea4..afaad64723 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -56,6 +56,7 @@ import { import API from "./api" import { cloneDeep } from "lodash" +import jwt, { Secret } from "jsonwebtoken" mocks.licenses.init(pro) @@ -391,7 +392,7 @@ class TestConfiguration { sessionId: "sessionid", tenantId: this.getTenantId(), } - const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET) + const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret) // returning necessary request headers await cache.user.invalidateUser(userId) @@ -412,7 +413,7 @@ class TestConfiguration { sessionId: "sessionid", tenantId, } - const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET) + const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret) const headers: any = { Accept: "application/json", diff --git a/packages/types/src/api/web/global/index.ts b/packages/types/src/api/web/global/index.ts index efcb6dc39c..e6e2a78feb 100644 --- a/packages/types/src/api/web/global/index.ts +++ b/packages/types/src/api/web/global/index.ts @@ -4,3 +4,4 @@ export * from "./events" export * from "./configs" export * from "./scim" export * from "./license" +export * from "./sessions" diff --git a/packages/types/src/api/web/global/sessions.ts b/packages/types/src/api/web/global/sessions.ts new file mode 100644 index 0000000000..a6b94a3d24 --- /dev/null +++ b/packages/types/src/api/web/global/sessions.ts @@ -0,0 +1,4 @@ +export interface SessionInfo { + sessionId: string + userId: string +} diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 289f31079a..c43d1b9d13 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -35,6 +35,7 @@ import { ConfigType, } from "@budibase/types" import API from "./api" +import jwt, { Secret } from "jsonwebtoken" class TestConfiguration { server: any @@ -209,7 +210,7 @@ class TestConfiguration { sessionId: "sessionid", tenantId: user.tenantId, } - const authCookie = auth.jwt.sign(authToken, coreEnv.JWT_SECRET) + const authCookie = jwt.sign(authToken, coreEnv.JWT_SECRET as Secret) return { Accept: "application/json", ...this.cookieHeader([`${constants.Cookie.Auth}=${authCookie}`]), @@ -327,7 +328,7 @@ class TestConfiguration { // CONFIGS - OIDC getOIDConfigCookie(configId: string) { - const token = auth.jwt.sign(configId, coreEnv.JWT_SECRET) + const token = jwt.sign(configId, coreEnv.JWT_SECRET as Secret) return this.cookieHeader([[`${constants.Cookie.OIDC_CONFIG}=${token}`]]) } From 4cd870ef9e9e1ee3a37fcdf016433cf97cf211e7 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 5 Dec 2023 16:54:03 +0000 Subject: [PATCH 07/17] Don't read attachments into memory unless they are images --- packages/bbui/src/Form/Core/Dropzone.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index dc89476db2..fa0be630ba 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -53,7 +53,7 @@ $: { if (selectedImage?.url) { selectedUrl = selectedImage?.url - } else if (selectedImage) { + } else if (selectedImage && isImage) { try { let reader = new FileReader() reader.readAsDataURL(selectedImage) From 8628c67c90843042314f36fae020ab652951ec7d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Dec 2023 18:17:27 +0000 Subject: [PATCH 08/17] Fixing typing issues. --- packages/backend-core/src/auth/auth.ts | 4 ++-- packages/backend-core/src/middleware/authenticated.ts | 6 +++--- .../src/middleware/passport/datasource/google.ts | 2 +- packages/server/src/api/controllers/query/index.ts | 4 ++-- packages/types/src/api/web/cookies.ts | 9 +++++++++ packages/types/src/api/web/global/index.ts | 1 - packages/types/src/api/web/global/sessions.ts | 4 ---- packages/types/src/api/web/index.ts | 1 + packages/worker/src/api/controllers/global/auth.ts | 9 ++++++++- 9 files changed, 26 insertions(+), 14 deletions(-) create mode 100644 packages/types/src/api/web/cookies.ts delete mode 100644 packages/types/src/api/web/global/sessions.ts diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index 8de8904f3a..1951c7986c 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -19,7 +19,7 @@ import { GoogleInnerConfig, OIDCInnerConfig, PlatformLogoutOpts, - SessionInfo, + SessionCookie, SSOProviderType, } from "@budibase/types" import * as events from "../events" @@ -191,7 +191,7 @@ export async function platformLogout(opts: PlatformLogoutOpts) { if (!ctx) throw new Error("Koa context must be supplied to logout.") - const currentSession = getCookie(ctx, Cookie.Auth) + const currentSession = getCookie(ctx, Cookie.Auth) let sessions = await getSessionsForUser(userId) if (currentSession && keepActiveSession) { diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 3e82498bdd..16f658b90a 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { Ctx, EndpointMatcher, SessionInfo } from "@budibase/types" +import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types" import { InvalidAPIKeyError, ErrorCode } from "../errors" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD @@ -99,8 +99,8 @@ export default function ( let headerToken = ctx.request.headers[Header.TOKEN] const authCookie = - getCookie(ctx, Cookie.Auth) || - openJwt(headerToken) + getCookie(ctx, Cookie.Auth) || + openJwt(headerToken) let apiKey = ctx.request.headers[Header.API_KEY] if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) { diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 7f768f1623..ab4ffee9d2 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -3,7 +3,7 @@ import { Cookie } from "../../../constants" import * as configs from "../../../configs" import * as cache from "../../../cache" import * as utils from "../../../utils" -import { UserCtx, SSOProfile } from "@budibase/types" +import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 472c0d6272..4d307e9593 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro" import { events, context, utils, constants } from "@budibase/backend-core" import sdk from "../../../sdk" import { QueryEvent } from "../../../threads/definitions" -import { ConfigType, Query, UserCtx } from "@budibase/types" +import { ConfigType, Query, UserCtx, SessionCookie } from "@budibase/types" import { ValidQueryNameRegex } from "@budibase/shared-core" const Runner = new Thread(ThreadType.QUERY, { @@ -113,7 +113,7 @@ function getOAuthConfigCookieId(ctx: UserCtx) { } function getAuthConfig(ctx: UserCtx) { - const authCookie = utils.getCookie(ctx, constants.Cookie.Auth) + const authCookie = utils.getCookie(ctx, constants.Cookie.Auth) let authConfigCtx: any = {} authConfigCtx["configId"] = getOAuthConfigCookieId(ctx) authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null diff --git a/packages/types/src/api/web/cookies.ts b/packages/types/src/api/web/cookies.ts new file mode 100644 index 0000000000..27954a36a1 --- /dev/null +++ b/packages/types/src/api/web/cookies.ts @@ -0,0 +1,9 @@ +export interface DatasourceAuthCookie { + appId: string + provider: string +} + +export interface SessionCookie { + sessionId: string + userId: string +} diff --git a/packages/types/src/api/web/global/index.ts b/packages/types/src/api/web/global/index.ts index e6e2a78feb..efcb6dc39c 100644 --- a/packages/types/src/api/web/global/index.ts +++ b/packages/types/src/api/web/global/index.ts @@ -4,4 +4,3 @@ export * from "./events" export * from "./configs" export * from "./scim" export * from "./license" -export * from "./sessions" diff --git a/packages/types/src/api/web/global/sessions.ts b/packages/types/src/api/web/global/sessions.ts deleted file mode 100644 index a6b94a3d24..0000000000 --- a/packages/types/src/api/web/global/sessions.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface SessionInfo { - sessionId: string - userId: string -} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index cba1e04f9a..75c246ab9b 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -9,3 +9,4 @@ export * from "./app" export * from "./global" export * from "./pagination" export * from "./searchFilter" +export * from "./cookies" diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 279162fb08..a94ed082f7 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -15,6 +15,7 @@ import { PasswordResetRequest, PasswordResetUpdateRequest, GoogleInnerConfig, + DatasourceAuthCookie, } from "@budibase/types" import env from "../../../environment" @@ -148,7 +149,13 @@ export const datasourcePreAuth = async (ctx: any, next: any) => { } export const datasourceAuth = async (ctx: any, next: any) => { - const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) + const authStateCookie = getCookie( + ctx, + Cookie.DatasourceAuth + ) + if (!authStateCookie) { + throw new Error("Unable to retrieve datasource authentication cookie") + } const provider = authStateCookie.provider const { middleware } = require(`@budibase/backend-core`) const handler = middleware.datasource[provider] From be6cb0825c21bfc0aa463a198a9ae80c8996b66c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Dec 2023 18:27:56 +0000 Subject: [PATCH 09/17] Fixing an issue with import and images not displaying correctly. --- packages/server/src/utilities/rowProcessor/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 293e74312e..a6817ddf19 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -249,7 +249,9 @@ export async function outputProcessing( continue } row[property].forEach((attachment: RowAttachment) => { - attachment.url ??= objectStore.getAppFileUrl(attachment.key) + if (!attachment.url) { + attachment.url = objectStore.getAppFileUrl(attachment.key) + } }) } } else if ( From 2f6bcdd6206ecd688329dc3be8e57d627675cc72 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 10:59:03 +0000 Subject: [PATCH 10/17] Allowing development (only dev) of big app export/imports. --- hosting/nginx.dev.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index 915125cbce..2d3e73ae64 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -42,7 +42,7 @@ http { server { listen 10000 default_server; server_name _; - client_max_body_size 1000m; + client_max_body_size 2000m; ignore_invalid_headers off; proxy_buffering off; From 0727df6f98dc6641d8551b47b3bf705ffe08830b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 11:39:46 +0000 Subject: [PATCH 11/17] Breaking attachment out into constant that can be re-used, and fixing some imports. --- packages/server/src/sdk/app/backups/constants.ts | 1 + packages/server/src/sdk/app/backups/exports.ts | 12 +++++++----- packages/server/src/sdk/app/backups/imports.ts | 13 ++++++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts index 0584fcb3d0..ddb1954f8f 100644 --- a/packages/server/src/sdk/app/backups/constants.ts +++ b/packages/server/src/sdk/app/backups/constants.ts @@ -1,3 +1,4 @@ export const DB_EXPORT_FILE = "db.txt" export const GLOBAL_DB_EXPORT_FILE = "global.txt" export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"] +export const ATTACHMENT_DIRECTORY = "attachments" diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index cc0b78e34f..813f813177 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -8,13 +8,15 @@ import { TABLE_ROW_PREFIX, USER_METDATA_PREFIX, } from "../../../db/utils" -import { DB_EXPORT_FILE, STATIC_APP_FILES } from "./constants" +import { + DB_EXPORT_FILE, + STATIC_APP_FILES, + ATTACHMENT_DIRECTORY, +} from "./constants" import fs from "fs" import { join } from "path" import env from "../../../environment" - -const uuid = require("uuid/v4") - +import { v4 as uuid } from "uuid" import tar from "tar" const MemoryStream = require("memorystream") @@ -150,7 +152,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { const path = join(tmpPath, file) // skip the attachments - too big to encrypt - if (file !== "attachments") { + if (file !== ATTACHMENT_DIRECTORY) { await encryption.encryptFile( { dir: tmpPath, filename: file }, config.encryptPassword diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 4c0030abf9..7f76945107 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -8,15 +8,18 @@ import { } from "@budibase/types" import { getAutomationParams } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" -import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" +import { + DB_EXPORT_FILE, + GLOBAL_DB_EXPORT_FILE, + ATTACHMENT_DIRECTORY, +} from "./constants" import { downloadTemplate } from "../../../utilities/fileSystem" import { ObjectStoreBuckets } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" - -const uuid = require("uuid/v4") -const tar = require("tar") +import { v4 as uuid } from "uuid" +import tar from "tar" type TemplateType = { file?: { @@ -129,7 +132,7 @@ async function decryptFiles(path: string, password: string) { try { for (let file of fs.readdirSync(path)) { const inputPath = join(path, file) - if (!inputPath.endsWith("attachments")) { + if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) { const outputPath = inputPath.replace(/\.enc$/, "") await encryption.decryptFile(inputPath, outputPath, password) fs.rmSync(inputPath) From 936ce9fcc21e42b55de9026968faf200fde7f6fa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 12:29:37 +0000 Subject: [PATCH 12/17] Adding test to make sure attachments are output correctly. --- .../tests/outputProcessing.spec.ts | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index 03584ef53b..95ce340910 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -3,6 +3,7 @@ import { FieldType, FieldTypeSubtypes, INTERNAL_TABLE_SOURCE_ID, + RowAttachment, Table, TableSourceType, } from "@budibase/types" @@ -70,6 +71,49 @@ describe("rowProcessor - outputProcessing", () => { ) }) + it("should handle attachments correctly", async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + attach: { + type: FieldType.ATTACHMENT, + name: "attach", + constraints: {}, + }, + }, + } + + const row: { attach: RowAttachment[] } = { + attach: [ + { + size: 10, + name: "test", + extension: "jpg", + key: "test.jpg", + }, + ], + } + + const output = await outputProcessing(table, row, { squash: false }) + expect(output.attach[0].url).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) + + row.attach[0].url = "" + const output2 = await outputProcessing(table, row, { squash: false }) + expect(output2.attach[0].url).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) + + row.attach[0].url = "aaaa" + const output3 = await outputProcessing(table, row, { squash: false }) + expect(output3.attach[0].url).toBe("aaaa") + }) + it("process output even when the field is not empty", async () => { const table: Table = { _id: generator.guid(), From d71be658a5f12f47f316696f2ff3cae226d30997 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 12:49:49 +0000 Subject: [PATCH 13/17] Upping size to 50GB in development to allow testing basically anything without needing to mess with the local proxy. --- hosting/nginx.dev.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index 2d3e73ae64..f0a58a9a98 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -42,7 +42,7 @@ http { server { listen 10000 default_server; server_name _; - client_max_body_size 2000m; + client_max_body_size 50000m; ignore_invalid_headers off; proxy_buffering off; From 244a1d93292b58441f24eefaaa71eebefbe52ddb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 14:47:13 +0000 Subject: [PATCH 14/17] PR comment. --- packages/backend-core/src/utils/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 429978fc97..ee1ef6da0c 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -172,7 +172,7 @@ export function getCookie(ctx: Ctx, name: string) { return undefined } - return openJwt(cookie) as T + return openJwt(cookie) } /** From 3dcb3062f5e2713345164156702b8753628782c9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 16:38:03 +0000 Subject: [PATCH 15/17] Adding a check to make sure that encryption does not execute upon directories, as well as adding copy to warn that attachments are not encrypted. --- .../backend-core/src/security/encryption.ts | 6 ++++++ .../src/components/start/ExportAppModal.svelte | 18 +++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index 7a8cfaf04a..45ed566a92 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -73,6 +73,9 @@ export async function encryptFile( const outputFileName = `${filename}.enc` const filePath = join(dir, filename) + if (fs.lstatSync(filePath).isDirectory()) { + throw new Error("Unable to encrypt directory") + } const inputFile = fs.createReadStream(filePath) const outputFile = fs.createWriteStream(join(dir, outputFileName)) @@ -110,6 +113,9 @@ export async function decryptFile( outputPath: string, secret: string ) { + if (fs.lstatSync(inputPath).isDirectory()) { + throw new Error("Unable to encrypt directory") + } const { salt, iv } = await getSaltAndIV(inputPath) const inputFile = fs.createReadStream(inputPath, { start: SALT_LENGTH + IV_LENGTH, diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index c05306c679..9a3a83aa89 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -13,7 +13,7 @@ export let app export let published let includeInternalTablesRows = true - let encypt = true + let encrypt = true let password = null const validation = createValidationStore() @@ -27,9 +27,9 @@ $: stepConfig = { [Step.CONFIG]: { title: published ? "Export published app" : "Export latest app", - confirmText: encypt ? "Continue" : exportButtonText, + confirmText: encrypt ? "Continue" : exportButtonText, onConfirm: () => { - if (!encypt) { + if (!encrypt) { exportApp() } else { currentStep = Step.SET_PASSWORD @@ -109,13 +109,13 @@ text="Export rows from internal tables" bind:value={includeInternalTablesRows} /> - + - {#if !encypt} - - {/if} + {/if} {#if currentStep === Step.SET_PASSWORD} Date: Wed, 6 Dec 2023 16:40:40 +0000 Subject: [PATCH 16/17] fixing wording. --- packages/builder/src/components/start/ExportAppModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index 9a3a83aa89..0727a2e493 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -113,7 +113,7 @@ {/if} From df81d7115602f0e23c6019a72cd5202321922db0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Dec 2023 16:41:44 +0000 Subject: [PATCH 17/17] Removing errant the. --- packages/builder/src/components/start/ExportAppModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index 0727a2e493..734e4448a1 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -113,7 +113,7 @@ {/if}