From 08de8f48dd9620dcb18ffb5e2831cee5495cdefc Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 9 Apr 2024 08:50:17 +0100 Subject: [PATCH 01/12] Rename borderless to quiet --- packages/client/manifest.json | 6 ++++++ packages/client/src/components/app/GridBlock.svelte | 2 ++ .../frontend-core/src/components/grid/layout/Grid.svelte | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 08d614391b..1d5f35905c 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -6708,6 +6708,12 @@ "key": "stripeRows", "defaultValue": false }, + { + "type": "boolean", + "label": "Quiet", + "key": "quiet", + "defaultValue": false + }, { "section": true, "name": "Columns", diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte index 46a507387d..9bf113a46e 100644 --- a/packages/client/src/components/app/GridBlock.svelte +++ b/packages/client/src/components/app/GridBlock.svelte @@ -11,6 +11,7 @@ export let allowEditRows = true export let allowDeleteRows = true export let stripeRows = false + export let quiet = false export let initialFilter = null export let initialSortColumn = null export let initialSortOrder = null @@ -116,6 +117,7 @@ datasource={table} {API} {stripeRows} + {quiet} {initialFilter} {initialSortColumn} {initialSortOrder} diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 1d2220951c..b6c686fd62 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -39,6 +39,7 @@ export let canEditColumns = true export let canSaveSchema = true export let stripeRows = false + export let quiet = false export let collaboration = true export let showAvatars = true export let showControls = true @@ -91,6 +92,7 @@ canEditColumns, canSaveSchema, stripeRows, + quiet, collaboration, showAvatars, showControls, @@ -124,6 +126,7 @@ class:is-resizing={$isResizing} class:is-reordering={$isReordering} class:stripe={stripeRows} + class:quiet on:mouseenter={() => gridFocused.set(true)} on:mouseleave={() => gridFocused.set(false)} style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-height:{MaxCellRenderHeight}px; --max-cell-render-width-overflow:{MaxCellRenderWidthOverflow}px; --content-lines:{$contentLines};" @@ -331,4 +334,9 @@ .grid-data-outer :global(.spectrum-Checkbox-partialCheckmark) { transition: none; } + + /* Overrides */ + .grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)) { + border-right: none; + } From 4c5f20bfe8d179ff102615c90774ac362cdcc927 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Apr 2024 18:21:37 +0200 Subject: [PATCH 02/12] WIP --- packages/types/src/sdk/licensing/plan.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 5ac8b1c9f6..016caf8c38 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -7,6 +7,7 @@ export enum PlanType { /** @deprecated */ PREMIUM = "premium", PREMIUM_PLUS = "premium_plus", + PREMIUM_PLUS_TRIAL = "premium_plus_trial", /** @deprecated */ BUSINESS = "business", ENTERPRISE_BASIC = "enterprise_basic", From cb8564f73eab62af6715aec2b4c936dd349278bd Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 19 Apr 2024 12:53:23 +0100 Subject: [PATCH 03/12] Clear the onEmptyFilter from datasource filtering when no fields are specified --- packages/frontend-core/src/components/FilterBuilder.svelte | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 1b252d5b06..b90882449f 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -67,6 +67,12 @@ const removeFilter = id => { filters = filters.filter(field => field.id !== id) + + // Clear all filters when no fields are specified + let [first] = filters + if (filters.length == 1 && first?.onEmptyFilter) { + filters = [] + } } const duplicateFilter = id => { From 659efe67d7859ad87fa7534d54cd968dfface9e3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 19 Apr 2024 16:40:45 +0100 Subject: [PATCH 04/12] Adding edge case handling to the binding readable/runtime conversion, checking if it is trying to replace a binding which a substring of a helper name, which causes the helper to become un-usable. --- packages/builder/src/dataBinding.js | 19 +++++++++++++++++++ packages/string-templates/src/index.ts | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 8fdf71c156..17acd48764 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -22,12 +22,14 @@ import { isJSBinding, decodeJSBinding, encodeJSBinding, + getJsHelperList, } from "@budibase/string-templates" import { TableNames } from "./constants" import { JSONUtils, Constants } from "@budibase/frontend-core" import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json" import { environment, licensing } from "stores/portal" import { convertOldFieldFormat } from "components/design/settings/controls/FieldConfiguration/utils" +import { helpersToCompletion } from "components/common/CodeEditor" const { ContextScopes } = Constants @@ -1210,6 +1212,23 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => { if (!currentValue?.includes(from)) { return false } + const helperNames = Object.keys(getJsHelperList()) + const matchedHelperNames = helperNames.filter( + name => name.includes(from) && currentValue.includes(name) + ) + // edge case - if the binding is part of a helper it may accidentally replace it + if (matchedHelperNames.length > 0) { + const indexStart = currentValue.indexOf(from), + indexEnd = indexStart + from.length + for (let helperName of matchedHelperNames) { + const helperIndexStart = currentValue.indexOf(helperName), + helperIndexEnd = helperIndexStart + helperName.length + if (indexStart >= helperIndexStart && indexEnd <= helperIndexEnd) { + return false + } + } + } + if (convertTo === "readableBinding") { // Dont replace if the value already matches the readable binding return currentValue.indexOf(binding.readableBinding) === -1 diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index 847567cb5a..0992813e9d 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -16,7 +16,7 @@ import { setJSRunner, removeJSRunner } from "./helpers/javascript" import manifest from "./manifest.json" import { ProcessOptions } from "./types" -export { helpersToRemoveForJs } from "./helpers/list" +export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list" export { FIND_ANY_HBS_REGEX } from "./utilities" export { setJSRunner, setOnErrorLog } from "./helpers/javascript" export { iifeWrapper } from "./iife" From d3286474476f415919419476d0fc256240c2a7d5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 19 Apr 2024 16:49:58 +0100 Subject: [PATCH 05/12] Removing accidental import. --- packages/builder/src/dataBinding.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 17acd48764..316b111f35 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -29,7 +29,6 @@ import { JSONUtils, Constants } from "@budibase/frontend-core" import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json" import { environment, licensing } from "stores/portal" import { convertOldFieldFormat } from "components/design/settings/controls/FieldConfiguration/utils" -import { helpersToCompletion } from "components/common/CodeEditor" const { ContextScopes } = Constants From 0b830ae2e20f4561659f702085b833bdaf760918 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 19 Apr 2024 17:05:34 +0100 Subject: [PATCH 06/12] Fixing formulas not being converted back to readable. --- packages/builder/src/dataBinding.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 316b111f35..388b2411f5 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -1211,6 +1211,9 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => { if (!currentValue?.includes(from)) { return false } + // some cases we have the same binding for readable/runtime, specific logic for this + const sameBindings = binding.runtimeBinding.includes(binding.readableBinding) + const convertingToReadable = convertTo === "readableBinding" const helperNames = Object.keys(getJsHelperList()) const matchedHelperNames = helperNames.filter( name => name.includes(from) && currentValue.includes(name) @@ -1228,9 +1231,12 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => { } } - if (convertTo === "readableBinding") { - // Dont replace if the value already matches the readable binding + if (convertingToReadable && !sameBindings) { + // Don't replace if the value already matches the readable binding return currentValue.indexOf(binding.readableBinding) === -1 + } else if (convertingToReadable) { + // if the runtime and readable bindings are very similar, all we can do is check runtime is there + return currentValue.indexOf(binding.runtimeBinding) !== -1 } // remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then // this makes sure it is detected From 51933c124418c2673c8c51e3074a514200d39c41 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 22 Apr 2024 10:11:31 +0200 Subject: [PATCH 07/12] Update submodules --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 06b1064f7e..b55d5b3200 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 06b1064f7e2f7cac5d4bef2ee999796a2a1f0f2c +Subproject commit b55d5b32003e3e999a1cbf2e5f3e6ce8d71eace7 From ed3073a20dee097cb9d1a9c38d3ac735173c97ed Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 22 Apr 2024 09:12:05 +0100 Subject: [PATCH 08/12] PR feedback --- packages/frontend-core/src/components/FilterBuilder.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index b90882449f..074c2dbd9b 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -69,8 +69,7 @@ filters = filters.filter(field => field.id !== id) // Clear all filters when no fields are specified - let [first] = filters - if (filters.length == 1 && first?.onEmptyFilter) { + if (filters.length === 1 && filters[0].onEmptyFilter) { filters = [] } } From a18799a139820f4bfe353aaf9e1a19546b28d84e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 22 Apr 2024 11:50:08 +0100 Subject: [PATCH 09/12] Getting rid of previous check for docker-compose/docker compose as due to recent changes there is no alias for the docker compose command anymore to find. --- packages/cli/src/hosting/utils.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index cbf6d9b0c3..5c3ac33f44 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -54,11 +54,9 @@ export async function downloadDockerCompose() { export async function checkDockerConfigured() { const error = - "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose" + "docker has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose" const docker = await lookpath("docker") - const compose = await lookpath("docker-compose") - const composeV2 = await lookpath("docker compose") - if (!docker || (!compose && !composeV2)) { + if (!docker) { throw error } } From 0266be8138b93ccc867fd36f64902a84d803ceca Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 22 Apr 2024 12:26:30 +0100 Subject: [PATCH 10/12] Quick fix - no need to double check. --- packages/builder/src/dataBinding.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 388b2411f5..5efbb79611 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -1235,8 +1235,8 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => { // Don't replace if the value already matches the readable binding return currentValue.indexOf(binding.readableBinding) === -1 } else if (convertingToReadable) { - // if the runtime and readable bindings are very similar, all we can do is check runtime is there - return currentValue.indexOf(binding.runtimeBinding) !== -1 + // if the runtime and readable bindings are very similar we have to assume it should be replaced + return true } // remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then // this makes sure it is detected From b2f81276cdf17ab43118595e0ae9c79c626a7b46 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 22 Apr 2024 14:14:24 +0000 Subject: [PATCH 11/12] Bump version to 2.23.11 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index e8bcd4429c..728cddc194 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.10", + "version": "2.23.11", "npmClient": "yarn", "packages": [ "packages/*", From a4c0328c53610fab0209be6e25af502609d9e38d Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 22 Apr 2024 16:30:57 +0100 Subject: [PATCH 12/12] REST file handling and SMTP automation block attachments (#13403) * handle files in rest connector * fetch presigned url and return * further updates to handle files in rest connector * remove unused important and fix extension bug * wrong expiry param * tests * add const for temp bucket * handle ttl on bucket * more bucket ttl work * split out fileresponse and xmlresponse into utils * lint * remove log * fix tests * some pr comments * update function naming and lint * adding back needed response for frontend * use fsp * handle different content-disposition and potential path traversal * add test container for s3 / minio * add test case for filename* and ascii filenames * move tests into separate describe * remove log * up timeout * switch to minio image instead of localstack * use minio image instead of s3 for testing * stream file upload instead * use streamUpload and update signatures * update bucketcreate return * throw real error * tidy up * pro * pro ref fix? * pro fix * pro fix? * move minio test provider to backend-core * update email builder to allow attachments * testing for sending files via smtp * use backend-core minio test container in server * handle different types of url * fix minio test provider * test with container host * lint * try different hostname? * Revert "try different hostname?" This reverts commit cfefdb8ded2b49462604053cf140e7292771c651. * fix issue with fetching of signed url with test minio * update autoamtion attachments to take filename and url * fix tests * pro ref * fix parsing of url object * pr comments and linting * pro ref * fix pro again * fix pro * account-portal * fix null issue * fix ref * ref * When sending a file attachment in email fetch it directly from our object store * add more checks to ensure we're working with a signed url * update test to account for direct object store read * formatting * fix time issues within test * update bucket and path extraction to regex * use const in regex * pro * Updating TTL handling in upload functions (#13539) * Updating TTL handling in upload functions * describe ttl type * account for ttl creation in existing buckets and update types * fix tests * pro * pro --- packages/backend-core/src/environment.ts | 2 + .../src/objectStore/objectStore.ts | 121 ++++++++++++++---- .../backend-core/src/objectStore/utils.ts | 26 ++++ .../tests/core/utilities/index.ts | 3 + .../tests/core/utilities/minio.ts | 34 +++++ .../SetupPanel/AutomationBlockSetup.svelte | 56 +++++++- .../integration/KeyValueBuilder.svelte | 29 +++-- packages/cli/src/backups/objectStore.ts | 4 +- packages/pro | 2 +- packages/server/package.json | 3 + .../src/automations/steps/sendSmtpEmail.ts | 12 +- .../automations/tests/sendSmtpEmail.spec.ts | 8 ++ packages/server/src/integrations/rest.ts | 68 +++++----- .../src/integrations/tests/rest.spec.ts | 116 ++++++++++++++++- .../server/src/integrations/utils/utils.ts | 78 ++++++++++- .../src/utilities/fileSystem/clientLibrary.ts | 28 ++-- .../server/src/utilities/workerRequests.ts | 5 +- .../types/src/documents/app/automation.ts | 8 ++ .../src/api/controllers/global/email.ts | 2 + .../api/routes/global/tests/realEmail.spec.ts | 47 ++++++- packages/worker/src/tests/api/email.ts | 4 +- packages/worker/src/tests/jestEnv.ts | 4 +- .../worker/src/tests/structures/configs.ts | 4 +- packages/worker/src/utilities/email.ts | 40 +++++- yarn.lock | 19 ++- 25 files changed, 619 insertions(+), 104 deletions(-) create mode 100644 packages/backend-core/tests/core/utilities/minio.ts diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 8dbc904643..9ade81b9d7 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -29,6 +29,7 @@ const DefaultBucketName = { TEMPLATES: "templates", GLOBAL: "global", PLUGINS: "plugins", + TEMP: "tmp-file-attachments", } const selfHosted = !!parseInt(process.env.SELF_HOSTED || "") @@ -146,6 +147,7 @@ const environment = { process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL, PLUGIN_BUCKET_NAME: process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS, + TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP, USE_COUCH: process.env.USE_COUCH || true, MOCK_REDIS: process.env.MOCK_REDIS, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 8d18fb97fd..aa5365c5c3 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -7,31 +7,41 @@ import tar from "tar-fs" import zlib from "zlib" import { promisify } from "util" import { join } from "path" -import fs, { ReadStream } from "fs" +import fs, { PathLike, ReadStream } from "fs" import env from "../environment" -import { budibaseTempDir } from "./utils" +import { bucketTTLConfig, budibaseTempDir } from "./utils" import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" +import fsp from "fs/promises" const streamPipeline = promisify(stream.pipeline) // use this as a temporary store of buckets that are being created const STATE = { bucketCreationPromises: {}, } +const signedFilePrefix = "/files/signed" type ListParams = { ContinuationToken?: string } -type UploadParams = { +type BaseUploadParams = { bucket: string filename: string - path: string type?: string | null - // can be undefined, we will remove it - metadata?: { - [key: string]: string | undefined - } + metadata?: { [key: string]: string | undefined } + body?: ReadableStream | Buffer + ttl?: number + addTTL?: boolean + extra?: any +} + +type UploadParams = BaseUploadParams & { + path?: string | PathLike +} + +type StreamUploadParams = BaseUploadParams & { + stream: ReadStream } const CONTENT_TYPE_MAP: any = { @@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = { js: "application/javascript", json: "application/json", gz: "application/gzip", + svg: "image/svg+xml", + form: "multipart/form-data", } const STRING_CONTENT_TYPES = [ @@ -105,7 +117,10 @@ export function ObjectStore( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export async function makeSureBucketExists(client: any, bucketName: string) { +export async function createBucketIfNotExists( + client: any, + bucketName: string +): Promise<{ created: boolean; exists: boolean }> { bucketName = sanitizeBucket(bucketName) try { await client @@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) { Bucket: bucketName, }) .promise() + return { created: false, exists: true } } catch (err: any) { const promises: any = STATE.bucketCreationPromises const doesntExist = err.statusCode === 404, noAccess = err.statusCode === 403 if (promises[bucketName]) { await promises[bucketName] + return { created: false, exists: true } } else if (doesntExist || noAccess) { if (doesntExist) { - // bucket doesn't exist create it promises[bucketName] = client .createBucket({ Bucket: bucketName, @@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) { .promise() await promises[bucketName] delete promises[bucketName] + return { created: true, exists: false } + } else { + throw new Error("Access denied to object store bucket." + err) } } else { throw new Error("Unable to write to object store bucket.") } } } - /** * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). @@ -146,12 +164,22 @@ export async function upload({ path, type, metadata, + body, + ttl, }: UploadParams) { const extension = filename.split(".").pop() - const fileBytes = fs.readFileSync(path) + + const fileBytes = path ? (await fsp.open(path)).createReadStream() : body const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) + + if (ttl && (bucketCreated.created || bucketCreated.exists)) { + let ttlConfig = bucketTTLConfig(bucketName, ttl) + if (objectStore.putBucketLifecycleConfiguration) { + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() + } + } let contentType = type if (!contentType) { @@ -174,6 +202,7 @@ export async function upload({ } config.Metadata = metadata } + return objectStore.upload(config).promise() } @@ -181,14 +210,24 @@ export async function upload({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export async function streamUpload( - bucketName: string, - filename: string, - stream: ReadStream | ReadableStream, - extra = {} -) { +export async function streamUpload({ + bucket: bucketName, + stream, + filename, + type, + extra, + ttl, +}: StreamUploadParams) { + const extension = filename.split(".").pop() const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) + + if (ttl && (bucketCreated.created || bucketCreated.exists)) { + let ttlConfig = bucketTTLConfig(bucketName, ttl) + if (objectStore.putBucketLifecycleConfiguration) { + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() + } + } // Set content type for certain known extensions if (filename?.endsWith(".js")) { @@ -203,10 +242,18 @@ export async function streamUpload( } } + let contentType = type + if (!contentType) { + contentType = extension + ? CONTENT_TYPE_MAP[extension.toLowerCase()] + : CONTENT_TYPE_MAP.txt + } + const params = { Bucket: sanitizeBucket(bucketName), Key: sanitizeKey(filename), Body: stream, + ContentType: contentType, ...extra, } return objectStore.upload(params).promise() @@ -286,7 +333,7 @@ export function getPresignedUrl( const signedUrl = new URL(url) const path = signedUrl.pathname const query = signedUrl.search - return `/files/signed${path}${query}` + return `${signedFilePrefix}${path}${query}` } } @@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) { */ export async function deleteFile(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + await createBucketIfNotExists(objectStore, bucketName) const params = { Bucket: bucketName, Key: sanitizeKey(filepath), @@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) { export async function deleteFiles(bucketName: string, filepaths: string[]) { const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + await createBucketIfNotExists(objectStore, bucketName) const params = { Bucket: bucketName, Delete: { @@ -412,7 +459,13 @@ export async function uploadDirectory( if (file.isDirectory()) { uploads.push(uploadDirectory(bucketName, local, path)) } else { - uploads.push(streamUpload(bucketName, path, fs.createReadStream(local))) + uploads.push( + streamUpload({ + bucket: bucketName, + filename: path, + stream: fs.createReadStream(local), + }) + ) } } await Promise.all(uploads) @@ -467,3 +520,23 @@ export async function getReadStream( } return client.getObject(params).createReadStream() } + +/* +Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract +the bucket and the path from it +*/ +export function extractBucketAndPath( + url: string +): { bucket: string; path: string } | null { + const baseUrl = url.split("?")[0] + + const regex = new RegExp(`^${signedFilePrefix}/(?[^/]+)/(?.+)$`) + const match = baseUrl.match(regex) + + if (match && match.groups) { + const { bucket, path } = match.groups + return { bucket, path } + } + + return null +} diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index 4c3a84ba91..08b5238ff6 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -2,6 +2,7 @@ import { join } from "path" import { tmpdir } from "os" import fs from "fs" import env from "../environment" +import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3" /**************************************************** * NOTE: When adding a new bucket - name * @@ -15,6 +16,7 @@ export const ObjectStoreBuckets = { TEMPLATES: env.TEMPLATES_BUCKET_NAME, GLOBAL: env.GLOBAL_BUCKET_NAME, PLUGINS: env.PLUGIN_BUCKET_NAME, + TEMP: env.TEMP_BUCKET_NAME, } const bbTmp = join(tmpdir(), ".budibase") @@ -29,3 +31,27 @@ try { export function budibaseTempDir() { return bbTmp } + +export const bucketTTLConfig = ( + bucketName: string, + days: number +): PutBucketLifecycleConfigurationRequest => { + const lifecycleRule = { + ID: `${bucketName}-ExpireAfter${days}days`, + Prefix: "", + Status: "Enabled", + Expiration: { + Days: days, + }, + } + const lifecycleConfiguration = { + Rules: [lifecycleRule], + } + + const params = { + Bucket: bucketName, + LifecycleConfiguration: lifecycleConfiguration, + } + + return params +} diff --git a/packages/backend-core/tests/core/utilities/index.ts b/packages/backend-core/tests/core/utilities/index.ts index 787d69be2c..b2f19a0286 100644 --- a/packages/backend-core/tests/core/utilities/index.ts +++ b/packages/backend-core/tests/core/utilities/index.ts @@ -4,3 +4,6 @@ export { generator } from "./structures" export * as testContainerUtils from "./testContainerUtils" export * as utils from "./utils" export * from "./jestUtils" +import * as minio from "./minio" + +export const objectStoreTestProviders = { minio } diff --git a/packages/backend-core/tests/core/utilities/minio.ts b/packages/backend-core/tests/core/utilities/minio.ts new file mode 100644 index 0000000000..cef33daa91 --- /dev/null +++ b/packages/backend-core/tests/core/utilities/minio.ts @@ -0,0 +1,34 @@ +import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" +import env from "../../../src/environment" + +let container: StartedTestContainer | undefined + +class ObjectStoreWaitStrategy extends AbstractWaitStrategy { + async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { + const logs = Wait.forListeningPorts() + await logs.waitUntilReady(container, boundPorts, startTime) + } +} + +export async function start(): Promise { + container = await new GenericContainer("minio/minio") + .withExposedPorts(9000) + .withCommand(["server", "/data"]) + .withEnvironment({ + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + }) + .withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000)) + .start() + + const port = container.getMappedPort(9000) + env._set("MINIO_URL", `http://0.0.0.0:${port}`) +} + +export async function stop() { + if (container) { + await container.stop() + container = undefined + } +} diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 6434c7710d..2d2022299c 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -32,6 +32,7 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte" import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte" + import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import { BindingHelpers, BindingType } from "components/common/bindings/utils" import { bindingsToCompletions, @@ -356,7 +357,8 @@ value.customType !== "queryParams" && value.customType !== "cron" && value.customType !== "triggerSchema" && - value.customType !== "automationFields" + value.customType !== "automationFields" && + value.type !== "attachment" ) } @@ -372,6 +374,15 @@ console.error(error) } }) + const handleAttachmentParams = keyValuObj => { + let params = {} + if (keyValuObj?.length) { + for (let param of keyValuObj) { + params[param.url] = param.filename + } + } + return params + }
@@ -437,6 +448,33 @@ value={inputData[key]} options={Object.keys(table?.schema || {})} /> + {:else if value.type === "attachment"} +
+
+ +
+
+ + onChange( + { + detail: e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + key + )} + object={handleAttachmentParams(inputData[key])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + /> +
+
{:else if value.customType === "filters"} Define filters @@ -651,14 +689,22 @@ } .block-field { - display: flex; /* Use Flexbox */ + display: flex; justify-content: space-between; - flex-direction: row; /* Arrange label and field side by side */ - align-items: center; /* Align vertically in the center */ - gap: 10px; /* Add some space between label and field */ + flex-direction: row; + align-items: center; + gap: 10px; flex: 1; } + .attachment-field-width { + margin-top: var(--spacing-xs); + } + + .label-wrapper { + margin-top: var(--spacing-s); + } + .test :global(.drawer) { width: 10000px !important; } diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 74636fc50c..5ed18a970a 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -35,6 +35,8 @@ export let bindingDrawerLeft export let allowHelpers = true export let customButtonText = null + export let keyBindings = false + export let allowJS = false export let compare = (option, value) => option === value let fields = Object.entries(object || {}).map(([name, value]) => ({ @@ -116,12 +118,23 @@ class:readOnly-menu={readOnly && showMenu} > {#each fields as field, idx} - + {#if keyBindings} + { + field.name = e.detail + changed() + }} + disabled={readOnly} + value={field.name} + {allowJS} + {allowHelpers} + drawerLeft={bindingDrawerLeft} + /> + {:else} + + {/if} {#if isJsonArray(field.value)}