diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 6cea7efeba..4beb02c9c7 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -281,7 +281,7 @@ export function doInScimContext(task: any) { return newContext(updates, task) } -export async function ensureSnippetContext() { +export async function ensureSnippetContext(enabled = !env.isTest()) { const ctx = getCurrentContext() // If we've already added snippets to context, continue @@ -292,7 +292,7 @@ export async function ensureSnippetContext() { // Otherwise get snippets for this app and update context let snippets: Snippet[] | undefined const db = getAppDB() - if (db && !env.isTest()) { + if (db && enabled) { const app = await db.get(DocumentType.APP_METADATA) snippets = app.snippets } diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 2bef91ffef..0ac2c35179 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils" import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" import fsp from "fs/promises" +import { HeadObjectOutput } from "aws-sdk/clients/s3" const streamPipeline = promisify(stream.pipeline) // use this as a temporary store of buckets that are being created const STATE = { bucketCreationPromises: {}, } -const signedFilePrefix = "/files/signed" +export const SIGNED_FILE_PREFIX = "/files/signed" type ListParams = { ContinuationToken?: string @@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & { path?: string | PathLike } -type StreamUploadParams = BaseUploadParams & { - stream: ReadStream +export type StreamTypes = + | ReadStream + | NodeJS.ReadableStream + | ReadableStream + +export type StreamUploadParams = BaseUploadParams & { + stream?: StreamTypes } const CONTENT_TYPE_MAP: any = { @@ -174,11 +180,9 @@ export async function upload({ const objectStore = ObjectStore(bucketName) const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) - if (ttl && (bucketCreated.created || bucketCreated.exists)) { + if (ttl && bucketCreated.created) { let ttlConfig = bucketTTLConfig(bucketName, ttl) - if (objectStore.putBucketLifecycleConfiguration) { - await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() - } + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() } let contentType = type @@ -222,11 +226,9 @@ export async function streamUpload({ const objectStore = ObjectStore(bucketName) const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) - if (ttl && (bucketCreated.created || bucketCreated.exists)) { + if (ttl && bucketCreated.created) { let ttlConfig = bucketTTLConfig(bucketName, ttl) - if (objectStore.putBucketLifecycleConfiguration) { - await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() - } + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() } // Set content type for certain known extensions @@ -333,7 +335,7 @@ export function getPresignedUrl( const signedUrl = new URL(url) const path = signedUrl.pathname const query = signedUrl.search - return `${signedFilePrefix}${path}${query}` + return `${SIGNED_FILE_PREFIX}${path}${query}` } } @@ -521,6 +523,26 @@ export async function getReadStream( return client.getObject(params).createReadStream() } +export async function getObjectMetadata( + bucket: string, + path: string +): Promise { + bucket = sanitizeBucket(bucket) + path = sanitizeKey(path) + + const client = ObjectStore(bucket) + const params = { + Bucket: bucket, + Key: path, + } + + try { + return await client.headObject(params).promise() + } catch (err: any) { + throw new Error("Unable to retrieve metadata from object") + } +} + /* Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract the bucket and the path from it @@ -530,7 +552,9 @@ export function extractBucketAndPath( ): { bucket: string; path: string } | null { const baseUrl = url.split("?")[0] - const regex = new RegExp(`^${signedFilePrefix}/(?[^/]+)/(?.+)$`) + const regex = new RegExp( + `^${SIGNED_FILE_PREFIX}/(?[^/]+)/(?.+)$` + ) const match = baseUrl.match(regex) if (match && match.groups) { diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index 08b5238ff6..5b9c2e3646 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -1,9 +1,14 @@ -import { join } from "path" +import path, { join } from "path" import { tmpdir } from "os" import fs from "fs" import env from "../environment" import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3" - +import * as objectStore from "./objectStore" +import { + AutomationAttachment, + AutomationAttachmentContent, + BucketedContent, +} from "@budibase/types" /**************************************************** * NOTE: When adding a new bucket - name * * sure that S3 usages (like budibase-infra) * @@ -55,3 +60,50 @@ export const bucketTTLConfig = ( return params } + +async function processUrlAttachment( + attachment: AutomationAttachment +): Promise { + const response = await fetch(attachment.url) + if (!response.ok || !response.body) { + throw new Error(`Unexpected response ${response.statusText}`) + } + const fallbackFilename = path.basename(new URL(attachment.url).pathname) + return { + filename: attachment.filename || fallbackFilename, + content: response.body, + } +} + +export async function processObjectStoreAttachment( + attachment: AutomationAttachment +): Promise { + const result = objectStore.extractBucketAndPath(attachment.url) + + if (result === null) { + throw new Error("Invalid signed URL") + } + + const { bucket, path: objectPath } = result + const readStream = await objectStore.getReadStream(bucket, objectPath) + const fallbackFilename = path.basename(objectPath) + return { + bucket, + path: objectPath, + filename: attachment.filename || fallbackFilename, + content: readStream, + } +} + +export async function processAutomationAttachment( + attachment: AutomationAttachment +): Promise { + const isFullyFormedUrl = + attachment.url?.startsWith("http://") || + attachment.url?.startsWith("https://") + if (isFullyFormedUrl) { + return await processUrlAttachment(attachment) + } else { + return await processObjectStoreAttachment(attachment) + } +} diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 2d2022299c..0cf0f6c740 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -358,7 +358,8 @@ value.customType !== "cron" && value.customType !== "triggerSchema" && value.customType !== "automationFields" && - value.type !== "attachment" + value.type !== "attachment" && + value.type !== "attachment_single" ) } diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index 0d15df6c87..ab020aad08 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -2,6 +2,8 @@ import { tables } from "stores/builder" import { Select, Checkbox, Label } from "@budibase/bbui" import { createEventDispatcher } from "svelte" + import { FieldType } from "@budibase/types" + import RowSelectorTypes from "./RowSelectorTypes.svelte" import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte" import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte" @@ -14,7 +16,6 @@ export let bindings export let isTestModal export let isUpdateRow - $: parsedBindings = bindings.map(binding => { let clone = Object.assign({}, binding) clone.icon = "ShareAndroid" @@ -26,15 +27,19 @@ $: { table = $tables.list.find(table => table._id === value?.tableId) - schemaFields = Object.entries(table?.schema ?? {}) - // surface the schema so the user can see it in the json - schemaFields.map(([, schema]) => { + + // Just sorting attachment types to the bottom here for a cleaner UX + schemaFields = Object.entries(table?.schema ?? {}).sort( + ([, schemaA], [, schemaB]) => + (schemaA.type === "attachment") - (schemaB.type === "attachment") + ) + + schemaFields.forEach(([, schema]) => { if (!schema.autocolumn && !value[schema.name]) { value[schema.name] = "" } }) } - const onChangeTable = e => { value["tableId"] = e.detail dispatch("change", value) @@ -114,10 +119,16 @@ {#if schemaFields.length} {#each schemaFields as [field, schema]} - {#if !schema.autocolumn && schema.type !== "attachment"} -
+ {#if !schema.autocolumn} +
-
+
{#if isTestModal} import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui" + import { FieldType } from "@budibase/types" import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte" import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte" import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte" import Editor from "components/integration/QueryEditor.svelte" + import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" export let onChange export let field @@ -22,6 +24,27 @@ function schemaHasOptions(schema) { return !!schema.constraints?.inclusion?.length } + + const handleAttachmentParams = keyValuObj => { + let params = {} + + if ( + schema.type === FieldType.ATTACHMENT_SINGLE && + Object.keys(keyValuObj).length === 0 + ) { + return [] + } + if (!Array.isArray(keyValuObj)) { + keyValuObj = [keyValuObj] + } + + if (keyValuObj.length) { + for (let param of keyValuObj) { + params[param.url] = param.filename + } + } + return params + } {#if schemaHasOptions(schema) && schema.type !== "array"} @@ -77,6 +100,35 @@ on:change={e => onChange(e, field)} useLabel={false} /> +{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE} +
+ + onChange( + { + detail: + schema.type === FieldType.ATTACHMENT_SINGLE + ? e.detail.length > 0 + ? { url: e.detail[0].name, filename: e.detail[0].value } + : {} + : e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + field + )} + object={handleAttachmentParams(value[field])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE && + Object.keys(value[field]).length >= 1} + /> +
{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)} {/if} + + diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index e3937ab772..decf77069f 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -1,7 +1,9 @@ - + {text} - - dispatch("change", tempValue)} - > -
- (tempValue = e.detail)} - /> -
-
-
- + + + + (tempValue = e.detail)} + {bindings} + /> + + diff --git a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte index 8ce9dda209..fb448cca8d 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte @@ -4,6 +4,7 @@ readableToRuntimeBinding, runtimeToReadableBinding, } from "dataBinding" + import { FieldType } from "@budibase/types" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { createEventDispatcher, setContext } from "svelte" @@ -102,6 +103,8 @@ longform: value => !isJSBinding(value), json: value => !isJSBinding(value), boolean: isValidBoolean, + attachment: false, + attachment_single: false, } const isValid = value => { @@ -116,7 +119,16 @@ if (type === "json" && !isJSBinding(value)) { return "json-slot-icon" } - if (!["string", "number", "bigint", "barcodeqr"].includes(type)) { + if ( + ![ + "string", + "number", + "bigint", + "barcodeqr", + "attachment", + "attachment_single", + ].includes(type) + ) { return "slot-icon" } return "" @@ -157,7 +169,7 @@ {updateOnChange} /> {/if} - {#if !disabled && type !== "formula"} + {#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
{ diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 5ed18a970a..6f69e71ccb 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -37,6 +37,7 @@ export let customButtonText = null export let keyBindings = false export let allowJS = false + export let actionButtonDisabled = false export let compare = (option, value) => option === value let fields = Object.entries(object || {}).map(([name, value]) => ({ @@ -189,7 +190,14 @@ {/if} {#if !readOnly && !noAddButton}
- + {#if customButtonText} {customButtonText} {:else} diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 1c569cb459..2e61238cc8 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -289,6 +289,7 @@ OperatorOptions.ContainsAny.value, ].includes(filter.operator)} disabled={filter.noValue} + type={filter.valueType} /> {:else} @@ -325,8 +326,6 @@