MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

REST file handling and SMTP automation block attachments (#13403) 

* handle files in rest connector

* fetch presigned url and return

* further updates to handle files in rest connector

* remove unused important and fix extension bug

* wrong expiry param

* tests

* add const for temp bucket

* handle ttl on bucket

* more bucket ttl work

* split out fileresponse and xmlresponse into utils

* lint

* remove log

* fix tests

* some pr comments

* update function naming and lint

* adding back needed response for frontend

* use fsp

* handle different content-disposition and potential path traversal

* add test container for s3 / minio

* add test case for filename* and ascii filenames

* move tests into separate describe

* remove log

* up timeout

* switch to minio image instead of localstack

* use minio image instead of s3 for testing

* stream file upload instead

* use streamUpload and update signatures

* update bucketcreate return

* throw real error

* tidy up

* pro

* pro ref fix?

* pro fix

* pro fix?

* move minio test provider to backend-core

* update email builder to allow attachments

* testing for sending files via smtp

* use backend-core minio test container in server

* handle different types of url

* fix minio test provider

* test with container host

* lint

* try different hostname?

* Revert "try different hostname?"

This reverts commit cfefdb8ded.

* fix issue with fetching of signed url with test minio

* update autoamtion attachments to take filename and url

* fix tests

* pro ref

* fix parsing of url object

* pr comments and linting

* pro ref

* fix pro again

* fix pro

* account-portal

* fix null issue

* fix ref

* ref

* When sending a file attachment in email fetch it directly from our object store

* add more checks to ensure we're working with a signed url

* update test to account for direct object store read

* formatting

* fix time issues within test

* update bucket and path extraction to regex

* use const in regex

* pro

* Updating TTL handling in upload functions (#13539)

* Updating TTL handling in upload functions

* describe ttl type

* account for ttl creation in existing buckets and update types

* fix tests

* pro

* pro
This commit is contained in:
Peter Clement 2024-04-22 16:30:57 +01:00 committed by GitHub
parent b2f81276cd
commit a4c0328c53
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 619 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend-core/src/environment.ts
View File

@ -29,6 +29,7 @@ const DefaultBucketName = {
TEMPLATES: "templates",
GLOBAL: "global",
PLUGINS: "plugins",
TEMP: "tmp-file-attachments",
}
const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
@ -146,6 +147,7 @@ const environment = {
process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
PLUGIN_BUCKET_NAME:
process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP,
USE_COUCH: process.env.USE_COUCH || true,
MOCK_REDIS: process.env.MOCK_REDIS,
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,

121
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -7,31 +7,41 @@ import tar from "tar-fs"
import zlib from "zlib"
import { promisify } from "util"
import { join } from "path"
import fs, { ReadStream } from "fs"
import fs, { PathLike, ReadStream } from "fs"
import env from "../environment"
import { budibaseTempDir } from "./utils"
import { bucketTTLConfig, budibaseTempDir } from "./utils"
import { v4 } from "uuid"
import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
import fsp from "fs/promises"
const streamPipeline = promisify(stream.pipeline)
// use this as a temporary store of buckets that are being created
const STATE = {
bucketCreationPromises: {},
}
const signedFilePrefix = "/files/signed"
type ListParams = {
ContinuationToken?: string
}
type UploadParams = {
type BaseUploadParams = {
bucket: string
filename: string
path: string
type?: string | null
// can be undefined, we will remove it
metadata?: {
[key: string]: string | undefined
}
metadata?: { [key: string]: string | undefined }
body?: ReadableStream | Buffer
ttl?: number
addTTL?: boolean
extra?: any
}
type UploadParams = BaseUploadParams & {
path?: string | PathLike
}
type StreamUploadParams = BaseUploadParams & {
stream: ReadStream
}
const CONTENT_TYPE_MAP: any = {
@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = {
js: "application/javascript",
json: "application/json",
gz: "application/gzip",
svg: "image/svg+xml",
form: "multipart/form-data",
}
const STRING_CONTENT_TYPES = [
@ -105,7 +117,10 @@ export function ObjectStore(
* Given an object store and a bucket name this will make sure the bucket exists,
* if it does not exist then it will create it.
*/
export async function makeSureBucketExists(client: any, bucketName: string) {
export async function createBucketIfNotExists(
client: any,
bucketName: string
): Promise<{ created: boolean; exists: boolean }> {
bucketName = sanitizeBucket(bucketName)
try {
await client
@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
Bucket: bucketName,
})
.promise()
return { created: false, exists: true }
} catch (err: any) {
const promises: any = STATE.bucketCreationPromises
const doesntExist = err.statusCode === 404,
noAccess = err.statusCode === 403
if (promises[bucketName]) {
await promises[bucketName]
return { created: false, exists: true }
} else if (doesntExist || noAccess) {
if (doesntExist) {
// bucket doesn't exist create it
promises[bucketName] = client
.createBucket({
Bucket: bucketName,
@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
.promise()
await promises[bucketName]
delete promises[bucketName]
return { created: true, exists: false }
} else {
throw new Error("Access denied to object store bucket." + err)
}
} else {
throw new Error("Unable to write to object store bucket.")
}
}
}
/**
* Uploads the contents of a file given the required parameters, useful when
* temp files in use (for example file uploaded as an attachment).
@ -146,12 +164,22 @@ export async function upload({
path,
type,
metadata,
body,
ttl,
}: UploadParams) {
const extension = filename.split(".").pop()
const fileBytes = fs.readFileSync(path)
const fileBytes = path ? (await fsp.open(path)).createReadStream() : body
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
let ttlConfig = bucketTTLConfig(bucketName, ttl)
if (objectStore.putBucketLifecycleConfiguration) {
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
}
}
let contentType = type
if (!contentType) {
@ -174,6 +202,7 @@ export async function upload({
}
config.Metadata = metadata
}
return objectStore.upload(config).promise()
}
@ -181,14 +210,24 @@ export async function upload({
* Similar to the upload function but can be used to send a file stream
* through to the object store.
*/
export async function streamUpload(
bucketName: string,
filename: string,
stream: ReadStream | ReadableStream,
extra = {}
) {
export async function streamUpload({
bucket: bucketName,
stream,
filename,
type,
extra,
ttl,
}: StreamUploadParams) {
const extension = filename.split(".").pop()
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
let ttlConfig = bucketTTLConfig(bucketName, ttl)
if (objectStore.putBucketLifecycleConfiguration) {
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
}
}
// Set content type for certain known extensions
if (filename?.endsWith(".js")) {
@ -203,10 +242,18 @@ export async function streamUpload(
}
}
let contentType = type
if (!contentType) {
contentType = extension
? CONTENT_TYPE_MAP[extension.toLowerCase()]
: CONTENT_TYPE_MAP.txt
}
const params = {
Bucket: sanitizeBucket(bucketName),
Key: sanitizeKey(filename),
Body: stream,
ContentType: contentType,
...extra,
}
return objectStore.upload(params).promise()
@ -286,7 +333,7 @@ export function getPresignedUrl(
const signedUrl = new URL(url)
const path = signedUrl.pathname
const query = signedUrl.search
return `/files/signed${path}${query}`
return `${signedFilePrefix}${path}${query}`
}
}
@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
*/
export async function deleteFile(bucketName: string, filepath: string) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
await createBucketIfNotExists(objectStore, bucketName)
const params = {
Bucket: bucketName,
Key: sanitizeKey(filepath),
@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) {
export async function deleteFiles(bucketName: string, filepaths: string[]) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
await createBucketIfNotExists(objectStore, bucketName)
const params = {
Bucket: bucketName,
Delete: {
@ -412,7 +459,13 @@ export async function uploadDirectory(
if (file.isDirectory()) {
uploads.push(uploadDirectory(bucketName, local, path))
} else {
uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
uploads.push(
streamUpload({
bucket: bucketName,
filename: path,
stream: fs.createReadStream(local),
})
)
}
}
await Promise.all(uploads)
@ -467,3 +520,23 @@ export async function getReadStream(
}
return client.getObject(params).createReadStream()
}
/*
Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
the bucket and the path from it
*/
export function extractBucketAndPath(
url: string
): { bucket: string; path: string } | null {
const baseUrl = url.split("?")[0]
const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
const match = baseUrl.match(regex)
if (match && match.groups) {
const { bucket, path } = match.groups
return { bucket, path }
}
return null
}

26
packages/backend-core/src/objectStore/utils.ts
View File

@ -2,6 +2,7 @@ import { join } from "path"
import { tmpdir } from "os"
import fs from "fs"
import env from "../environment"
import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
/****************************************************
* NOTE: When adding a new bucket - name *
@ -15,6 +16,7 @@ export const ObjectStoreBuckets = {
TEMPLATES: env.TEMPLATES_BUCKET_NAME,
GLOBAL: env.GLOBAL_BUCKET_NAME,
PLUGINS: env.PLUGIN_BUCKET_NAME,
TEMP: env.TEMP_BUCKET_NAME,
}
const bbTmp = join(tmpdir(), ".budibase")
@ -29,3 +31,27 @@ try {
export function budibaseTempDir() {
return bbTmp
}
export const bucketTTLConfig = (
bucketName: string,
days: number
): PutBucketLifecycleConfigurationRequest => {
const lifecycleRule = {
ID: `${bucketName}-ExpireAfter${days}days`,
Prefix: "",
Status: "Enabled",
Expiration: {
Days: days,
},
}
const lifecycleConfiguration = {
Rules: [lifecycleRule],
}
const params = {
Bucket: bucketName,
LifecycleConfiguration: lifecycleConfiguration,
}
return params
}

3
packages/backend-core/tests/core/utilities/index.ts
View File

@ -4,3 +4,6 @@ export { generator } from "./structures"
export * as testContainerUtils from "./testContainerUtils"
export * as utils from "./utils"
export * from "./jestUtils"
import * as minio from "./minio"
export const objectStoreTestProviders = { minio }

34
packages/backend-core/tests/core/utilities/minio.ts Normal file
View File

@ -0,0 +1,34 @@
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
import env from "../../../src/environment"
let container: StartedTestContainer | undefined
class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
const logs = Wait.forListeningPorts()
await logs.waitUntilReady(container, boundPorts, startTime)
}
}
export async function start(): Promise<void> {
container = await new GenericContainer("minio/minio")
.withExposedPorts(9000)
.withCommand(["server", "/data"])
.withEnvironment({
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
})
.withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
.start()
const port = container.getMappedPort(9000)
env._set("MINIO_URL", `http://0.0.0.0:${port}`)
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
}
}

56
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -32,6 +32,7 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import { BindingHelpers, BindingType } from "components/common/bindings/utils"
import {
bindingsToCompletions,
@ -356,7 +357,8 @@
value.customType !== "queryParams" &&
value.customType !== "cron" &&
value.customType !== "triggerSchema" &&
value.customType !== "automationFields"
value.customType !== "automationFields" &&
value.type !== "attachment"
)
}
@ -372,6 +374,15 @@
console.error(error)
}
})
const handleAttachmentParams = keyValuObj => {
let params = {}
if (keyValuObj?.length) {
for (let param of keyValuObj) {
params[param.url] = param.filename
}
}
return params
}
</script>
<div class="fields">
@ -437,6 +448,33 @@
value={inputData[key]}
options={Object.keys(table?.schema || {})}
/>
{:else if value.type === "attachment"}
<div class="attachment-field-wrapper">
<div class="label-wrapper">
<Label>{label}</Label>
</div>
<div class="attachment-field-width">
<KeyValueBuilder
on:change={e =>
onChange(
{
detail: e.detail.map(({ name, value }) => ({
url: name,
filename: value,
})),
},
key
)}
object={handleAttachmentParams(inputData[key])}
allowJS
{bindings}
keyBindings
customButtonText={"Add attachment"}
keyPlaceholder={"URL"}
valuePlaceholder={"Filename"}
/>
</div>
</div>
{:else if value.customType === "filters"}
<ActionButton on:click={drawer.show}>Define filters</ActionButton>
<Drawer bind:this={drawer} title="Filtering">
@ -651,14 +689,22 @@
}
.block-field {
display: flex; /* Use Flexbox */
display: flex;
justify-content: space-between;
flex-direction: row; /* Arrange label and field side by side */
align-items: center; /* Align vertically in the center */
gap: 10px; /* Add some space between label and field */
flex-direction: row;
align-items: center;
gap: 10px;
flex: 1;
}
.attachment-field-width {
margin-top: var(--spacing-xs);
}
.label-wrapper {
margin-top: var(--spacing-s);
}
.test :global(.drawer) {
width: 10000px !important;
}

29
packages/builder/src/components/integration/KeyValueBuilder.svelte
View File

@ -35,6 +35,8 @@
export let bindingDrawerLeft
export let allowHelpers = true
export let customButtonText = null
export let keyBindings = false
export let allowJS = false
export let compare = (option, value) => option === value
let fields = Object.entries(object || {}).map(([name, value]) => ({
@ -116,12 +118,23 @@
class:readOnly-menu={readOnly && showMenu}
>
{#each fields as field, idx}
<Input
placeholder={keyPlaceholder}
readonly={readOnly}
bind:value={field.name}
on:blur={changed}
/>
{#if keyBindings}
<DrawerBindableInput
{bindings}
placeholder={keyPlaceholder}
on:blur={e => {
field.name = e.detail
changed()
}}
disabled={readOnly}
value={field.name}
{allowJS}
{allowHelpers}
drawerLeft={bindingDrawerLeft}
/>
{:else}
<Input readonly={readOnly} bind:value={field.name} on:blur={changed} />
{/if}
{#if isJsonArray(field.value)}
<Select readonly={true} value="Array" options={["Array"]} />
{:else if options}
@ -134,14 +147,14 @@
{:else if bindings && bindings.length}
<DrawerBindableInput
{bindings}
placeholder="Value"
placeholder={valuePlaceholder}
on:blur={e => {
field.value = e.detail
changed()
}}
disabled={readOnly}
value={field.value}
allowJS={false}
{allowJS}
{allowHelpers}
drawerLeft={bindingDrawerLeft}
/>

4
packages/cli/src/backups/objectStore.ts
View File

@ -9,7 +9,7 @@ const {
ObjectStore,
retrieve,
uploadDirectory,
makeSureBucketExists,
createBucketIfNotExists,
} = objectStore
const bucketList = Object.values(ObjectStoreBuckets)
@ -61,7 +61,7 @@ export async function importObjects() {
let count = 0
for (let bucket of buckets) {
const client = ObjectStore(bucket)
await makeSureBucketExists(client, bucket)
await createBucketIfNotExists(client, bucket)
const files = await uploadDirectory(bucket, join(path, bucket), "/")
count += files.length
bar.update(count)

2
packages/pro

@ -1 +1 @@
Subproject commit b55d5b32003e3e999a1cbf2e5f3e6ce8d71eace7
Subproject commit dff7b5a9dd1fd770f8a48fb8e6df1740be605f18

3
packages/server/package.json
View File

@ -61,14 +61,17 @@
"@google-cloud/firestore": "6.8.0",
"@koa/router": "8.0.8",
"@socket.io/redis-adapter": "^8.2.1",
"@types/xml2js": "^0.4.14",
"airtable": "0.10.1",
"arangojs": "7.2.0",
"archiver": "7.0.1",
"aws-sdk": "2.1030.0",
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bl": "^6.0.12",
"bull": "4.10.1",
"chokidar": "3.5.3",
"content-disposition": "^0.5.4",
"cookies": "0.8.0",
"csvtojson": "2.0.10",
"curlconverter": "3.21.0",

12
packages/server/src/automations/steps/sendSmtpEmail.ts
View File

@ -7,6 +7,7 @@ import {
AutomationStepType,
AutomationIOType,
AutomationFeature,
AutomationCustomIOType,
} from "@budibase/types"
export const definition: AutomationStepSchema = {
@ -72,10 +73,10 @@ export const definition: AutomationStepSchema = {
title: "Location",
dependsOn: "addInvite",
},
url: {
type: AutomationIOType.STRING,
title: "URL",
dependsOn: "addInvite",
attachments: {
type: AutomationIOType.ATTACHMENT,
customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
title: "Attachments",
},
},
required: ["to", "from", "subject", "contents"],
@ -110,11 +111,13 @@ export async function run({ inputs }: AutomationStepInput) {
summary,
location,
url,
attachments,
} = inputs
if (!contents) {
contents = "<h1>No content</h1>"
}
to = to || undefined
try {
let response = await sendSmtpEmail({
to,
@ -124,6 +127,7 @@ export async function run({ inputs }: AutomationStepInput) {
cc,
bcc,
automation: true,
attachments,
invite: addInvite
? {
startTime,

8
packages/server/src/automations/tests/sendSmtpEmail.spec.ts
View File

@ -50,6 +50,10 @@ describe("test the outgoing webhook action", () => {
cc: "cc",
bcc: "bcc",
addInvite: true,
attachments: [
{ url: "attachment1", filename: "attachment1.txt" },
{ url: "attachment2", filename: "attachment2.txt" },
],
...invite,
}
let resp = generateResponse(inputs.to, inputs.from)
@ -69,6 +73,10 @@ describe("test the outgoing webhook action", () => {
bcc: "bcc",
invite,
automation: true,
attachments: [
{ url: "attachment1", filename: "attachment1.txt" },
{ url: "attachment2", filename: "attachment2.txt" },
],
})
})
})

68
packages/server/src/integrations/rest.ts
View File

@ -21,6 +21,10 @@ import { performance } from "perf_hooks"
import FormData from "form-data"
import { URLSearchParams } from "url"
import { blacklist } from "@budibase/backend-core"
import { handleFileResponse, handleXml } from "./utils"
import { parse } from "content-disposition"
import path from "path"
import { Builder as XmlBuilder } from "xml2js"
const BodyTypes = {
NONE: "none",
@ -57,8 +61,6 @@ const coreFields = {
},
}
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch",
description:
@ -129,42 +131,44 @@ class RestIntegration implements IntegrationBase {
}
async parseResponse(response: any, pagination: PaginationConfig | null) {
let data, raw, headers
let data, raw, headers, filename
const contentType = response.headers.get("content-type") || ""
const contentDisposition = response.headers.get("content-disposition") || ""
if (
contentDisposition.includes("attachment") ||
contentDisposition.includes("form-data")
) {
filename =
path.basename(parse(contentDisposition).parameters?.filename) || ""
}
try {
if (response.status === 204) {
data = []
raw = []
} else if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (
contentType.includes("text/xml") ||
contentType.includes("application/xml")
) {
const rawXml = await response.text()
data =
(await xmlParser(rawXml, {
explicitArray: false,
trim: true,
explicitRoot: false,
})) || {}
// there is only one structure, its an array, return the array so it appears as rows
const keys = Object.keys(data)
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
data = data[keys[0]]
}
raw = rawXml
} else if (contentType.includes("application/pdf")) {
data = await response.arrayBuffer() // Save PDF as ArrayBuffer
raw = Buffer.from(data)
if (filename) {
return handleFileResponse(response, filename, this.startTimeMs)
} else {
data = await response.text()
raw = data
if (response.status === 204) {
data = []
raw = []
} else if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (
contentType.includes("text/xml") ||
contentType.includes("application/xml")
) {
let xmlResponse = await handleXml(response)
data = xmlResponse.data
raw = xmlResponse.rawXml
} else {
data = await response.text()
raw = data
}
}
} catch (err) {
throw "Failed to parse response body."
throw `Failed to parse response body: ${err}`
}
const size = formatBytes(
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
)

116
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -13,9 +13,23 @@ jest.mock("node-fetch", () => {
}))
})
import fetch from "node-fetch"
jest.mock("@budibase/backend-core", () => {
const core = jest.requireActual("@budibase/backend-core")
return {
...core,
context: {
...core.context,
getProdAppId: jest.fn(() => "app-id"),
},
}
})
jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
import { default as RestIntegration } from "../rest"
import { RestAuthType } from "@budibase/types"
import fetch from "node-fetch"
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
import { Readable } from "stream"
const FormData = require("form-data")
const { URLSearchParams } = require("url")
@ -611,4 +625,104 @@ describe("REST Integration", () => {
expect(calledConfig.headers).toEqual({})
expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
})
describe("File Handling", () => {
beforeAll(async () => {
jest.unmock("aws-sdk")
await objectStoreTestProviders.minio.start()
})
afterAll(async () => {
await objectStoreTestProviders.minio.stop()
})
it("uploads file to object store and returns signed URL", async () => {
const responseData = Buffer.from("teest file contnt")
const filename = "test.tar.gz"
const contentType = "application/gzip"
const mockReadable = new Readable()
mockReadable.push(responseData)
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
headers: {
raw: () => ({
"content-type": [contentType],
"content-disposition": [`attachment; filename="${filename}"`],
}),
get: (header: any) => {
if (header === "content-type") return contentType
if (header === "content-disposition")
return `attachment; filename="${filename}"`
},
},
body: mockReadable,
})
)
const query = {
path: "api",
}
const response = await config.integration.read(query)
expect(response.data).toEqual({
size: responseData.byteLength,
name: "00000000-0000-0000-0000-000000000000.tar.gz",
url: expect.stringContaining(
"/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.tar.gz"
),
extension: "tar.gz",
key: expect.stringContaining(
"app-id/00000000-0000-0000-0000-000000000000.tar.gz"
),
})
})
it("uploads file with non ascii filename to object store and returns signed URL", async () => {
const responseData = Buffer.from("teest file contnt")
const contentType = "text/plain"
const mockReadable = new Readable()
mockReadable.push(responseData)
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
headers: {
raw: () => ({
"content-type": [contentType],
"content-disposition": [
// eslint-disable-next-line no-useless-escape
`attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`,
],
}),
get: (header: any) => {
if (header === "content-type") return contentType
if (header === "content-disposition")
// eslint-disable-next-line no-useless-escape
return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`
},
},
body: mockReadable,
})
)
const query = {
path: "api",
}
const response = await config.integration.read(query)
expect(response.data).toEqual({
size: responseData.byteLength,
name: "00000000-0000-0000-0000-000000000000.pdf",
url: expect.stringContaining(
"/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.pdf"
),
extension: "pdf",
key: expect.stringContaining(
"app-id/00000000-0000-0000-0000-000000000000.pdf"
),
})
})
})
})

78
packages/server/src/integrations/utils/utils.ts
View File

@ -6,10 +6,15 @@ import {
TableSourceType,
FieldSchema,
} from "@budibase/types"
import { context, objectStore } from "@budibase/backend-core"
import { v4 } from "uuid"
import { parseStringPromise as xmlParser } from "xml2js"
import { formatBytes } from "../../utilities"
import bl from "bl"
import env from "../../environment"
import { DocumentType, SEPARATOR } from "../../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
import { helpers, utils } from "@budibase/shared-core"
import env from "../../environment"
import { Knex } from "knex"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
@ -467,3 +472,74 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
export function isValidFilter(value: any) {
return value != null && value !== ""
}
export async function handleXml(response: any) {
let data,
rawXml = await response.text()
data =
(await xmlParser(rawXml, {
explicitArray: false,
trim: true,
explicitRoot: false,
})) || {}
// there is only one structure, its an array, return the array so it appears as rows
const keys = Object.keys(data)
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
data = data[keys[0]]
}
return { data, rawXml }
}
export async function handleFileResponse(
response: any,
filename: string,
startTime: number
) {
let presignedUrl,
size = 0
const fileExtension = filename.includes(".")
? filename.split(".").slice(1).join(".")
: ""
const processedFileName = `${v4()}.${fileExtension}`
const key = `${context.getProdAppId()}/${processedFileName}`
const bucket = objectStore.ObjectStoreBuckets.TEMP
const stream = response.body.pipe(bl((error, data) => data))
if (response.body) {
const contentLength = response.headers.get("content-length")
if (contentLength) {
size = parseInt(contentLength, 10)
} else {
const chunks: Buffer[] = []
for await (const chunk of response.body) {
chunks.push(chunk)
size += chunk.length
}
}
await objectStore.streamUpload({
bucket,
filename: key,
stream,
ttl: 1,
type: response.headers["content-type"],
})
}
presignedUrl = await objectStore.getPresignedUrl(bucket, key)
return {
data: {
size,
name: processedFileName,
url: presignedUrl,
extension: fileExtension,
key: key,
},
info: {
code: response.status,
size: formatBytes(size.toString()),
time: `${Math.round(performance.now() - startTime)}ms`,
},
}
}

28
packages/server/src/utilities/fileSystem/clientLibrary.ts
View File

@ -106,22 +106,22 @@ export async function updateClientLibrary(appId: string) {
}
// Upload latest manifest and client library
const manifestUpload = objectStore.streamUpload(
ObjectStoreBuckets.APPS,
join(appId, "manifest.json"),
fs.createReadStream(manifest),
{
const manifestUpload = objectStore.streamUpload({
bucket: ObjectStoreBuckets.APPS,
filename: join(appId, "manifest.json"),
stream: fs.createReadStream(manifest),
extra: {
ContentType: "application/json",
}
)
const clientUpload = objectStore.streamUpload(
ObjectStoreBuckets.APPS,
join(appId, "budibase-client.js"),
fs.createReadStream(client),
{
},
})
const clientUpload = objectStore.streamUpload({
bucket: ObjectStoreBuckets.APPS,
filename: join(appId, "budibase-client.js"),
stream: fs.createReadStream(client),
extra: {
ContentType: "application/javascript",
}
)
},
})
const manifestSrc = fs.promises.readFile(manifest, "utf8")

5
packages/server/src/utilities/workerRequests.ts
View File

@ -8,7 +8,7 @@ import {
logging,
env as coreEnv,
} from "@budibase/backend-core"
import { Ctx, User, EmailInvite } from "@budibase/types"
import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
interface Request {
ctx?: Ctx
@ -97,6 +97,7 @@ export async function sendSmtpEmail({
bcc,
automation,
invite,
attachments,
}: {
to: string
from: string
@ -105,6 +106,7 @@ export async function sendSmtpEmail({
cc: string
bcc: string
automation: boolean
attachments?: EmailAttachment[]
invite?: EmailInvite
}) {
// tenant ID will be set in header
@ -122,6 +124,7 @@ export async function sendSmtpEmail({
purpose: "custom",
automation,
invite,
attachments,
},
})
)

8
packages/types/src/documents/app/automation.ts
View File

@ -10,6 +10,7 @@ export enum AutomationIOType {
ARRAY = "array",
JSON = "json",
DATE = "date",
ATTACHMENT = "attachment",
}
export enum AutomationCustomIOType {
@ -30,6 +31,7 @@ export enum AutomationCustomIOType {
WEBHOOK_URL = "webhookUrl",
AUTOMATION = "automation",
AUTOMATION_FIELDS = "automationFields",
MULTI_ATTACHMENTS = "multi_attachments",
}
export enum AutomationTriggerStepId {
@ -80,6 +82,11 @@ export interface EmailInvite {
url?: string
}
export interface EmailAttachment {
url: string
filename: string
}
export interface SendEmailOpts {
// workspaceId If finer grain controls being used then this will lookup config for workspace.
workspaceId?: string
@ -97,6 +104,7 @@ export interface SendEmailOpts {
bcc?: boolean
automation?: boolean
invite?: EmailInvite
attachments?: EmailAttachment[]
}
export const AutomationStepIdArray = [

2
packages/worker/src/api/controllers/global/email.ts
View File

@ -15,6 +15,7 @@ export async function sendEmail(ctx: BBContext) {
bcc,
automation,
invite,
attachments,
} = ctx.request.body
let user: any
if (userId) {
@ -31,6 +32,7 @@ export async function sendEmail(ctx: BBContext) {
bcc,
automation,
invite,
attachments,
})
ctx.body = {
...response,

47
packages/worker/src/api/routes/global/tests/realEmail.spec.ts
View File

@ -1,9 +1,15 @@
jest.unmock("node-fetch")
jest.unmock("aws-sdk")
import { TestConfiguration } from "../../../../tests"
import { EmailTemplatePurpose } from "../../../../constants"
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
import { objectStore } from "@budibase/backend-core"
import tk from "timekeeper"
import { EmailAttachment } from "@budibase/types"
const fetch = require("node-fetch")
const nodemailer = require("nodemailer")
const fetch = require("node-fetch")
// for the real email tests give them a long time to try complete/fail
jest.setTimeout(30000)
@ -12,14 +18,20 @@ describe("/api/global/email", () => {
const config = new TestConfiguration()
beforeAll(async () => {
tk.reset()
await objectStoreTestProviders.minio.start()
await config.beforeAll()
})
afterAll(async () => {
await objectStoreTestProviders.minio.stop()
await config.afterAll()
})
async function sendRealEmail(purpose: string) {
async function sendRealEmail(
purpose: string,
attachments?: EmailAttachment[]
) {
let response, text
try {
const timeout = () =>
@ -35,8 +47,14 @@ describe("/api/global/email", () => {
)
await Promise.race([config.saveEtherealSmtpConfig(), timeout()])
await Promise.race([config.saveSettingsConfig(), timeout()])
const res = await config.api.emails.sendEmail(purpose).timeout(20000)
let res
if (attachments) {
res = await config.api.emails
.sendEmail(purpose, attachments)
.timeout(20000)
} else {
res = await config.api.emails.sendEmail(purpose).timeout(20000)
}
// ethereal hiccup, can't test right now
if (res.status >= 300) {
return
@ -80,4 +98,25 @@ describe("/api/global/email", () => {
it("should be able to send a password recovery email", async () => {
await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
})
it("should be able to send an email with attachments", async () => {
let bucket = "testbucket"
let filename = "test.txt"
await objectStore.upload({
bucket,
filename,
body: Buffer.from("test data"),
})
let presignedUrl = await objectStore.getPresignedUrl(
bucket,
filename,
60000
)
let attachmentObject = {
url: presignedUrl,
filename,
}
await sendRealEmail(EmailTemplatePurpose.WELCOME, [attachmentObject])
})
})

4
packages/worker/src/tests/api/email.ts
View File

@ -1,3 +1,4 @@
import { EmailAttachment } from "@budibase/types"
import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base"
@ -6,11 +7,12 @@ export class EmailAPI extends TestAPI {
super(config)
}
sendEmail = (purpose: string) => {
sendEmail = (purpose: string, attachments?: EmailAttachment[]) => {
return this.request
.post(`/api/global/email/send`)
.send({
email: "test@example.com",
attachments,
purpose,
tenantId: this.config.getTenantId(),
userId: this.config.user?._id!,

4
packages/worker/src/tests/jestEnv.ts
View File

@ -4,8 +4,8 @@ process.env.JWT_SECRET = "test-jwtsecret"
process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
process.env.MULTI_TENANCY = "1"
process.env.MINIO_URL = "http://localhost"
process.env.MINIO_ACCESS_KEY = "test"
process.env.MINIO_SECRET_KEY = "test"
process.env.MINIO_ACCESS_KEY = "budibase"
process.env.MINIO_SECRET_KEY = "budibase"
process.env.PLATFORM_URL = "http://localhost:10000"
process.env.INTERNAL_API_KEY = "tet"
process.env.DISABLE_ACCOUNT_PORTAL = "0"

4
packages/worker/src/tests/structures/configs.ts
View File

@ -62,8 +62,8 @@ export function smtpEthereal(): SMTPConfig {
from: "testfrom@example.com",
secure: false,
auth: {
user: "wyatt.zulauf29@ethereal.email",
pass: "tEwDtHBWWxusVWAPfa",
user: "mortimer.leuschke@ethereal.email",
pass: "5hSjsPbzRv7gEUsfzx",
},
connectionTimeout: 1000, // must be less than the jest default of 5000
},

40
packages/worker/src/utilities/email.ts
View File

@ -4,8 +4,10 @@ import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
import { getSettingsTemplateContext } from "./templates"
import { processString } from "@budibase/string-templates"
import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
import { configs, cache } from "@budibase/backend-core"
import { configs, cache, objectStore } from "@budibase/backend-core"
import ical from "ical-generator"
import fetch from "node-fetch"
import path from "path"
const nodemailer = require("nodemailer")
@ -162,6 +164,42 @@ export async function sendEmail(
contents: opts?.contents,
}),
}
if (opts?.attachments) {
const attachments = await Promise.all(
opts.attachments?.map(async attachment => {
const isFullyFormedUrl =
attachment.url.startsWith("http://") ||
attachment.url.startsWith("https://")
if (isFullyFormedUrl) {
const response = await fetch(attachment.url)
if (!response.ok) {
throw new Error(`unexpected response ${response.statusText}`)
}
const fallbackFilename = path.basename(
new URL(attachment.url).pathname
)
return {
filename: attachment.filename || fallbackFilename,
content: response?.body,
}
} else {
const url = attachment.url
const result = objectStore.extractBucketAndPath(url)
if (result === null) {
throw new Error("Invalid signed URL")
}
const { bucket, path } = result
const readStream = await objectStore.getReadStream(bucket, path)
const fallbackFilename = path.split("/").pop() || ""
return {
filename: attachment.filename || fallbackFilename,
content: readStream,
}
}
})
)
message = { ...message, attachments }
}
message = {
...message,

19
yarn.lock
View File

@ -6147,6 +6147,13 @@
dependencies:
"@types/webidl-conversions" "*"
"@types/xml2js@^0.4.14":
version "0.4.14"
resolved "https://registry.yarnpkg.com/@types/xml2js/-/xml2js-0.4.14.tgz#5d462a2a7330345e2309c6b549a183a376de8f9a"
integrity sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ==
dependencies:
"@types/node" "*"
"@types/yargs-parser@*":
version "21.0.0"
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
@ -7607,6 +7614,16 @@ bl@^4.0.3, bl@^4.1.0:
inherits "^2.0.4"
readable-stream "^3.4.0"
bl@^6.0.12:
version "6.0.12"
resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
dependencies:
"@types/readable-stream" "^4.0.0"
buffer "^6.0.3"
inherits "^2.0.4"
readable-stream "^4.2.0"
bl@^6.0.3:
version "6.0.9"
resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.9.tgz#df8fcb2ef7be2e5ee8f65afa493502914e0d816f"
@ -8781,7 +8798,7 @@ consolidate@^0.16.0:
dependencies:
bluebird "^3.7.2"
content-disposition@^0.5.2, content-disposition@^0.5.3, content-disposition@~0.5.2:
content-disposition@^0.5.2, content-disposition@^0.5.3, content-disposition@^0.5.4, content-disposition@~0.5.2:
version "0.5.4"
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe"
integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==