diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index ec16e056eb..33e353646d 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -205,10 +205,18 @@ export async function streamUpload( bucketName: string, filename: string, stream: ReadStream | ReadableStream, + addTTL?: boolean, + type?: string, extra = {} ) { + const extension = filename.split(".").pop() const objectStore = ObjectStore(bucketName) - await createBucketIfNotExists(objectStore, bucketName) + const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) + + if (addTTL && bucketCreated) { + let ttlConfig = bucketTTLConfig(bucketName, 1) + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() + } // Set content type for certain known extensions if (filename?.endsWith(".js")) { @@ -222,10 +230,19 @@ export async function streamUpload( ContentType: "image", } } + + let contentType = type + if (!contentType) { + contentType = extension + ? CONTENT_TYPE_MAP[extension.toLowerCase()] + : CONTENT_TYPE_MAP.txt + } + const params = { Bucket: sanitizeBucket(bucketName), Key: sanitizeKey(filename), Body: stream, + ContentType: contentType, ...extra, } return objectStore.upload(params).promise() diff --git a/packages/server/package.json b/packages/server/package.json index a72852b9ba..27c46fa02d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -65,6 +65,7 @@ "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", + "bl": "^6.0.12", "bull": "4.10.1", "chokidar": "3.5.3", "content-disposition": "^0.5.4", diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 0d97c376ea..7beee5a227 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -17,6 +17,7 @@ import { context, objectStore } from "@budibase/backend-core" import { v4 } from "uuid" import { parseStringPromise as xmlParser } from "xml2js" import { formatBytes } from "../utilities" +import bl from "bl" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g @@ -468,7 +469,6 @@ export async function handleFileResponse( startTime: number ) { let presignedUrl - const responseBuffer = await response.arrayBuffer() const fileExtension = filename.includes(".") ? filename.split(".").slice(1).join(".") : "" @@ -477,17 +477,20 @@ export async function handleFileResponse( const key = `${context.getProdAppId()}/${processedFileName}` const bucket = objectStore.ObjectStoreBuckets.TEMP - await objectStore.upload({ - bucket: bucket, - filename: key, - body: Buffer.from(responseBuffer), - addTTL: true, - }) - + const data = response.body.pipe(bl((error, data) => data)) + if (response.body) { + await objectStore.streamUpload( + bucket, + key, + data, + true, + response.headers["content-type"] + ) + } presignedUrl = await objectStore.getPresignedUrl(bucket, key, 600) return { data: { - size: responseBuffer.byteLength, + size: data.byteLength, name: processedFileName, url: presignedUrl, extension: fileExtension, @@ -495,7 +498,7 @@ export async function handleFileResponse( }, info: { code: response.status, - size: formatBytes(responseBuffer.byteLength), + size: formatBytes(data.byteLength), time: `${Math.round(performance.now() - startTime)}ms`, }, } diff --git a/yarn.lock b/yarn.lock index 6f3df1f87d..05392771d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5896,6 +5896,14 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== +"@types/readable-stream@^4.0.0": + version "4.0.10" + resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-4.0.10.tgz#009bb80d5211619cdccbe0a4c075ae18ef52eab1" + integrity sha512-AbUKBjcC8SHmImNi4yK2bbjogQlkFSg7shZCcicxPQapniOlajG8GCc39lvXzCWX4lLRRs7DM3VAeSlqmEVZUA== + dependencies: + "@types/node" "*" + safe-buffer "~5.1.1" + "@types/redis@^2.8.0": version "2.8.32" resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11" @@ -7426,6 +7434,16 @@ bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" +bl@^6.0.12: + version "6.0.12" + resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8" + integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w== + dependencies: + "@types/readable-stream" "^4.0.0" + buffer "^6.0.3" + inherits "^2.0.4" + readable-stream "^4.2.0" + bl@^6.0.3: version "6.0.9" resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.9.tgz#df8fcb2ef7be2e5ee8f65afa493502914e0d816f"