stream file upload instead

packages/backend-core/src/objectStore/objectStore.ts

@@ -205,10 +205,18 @@ export async function streamUpload(
   bucketName: string,
   filename: string,
   stream: ReadStream | ReadableStream,
+  addTTL?: boolean,
+  type?: string,
   extra = {}
 ) {
+  const extension = filename.split(".").pop()
   const objectStore = ObjectStore(bucketName)
-  await createBucketIfNotExists(objectStore, bucketName)
+  const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
+
+  if (addTTL && bucketCreated) {
+    let ttlConfig = bucketTTLConfig(bucketName, 1)
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+  }
 
   // Set content type for certain known extensions
   if (filename?.endsWith(".js")) {
@@ -222,10 +230,19 @@ export async function streamUpload(
       ContentType: "image",
     }
   }
+
+  let contentType = type
+  if (!contentType) {
+    contentType = extension
+      ? CONTENT_TYPE_MAP[extension.toLowerCase()]
+      : CONTENT_TYPE_MAP.txt
+  }
+
   const params = {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filename),
     Body: stream,
+    ContentType: contentType,
     ...extra,
   }
   return objectStore.upload(params).promise()

packages/server/package.json

@@ -65,6 +65,7 @@
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bl": "^6.0.12",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
     "content-disposition": "^0.5.4",

packages/server/src/integrations/utils.ts

@@ -17,6 +17,7 @@ import { context, objectStore } from "@budibase/backend-core"
 import { v4 } from "uuid"
 import { parseStringPromise as xmlParser } from "xml2js"
 import { formatBytes } from "../utilities"
+import bl from "bl"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -468,7 +469,6 @@ export async function handleFileResponse(
   startTime: number
 ) {
   let presignedUrl
-  const responseBuffer = await response.arrayBuffer()
   const fileExtension = filename.includes(".")
     ? filename.split(".").slice(1).join(".")
     : ""
@@ -477,17 +477,20 @@ export async function handleFileResponse(
   const key = `${context.getProdAppId()}/${processedFileName}`
   const bucket = objectStore.ObjectStoreBuckets.TEMP
 
-  await objectStore.upload({
-    bucket: bucket,
-    filename: key,
-    body: Buffer.from(responseBuffer),
-    addTTL: true,
-  })
-
+  const data = response.body.pipe(bl((error, data) => data))
+  if (response.body) {
+    await objectStore.streamUpload(
+      bucket,
+      key,
+      data,
+      true,
+      response.headers["content-type"]
+    )
+  }
   presignedUrl = await objectStore.getPresignedUrl(bucket, key, 600)
   return {
     data: {
-      size: responseBuffer.byteLength,
+      size: data.byteLength,
       name: processedFileName,
       url: presignedUrl,
       extension: fileExtension,
@@ -495,7 +498,7 @@ export async function handleFileResponse(
     },
     info: {
       code: response.status,
-      size: formatBytes(responseBuffer.byteLength),
+      size: formatBytes(data.byteLength),
       time: `${Math.round(performance.now() - startTime)}ms`,
     },
   }

yarn.lock

@@ -5896,6 +5896,14 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
   integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
 
+"@types/readable-stream@^4.0.0":
+  version "4.0.10"
+  resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-4.0.10.tgz#009bb80d5211619cdccbe0a4c075ae18ef52eab1"
+  integrity sha512-AbUKBjcC8SHmImNi4yK2bbjogQlkFSg7shZCcicxPQapniOlajG8GCc39lvXzCWX4lLRRs7DM3VAeSlqmEVZUA==
+  dependencies:
+    "@types/node" "*"
+    safe-buffer "~5.1.1"
+
 "@types/redis@^2.8.0":
   version "2.8.32"
   resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
@@ -7426,6 +7434,16 @@ bl@^4.0.3, bl@^4.1.0:
     inherits "^2.0.4"
     readable-stream "^3.4.0"
 
+bl@^6.0.12:
+  version "6.0.12"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
+  integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
+  dependencies:
+    "@types/readable-stream" "^4.0.0"
+    buffer "^6.0.3"
+    inherits "^2.0.4"
+    readable-stream "^4.2.0"
+
 bl@^6.0.3:
   version "6.0.9"
   resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.9.tgz#df8fcb2ef7be2e5ee8f65afa493502914e0d816f"