use streamUpload and update signatures

packages/backend-core/src/objectStore/objectStore.ts

@@ -13,6 +13,7 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
+import { add } from "lodash"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -35,6 +36,21 @@ type UploadParams = {
   }
   body?: ReadableStream | Buffer
   addTTL?: boolean
+  extra?: any
+}
+
+type StreamUploadParams = {
+  bucket: string
+  filename: string
+  stream: ReadStream
+  type?: string | null
+  // can be undefined, we will remove it
+  metadata?: {
+    [key: string]: string | undefined
+  }
+  body?: ReadableStream | Buffer
+  addTTL?: boolean
+  extra?: any
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -201,14 +217,14 @@ export async function upload({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export async function streamUpload(
+export async function streamUpload({
-  bucketName: string,
+  bucket: bucketName,
-  filename: string,
+  stream,
-  stream: ReadStream | ReadableStream,
+  filename,
-  addTTL?: boolean,
+  type,
-  type?: string,
+  extra,
-  extra = {}
+  addTTL,
-) {
+}: StreamUploadParams) {
   const extension = filename.split(".").pop()
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
@@ -448,7 +464,13 @@ export async function uploadDirectory(
     if (file.isDirectory()) {
       uploads.push(uploadDirectory(bucketName, local, path))
     } else {
-      uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
+      uploads.push(
+        streamUpload({
+          bucket: bucketName,
+          filename: path,
+          stream: fs.createReadStream(local),
+        })
+      )
     }
   }
   await Promise.all(uploads)

packages/pro

@@ -1 +1 @@
-Subproject commit c4c98ae70f2e936009250893898ecf11f4ddf2c3
+Subproject commit 9821b8235824f02c581ce9dc3eb6ca42fd771219

packages/server/src/integrations/utils.ts

@@ -477,20 +477,20 @@ export async function handleFileResponse(
   const key = `${context.getProdAppId()}/${processedFileName}`
   const bucket = objectStore.ObjectStoreBuckets.TEMP
 
-  const data = response.body.pipe(bl((error, data) => data))
+  const stream = response.body.pipe(bl((error, data) => data))
   if (response.body) {
-    await objectStore.streamUpload(
+    await objectStore.streamUpload({
       bucket,
-      key,
+      filename: key,
-      data,
+      stream,
-      true,
+      addTTL: true,
-      response.headers["content-type"]
+      type: response.headers["content-type"],
-    )
+    })
   }
   presignedUrl = await objectStore.getPresignedUrl(bucket, key, 600)
   return {
     data: {
-      size: data.byteLength,
+      size: stream.byteLength,
       name: processedFileName,
       url: presignedUrl,
       extension: fileExtension,
@@ -498,7 +498,7 @@ export async function handleFileResponse(
     },
     info: {
       code: response.status,
-      size: formatBytes(data.byteLength),
+      size: formatBytes(stream.byteLength),
       time: `${Math.round(performance.now() - startTime)}ms`,
     },
   }

packages/server/src/utilities/fileSystem/clientLibrary.ts

@@ -104,22 +104,22 @@ export async function updateClientLibrary(appId: string) {
   }
 
   // Upload latest manifest and client library
-  const manifestUpload = objectStore.streamUpload(
+  const manifestUpload = objectStore.streamUpload({
-    ObjectStoreBuckets.APPS,
+    bucket: ObjectStoreBuckets.APPS,
-    join(appId, "manifest.json"),
+    filename: join(appId, "manifest.json"),
-    fs.createReadStream(manifest),
+    stream: fs.createReadStream(manifest),
-    {
+    extra: {
       ContentType: "application/json",
-    }
+    },
-  )
+  })
-  const clientUpload = objectStore.streamUpload(
+  const clientUpload = objectStore.streamUpload({
-    ObjectStoreBuckets.APPS,
+    bucket: ObjectStoreBuckets.APPS,
-    join(appId, "budibase-client.js"),
+    filename: join(appId, "budibase-client.js"),
-    fs.createReadStream(client),
+    stream: fs.createReadStream(client),
-    {
+    extra: {
       ContentType: "application/javascript",
-    }
+    },
-  )
+  })
   await Promise.all([manifestUpload, clientUpload])
 }