Extract utils from controller

packages/server/src/api/controllers/ai.ts

@@ -21,6 +21,8 @@ import { pipeline } from "stream"
 import { promisify } from "util"
 import fetch from "node-fetch"
 import { ObjectStoreBuckets } from "../../constants"
+import { uploadUrl } from "../../utilities"
+import { uploadFile } from "../../utilities/fileUtils"
 
 async function generateTablesDelegate(
   tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
@@ -114,7 +116,12 @@ async function generateDataDelegate(
           entry[column.name] = [entry[column.name]]
         }
         for (const attachmentValue of entry[column.name]) {
-          const attachment = await downloadFile(attachmentValue)
+          let attachment
+          if (typeof attachmentValue === "object") {
+            attachment = await uploadUrl(attachmentValue)
+          } else {
+            attachment = await uploadFile(attachmentValue)
+          }
           if (attachment) {
             attachmentData[column.name].push(attachment)
           }
@@ -208,84 +215,3 @@ export async function generateTables(
     createdTables,
   }
 }
-
-async function downloadFile(
-  file: string | { fileName: string; extension: string; content: string }
-): Promise<Upload | undefined> {
-  if (typeof file === "object") {
-    return createFile(file)
-  }
-
-  try {
-    const res = await fetch(file)
-
-    const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
-
-    if (!fs.existsSync(tmpPath)) {
-      mkdirSync(tmpPath)
-    }
-
-    const extension = [...res.url.split(".")].pop()!.split("?")[0]
-
-    const destination = path.resolve(tmpPath, `${uuid.v4()}${extension}`)
-    const fileStream = fs.createWriteStream(destination, { flags: "wx" })
-
-    await promisify(pipeline)(res.body, fileStream)
-
-    const processedFileName = path.basename(destination)
-
-    const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
-
-    const response = await objectStore.upload({
-      bucket: ObjectStoreBuckets.APPS,
-      filename: s3Key,
-      path: destination,
-      type: "image/jpeg",
-    })
-
-    return {
-      size: fileStream.bytesWritten,
-      name: processedFileName,
-      url: await objectStore.getAppFileUrl(s3Key),
-      extension,
-      key: response.Key!,
-    }
-  } catch (e) {
-    console.error("Error downloading file", e)
-    return
-  }
-}
-
-async function createFile(file: {
-  fileName: string
-  extension: string
-  content: string
-}): Promise<Upload> {
-  const tmpPath = join(objectStore.budibaseTempDir(), "ai-downloads")
-
-  if (!fs.existsSync(tmpPath)) {
-    mkdirSync(tmpPath)
-  }
-
-  const destination = path.resolve(tmpPath, `${file.fileName}${file.extension}`)
-
-  fs.writeFileSync(destination, file.content)
-
-  const processedFileName = path.basename(destination)
-  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
-
-  const response = await objectStore.upload({
-    bucket: ObjectStoreBuckets.APPS,
-    filename: s3Key,
-    path: destination,
-    type: "text/plain",
-  })
-
-  return {
-    size: fs.readFileSync(destination).byteLength,
-    name: processedFileName,
-    url: await objectStore.getAppFileUrl(s3Key),
-    extension: file.extension,
-    key: response.Key!,
-  }
-}

packages/server/src/utilities/fileUtils.ts

@@ -0,0 +1,85 @@
+import fs from "fs"
+import path from "path"
+import { pipeline } from "stream"
+import { promisify } from "util"
+import * as uuid from "uuid"
+import fetch from "node-fetch"
+
+import { context, objectStore } from "@budibase/backend-core"
+import { Upload } from "@budibase/types"
+import { ObjectStoreBuckets } from "../constants"
+
+function getTmpPath() {
+  const tmpPath = path.join(objectStore.budibaseTempDir(), "ai-downloads")
+  if (!fs.existsSync(tmpPath)) {
+    fs.mkdirSync(tmpPath)
+  }
+
+  return tmpPath
+}
+
+export async function uploadUrl(url: string): Promise<Upload | undefined> {
+  try {
+    const res = await fetch(url)
+
+    const extension = [...res.url.split(".")].pop()!.split("?")[0]
+
+    const destination = path.resolve(getTmpPath(), `${uuid.v4()}${extension}`)
+    const fileStream = fs.createWriteStream(destination, { flags: "wx" })
+
+    await promisify(pipeline)(res.body, fileStream)
+
+    const processedFileName = path.basename(destination)
+
+    const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+    const response = await objectStore.upload({
+      bucket: ObjectStoreBuckets.APPS,
+      filename: s3Key,
+      path: destination,
+      type: "image/jpeg",
+    })
+
+    return {
+      size: fileStream.bytesWritten,
+      name: processedFileName,
+      url: await objectStore.getAppFileUrl(s3Key),
+      extension,
+      key: response.Key!,
+    }
+  } catch (e) {
+    console.error("Error downloading file", e)
+    return
+  }
+}
+
+export async function uploadFile(file: {
+  fileName: string
+  extension: string
+  content: string
+}): Promise<Upload> {
+  const destination = path.resolve(
+    getTmpPath(),
+    `${file.fileName}${file.extension}`
+  )
+
+  fs.writeFileSync(destination, file.content)
+
+  const processedFileName = path.basename(destination)
+  const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+  const response = await objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: s3Key,
+    path: destination,
+    type: "text/plain",
+  })
+
+  return {
+    size: fs.readFileSync(destination).byteLength,
+    name: processedFileName,
+    url: await objectStore.getAppFileUrl(s3Key),
+    extension: file.extension,
+    key: response.Key!,
+  }
+}

packages/server/src/utilities/index.ts

@@ -1,3 +1,4 @@
+export * from "./fileUtils"
 import env from "../environment"
 import { context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"