Revert "Revert "Client library fetching through server""

packages/backend-core/src/objectStore/buckets/app.ts

@@ -1,37 +1,50 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
+import qs from "querystring"
+import { DEFAULT_TENANT_ID, getTenantId } from "../../context"
+
+export function clientLibraryPath(appId: string) {
+  return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+}
 
 /**
- * In production the client library is stored in the object store, however in development
+ * Previously we used to serve the client library directly from Cloudfront, however
- * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * due to issues with the domain we were unable to continue doing this - keeping
- * via a specific endpoint (under /api/assets/client).
+ * incase we are able to switch back to CDN path again in future.
- * @param appId In production we need the appId to look up the correct bucket, as the
- * version of the client lib may differ between apps.
- * @param version The version to retrieve.
- * @return The URL to be inserted into appPackage response or server rendered
- * app index file.
  */
-export const clientLibraryUrl = (appId: string, version: string) => {
+export function clientLibraryCDNUrl(appId: string, version: string) {
-  if (env.isProd()) {
+  let file = clientLibraryPath(appId)
-    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+  if (env.CLOUDFRONT_CDN) {
-    if (env.CLOUDFRONT_CDN) {
+    // append app version to bust the cache
-      // append app version to bust the cache
+    if (version) {
-      if (version) {
+      file += `?v=${version}`
-        file += `?v=${version}`
-      }
-      // don't need to use presigned for client with cloudfront
-      // file is public
-      return cloudfront.getUrl(file)
-    } else {
-      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
     }
+    // don't need to use presigned for client with cloudfront
+    // file is public
+    return cloudfront.getUrl(file)
   } else {
-    return `/api/assets/client`
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
   }
 }
 
-export const getAppFileUrl = (s3Key: string) => {
+export function clientLibraryUrl(appId: string, version: string) {
+  let tenantId, qsParams: { appId: string; version: string; tenantId?: string }
+  try {
+    tenantId = getTenantId()
+  } finally {
+    qsParams = {
+      appId,
+      version,
+    }
+  }
+  if (tenantId && tenantId !== DEFAULT_TENANT_ID) {
+    qsParams.tenantId = tenantId
+  }
+  return `/api/assets/client?${qs.encode(qsParams)}`
+}
+
+export function getAppFileUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export const enrichPluginURLs = (plugins: Plugin[]) => {
+export function enrichPluginURLs(plugins: Plugin[]) {
   if (!plugins || !plugins.length) {
     return []
   }
@@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => {
   })
 }
 
-const getPluginJSUrl = (plugin: Plugin) => {
+function getPluginJSUrl(plugin: Plugin) {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+function getPluginIconUrl(plugin: Plugin): string | undefined {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
@@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => {
   return getPluginUrl(s3Key)
 }
 
-const getPluginUrl = (s3Key: string) => {
+function getPluginUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
@@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => {
 
 // S3 KEYS
 
-export const getPluginJSKey = (plugin: Plugin) => {
+export function getPluginJSKey(plugin: Plugin) {
   return getPluginS3Key(plugin, "plugin.min.js")
 }
 
-export const getPluginIconKey = (plugin: Plugin) => {
+export function getPluginIconKey(plugin: Plugin) {
   // stored iconUrl is deprecated - hardcode to icon.svg in this case
   const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
   if (!iconFileName) {
@@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => {
   return getPluginS3Key(plugin, iconFileName)
 }
 
-const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+function getPluginS3Key(plugin: Plugin, fileName: string) {
   const s3Key = getPluginS3Dir(plugin.name)
   return `${s3Key}/${fileName}`
 }
 
-export const getPluginS3Dir = (pluginName: string) => {
+export function getPluginS3Dir(pluginName: string) {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
     const tenantId = context.getTenantId()

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -1,5 +1,4 @@
 import * as app from "../app"
-import { getAppFileUrl } from "../app"
 import { testEnv } from "../../../../tests/extra"
 
 describe("app", () => {
@@ -7,6 +6,15 @@ describe("app", () => {
     testEnv.nodeJest()
   })
 
+  function baseCheck(url: string, tenantId?: string) {
+    expect(url).toContain("/api/assets/client")
+    if (tenantId) {
+      expect(url).toContain(`tenantId=${tenantId}`)
+    }
+    expect(url).toContain("appId=app_123")
+    expect(url).toContain("version=2.0.0")
+  }
+
   describe("clientLibraryUrl", () => {
     function getClientUrl() {
       return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
@@ -20,31 +28,19 @@ describe("app", () => {
       it("gets url in dev", () => {
         testEnv.nodeDev()
         const url = getClientUrl()
-        expect(url).toBe("/api/assets/client")
+        baseCheck(url)
-      })
-
-      it("gets url with embedded minio", () => {
-        testEnv.withMinio()
-        const url = getClientUrl()
-        expect(url).toBe(
-          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
       })
 
       it("gets url with custom S3", () => {
         testEnv.withS3()
         const url = getClientUrl()
-        expect(url).toBe(
+        baseCheck(url)
-          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
       })
 
       it("gets url with cloudfront + s3", () => {
         testEnv.withCloudfront()
         const url = getClientUrl()
-        expect(url).toBe(
+        baseCheck(url)
-          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-        )
       })
     })
 
@@ -57,7 +53,7 @@ describe("app", () => {
         testEnv.nodeDev()
         await testEnv.withTenant(tenantId => {
           const url = getClientUrl()
-          expect(url).toBe("/api/assets/client")
+          baseCheck(url, tenantId)
         })
       })
 
@@ -65,9 +61,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withMinio()
           const url = getClientUrl()
-          expect(url).toBe(
+          baseCheck(url, tenantId)
-            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
         })
       })
 
@@ -75,9 +69,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withS3()
           const url = getClientUrl()
-          expect(url).toBe(
+          baseCheck(url, tenantId)
-            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
         })
       })
 
@@ -85,9 +77,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withCloudfront()
           const url = getClientUrl()
-          expect(url).toBe(
+          baseCheck(url, tenantId)
-            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-          )
         })
       })
     })

packages/backend-core/src/objectStore/objectStore.ts

@@ -1,6 +1,6 @@
 const sanitize = require("sanitize-s3-objectkey")
 import AWS from "aws-sdk"
-import stream from "stream"
+import stream, { Readable } from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
 import zlib from "zlib"
@@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) {
  * @return an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (
+export function ObjectStore(
   bucket: string,
   opts: { presigning: boolean } = { presigning: false }
-) => {
+) {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -104,7 +104,7 @@ export const ObjectStore = (
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export const makeSureBucketExists = async (client: any, bucketName: string) => {
+export async function makeSureBucketExists(client: any, bucketName: string) {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-export const upload = async ({
+export async function upload({
   bucket: bucketName,
   filename,
   path,
   type,
   metadata,
-}: UploadParams) => {
+}: UploadParams) {
   const extension = filename.split(".").pop()
   const fileBytes = fs.readFileSync(path)
 
@@ -180,12 +180,12 @@ export const upload = async ({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export const streamUpload = async (
+export async function streamUpload(
   bucketName: string,
   filename: string,
   stream: any,
   extra = {}
-) => {
+) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
 
@@ -215,7 +215,7 @@ export const streamUpload = async (
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export const retrieve = async (bucketName: string, filepath: string) => {
+export async function retrieve(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => {
   }
 }
 
-export const listAllObjects = async (bucketName: string, path: string) => {
+export async function listAllObjects(bucketName: string, path: string) {
   const objectStore = ObjectStore(bucketName)
   const list = (params: ListParams = {}) => {
     return objectStore
@@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => {
 /**
  * Generate a presigned url with a default TTL of 1 hour
  */
-export const getPresignedUrl = (
+export function getPresignedUrl(
   bucketName: string,
   key: string,
   durationSeconds: number = 3600
-) => {
+) {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -291,7 +291,7 @@ export const getPresignedUrl = (
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-export const retrieveToTmp = async (bucketName: string, filepath: string) => {
+export async function retrieveToTmp(bucketName: string, filepath: string) {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
@@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => {
   return outputPath
 }
 
-export const retrieveDirectory = async (bucketName: string, path: string) => {
+export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
@@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => {
 /**
  * Delete a single file.
  */
-export const deleteFile = async (bucketName: string, filepath: string) => {
+export async function deleteFile(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   return objectStore.deleteObject(params).promise()
 }
 
-export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
+export async function deleteFiles(bucketName: string, filepaths: string[]) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
 /**
  * Delete a path, including everything within.
  */
-export const deleteFolder = async (
+export async function deleteFolder(
   bucketName: string,
   folder: string
-): Promise<any> => {
+): Promise<any> {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
   const client = ObjectStore(bucketName)
@@ -383,11 +383,11 @@ export const deleteFolder = async (
   }
 }
 
-export const uploadDirectory = async (
+export async function uploadDirectory(
   bucketName: string,
   localPath: string,
   bucketPath: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
@@ -404,11 +404,11 @@ export const uploadDirectory = async (
   return files
 }
 
-export const downloadTarballDirect = async (
+export async function downloadTarballDirect(
   url: string,
   path: string,
   headers = {}
-) => {
+) {
   path = sanitizeKey(path)
   const response = await fetch(url, { headers })
   if (!response.ok) {
@@ -418,11 +418,11 @@ export const downloadTarballDirect = async (
   await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
 }
 
-export const downloadTarball = async (
+export async function downloadTarball(
   url: string,
   bucketName: string,
   path: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
@@ -438,3 +438,17 @@ export const downloadTarball = async (
   // return the temporary path incase there is a use for it
   return tmpPath
 }
+
+export async function getReadStream(
+  bucketName: string,
+  path: string
+): Promise<Readable> {
+  bucketName = sanitizeBucket(bucketName)
+  path = sanitizeKey(path)
+  const client = ObjectStore(bucketName)
+  const params = {
+    Bucket: bucketName,
+    Key: path,
+  }
+  return client.getObject(params).createReadStream()
+}

packages/server/src/api/controllers/static/index.ts

@@ -16,7 +16,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App } from "@budibase/types"
+import { App, Ctx } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   }
 }
 
-export const toggleBetaUiFeature = async function (ctx: any) {
+export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
   if (ctx.cookies.get(cookieName)) {
@@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) {
   }
 }
 
-export const serveBuilder = async function (ctx: any) {
+export const serveBuilder = async function (ctx: Ctx) {
   const builderPath = join(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: any) {
+export const uploadFile = async function (ctx: Ctx) {
-  let files =
+  const file = ctx.request?.files?.file
-    ctx.request.files.file.length > 1
+  let files = file && Array.isArray(file) ? Array.from(file) : [file]
-      ? Array.from(ctx.request.files.file)
-      : [ctx.request.files.file]
 
   const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
@@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) {
   ctx.body = await Promise.all(uploads)
 }
 
-export const deleteObjects = async function (ctx: any) {
+export const deleteObjects = async function (ctx: Ctx) {
   ctx.body = await objectStore.deleteFiles(
     ObjectStoreBuckets.APPS,
     ctx.request.body.keys
   )
 }
 
-export const serveApp = async function (ctx: any) {
+export const serveApp = async function (ctx: Ctx) {
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) {
   }
 }
 
-export const serveBuilderPreview = async function (ctx: any) {
+export const serveBuilderPreview = async function (ctx: Ctx) {
   const db = context.getAppDB({ skip_setup: true })
   const appInfo = await db.get<App>(DocumentType.APP_METADATA)
 
@@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) {
   }
 }
 
-export const serveClientLibrary = async function (ctx: any) {
+export const serveClientLibrary = async function (ctx: Ctx) {
+  const appId = context.getAppId() || (ctx.request.query.appId as string)
   let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
-  // incase running from TS directly
+  if (!appId) {
-  if (env.isDev() && !fs.existsSync(rootPath)) {
+    ctx.throw(400, "No app ID provided - cannot fetch client library.")
-    rootPath = join(require.resolve("@budibase/client"), "..")
+  }
+  if (env.isProd()) {
+    ctx.body = await objectStore.getReadStream(
+      ObjectStoreBuckets.APPS,
+      objectStore.clientLibraryPath(appId!)
+    )
+  } else if (env.isDev()) {
+    // incase running from TS directly
+    const tsPath = join(require.resolve("@budibase/client"), "..")
+    return send(ctx, "budibase-client.js", {
+      root: !fs.existsSync(rootPath) ? tsPath : rootPath,
+    })
+  } else {
+    ctx.throw(500, "Unable to retrieve client library.")
   }
-  return send(ctx, "budibase-client.js", {
-    root: rootPath,
-  })
 }
 
-export const getSignedUploadURL = async function (ctx: any) {
+export const getSignedUploadURL = async function (ctx: Ctx) {
   // Ensure datasource is valid
   let datasource
   try {
@@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) {
       const params = { Bucket: bucket, Key: key }
       signedUrl = s3.getSignedUrl("putObject", params)
       publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
-    } catch (error) {
+    } catch (error: any) {
       ctx.throw(400, error)
     }
   }