From c6b377c5659067bb220006d94abfacc18a2b575b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 24 Oct 2023 18:13:36 +0100 Subject: [PATCH] Revert "Revert "Client library fetching through server"" --- .../src/objectStore/buckets/app.ts | 59 +++++++++++------- .../src/objectStore/buckets/plugins.ts | 16 ++--- .../src/objectStore/buckets/tests/app.spec.ts | 42 +++++-------- .../src/objectStore/objectStore.ts | 62 ++++++++++++------- .../src/api/controllers/static/index.ts | 49 +++++++++------ 5 files changed, 127 insertions(+), 101 deletions(-) diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index be9fddeaa6..43bc965c65 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -1,37 +1,50 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" +import qs from "querystring" +import { DEFAULT_TENANT_ID, getTenantId } from "../../context" + +export function clientLibraryPath(appId: string) { + return `${objectStore.sanitizeKey(appId)}/budibase-client.js` +} /** - * In production the client library is stored in the object store, however in development - * we use the symlinked version produced by lerna, located in node modules. We link to this - * via a specific endpoint (under /api/assets/client). - * @param appId In production we need the appId to look up the correct bucket, as the - * version of the client lib may differ between apps. - * @param version The version to retrieve. - * @return The URL to be inserted into appPackage response or server rendered - * app index file. + * Previously we used to serve the client library directly from Cloudfront, however + * due to issues with the domain we were unable to continue doing this - keeping + * incase we are able to switch back to CDN path again in future. */ -export const clientLibraryUrl = (appId: string, version: string) => { - if (env.isProd()) { - let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js` - if (env.CLOUDFRONT_CDN) { - // append app version to bust the cache - if (version) { - file += `?v=${version}` - } - // don't need to use presigned for client with cloudfront - // file is public - return cloudfront.getUrl(file) - } else { - return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) +export function clientLibraryCDNUrl(appId: string, version: string) { + let file = clientLibraryPath(appId) + if (env.CLOUDFRONT_CDN) { + // append app version to bust the cache + if (version) { + file += `?v=${version}` } + // don't need to use presigned for client with cloudfront + // file is public + return cloudfront.getUrl(file) } else { - return `/api/assets/client` + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) } } -export const getAppFileUrl = (s3Key: string) => { +export function clientLibraryUrl(appId: string, version: string) { + let tenantId, qsParams: { appId: string; version: string; tenantId?: string } + try { + tenantId = getTenantId() + } finally { + qsParams = { + appId, + version, + } + } + if (tenantId && tenantId !== DEFAULT_TENANT_ID) { + qsParams.tenantId = tenantId + } + return `/api/assets/client?${qs.encode(qsParams)}` +} + +export function getAppFileUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts index f7721afb23..6f1b7116ae 100644 --- a/packages/backend-core/src/objectStore/buckets/plugins.ts +++ b/packages/backend-core/src/objectStore/buckets/plugins.ts @@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types" // URLS -export const enrichPluginURLs = (plugins: Plugin[]) => { +export function enrichPluginURLs(plugins: Plugin[]) { if (!plugins || !plugins.length) { return [] } @@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => { }) } -const getPluginJSUrl = (plugin: Plugin) => { +function getPluginJSUrl(plugin: Plugin) { const s3Key = getPluginJSKey(plugin) return getPluginUrl(s3Key) } -const getPluginIconUrl = (plugin: Plugin): string | undefined => { +function getPluginIconUrl(plugin: Plugin): string | undefined { const s3Key = getPluginIconKey(plugin) if (!s3Key) { return @@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => { return getPluginUrl(s3Key) } -const getPluginUrl = (s3Key: string) => { +function getPluginUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { @@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => { // S3 KEYS -export const getPluginJSKey = (plugin: Plugin) => { +export function getPluginJSKey(plugin: Plugin) { return getPluginS3Key(plugin, "plugin.min.js") } -export const getPluginIconKey = (plugin: Plugin) => { +export function getPluginIconKey(plugin: Plugin) { // stored iconUrl is deprecated - hardcode to icon.svg in this case const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName if (!iconFileName) { @@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => { return getPluginS3Key(plugin, iconFileName) } -const getPluginS3Key = (plugin: Plugin, fileName: string) => { +function getPluginS3Key(plugin: Plugin, fileName: string) { const s3Key = getPluginS3Dir(plugin.name) return `${s3Key}/${fileName}` } -export const getPluginS3Dir = (pluginName: string) => { +export function getPluginS3Dir(pluginName: string) { let s3Key = `${pluginName}` if (env.MULTI_TENANCY) { const tenantId = context.getTenantId() diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts index aaa07ec9d3..cbbbee6255 100644 --- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -1,5 +1,4 @@ import * as app from "../app" -import { getAppFileUrl } from "../app" import { testEnv } from "../../../../tests/extra" describe("app", () => { @@ -7,6 +6,15 @@ describe("app", () => { testEnv.nodeJest() }) + function baseCheck(url: string, tenantId?: string) { + expect(url).toContain("/api/assets/client") + if (tenantId) { + expect(url).toContain(`tenantId=${tenantId}`) + } + expect(url).toContain("appId=app_123") + expect(url).toContain("version=2.0.0") + } + describe("clientLibraryUrl", () => { function getClientUrl() { return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0") @@ -20,31 +28,19 @@ describe("app", () => { it("gets url in dev", () => { testEnv.nodeDev() const url = getClientUrl() - expect(url).toBe("/api/assets/client") - }) - - it("gets url with embedded minio", () => { - testEnv.withMinio() - const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with custom S3", () => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with cloudfront + s3", () => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url) }) }) @@ -57,7 +53,7 @@ describe("app", () => { testEnv.nodeDev() await testEnv.withTenant(tenantId => { const url = getClientUrl() - expect(url).toBe("/api/assets/client") + baseCheck(url, tenantId) }) }) @@ -65,9 +61,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withMinio() const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -75,9 +69,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -85,9 +77,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url, tenantId) }) }) }) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index c36a09915e..76d2dd6689 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -1,6 +1,6 @@ const sanitize = require("sanitize-s3-objectkey") import AWS from "aws-sdk" -import stream from "stream" +import stream, { Readable } from "stream" import fetch from "node-fetch" import tar from "tar-fs" import zlib from "zlib" @@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) { * @return an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = ( +export function ObjectStore( bucket: string, opts: { presigning: boolean } = { presigning: false } -) => { +) { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -104,7 +104,7 @@ export const ObjectStore = ( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: string) => { +export async function makeSureBucketExists(client: any, bucketName: string) { bucketName = sanitizeBucket(bucketName) try { await client @@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => { * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). */ -export const upload = async ({ +export async function upload({ bucket: bucketName, filename, path, type, metadata, -}: UploadParams) => { +}: UploadParams) { const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) @@ -180,12 +180,12 @@ export const upload = async ({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export const streamUpload = async ( +export async function streamUpload( bucketName: string, filename: string, stream: any, extra = {} -) => { +) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) @@ -215,7 +215,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: string, filepath: string) => { +export async function retrieve(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => { } } -export const listAllObjects = async (bucketName: string, path: string) => { +export async function listAllObjects(bucketName: string, path: string) { const objectStore = ObjectStore(bucketName) const list = (params: ListParams = {}) => { return objectStore @@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => { /** * Generate a presigned url with a default TTL of 1 hour */ -export const getPresignedUrl = ( +export function getPresignedUrl( bucketName: string, key: string, durationSeconds: number = 3600 -) => { +) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { Bucket: sanitizeBucket(bucketName), @@ -291,7 +291,7 @@ export const getPresignedUrl = ( /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: string, filepath: string) => { +export async function retrieveToTmp(bucketName: string, filepath: string) { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => { return outputPath } -export const retrieveDirectory = async (bucketName: string, path: string) => { +export async function retrieveDirectory(bucketName: string, path: string) { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) @@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => { /** * Delete a single file. */ -export const deleteFile = async (bucketName: string, filepath: string) => { +export async function deleteFile(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => { return objectStore.deleteObject(params).promise() } -export const deleteFiles = async (bucketName: string, filepaths: string[]) => { +export async function deleteFiles(bucketName: string, filepaths: string[]) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => { /** * Delete a path, including everything within. */ -export const deleteFolder = async ( +export async function deleteFolder( bucketName: string, folder: string -): Promise => { +): Promise { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) const client = ObjectStore(bucketName) @@ -383,11 +383,11 @@ export const deleteFolder = async ( } } -export const uploadDirectory = async ( +export async function uploadDirectory( bucketName: string, localPath: string, bucketPath: string -) => { +) { bucketName = sanitizeBucket(bucketName) let uploads = [] const files = fs.readdirSync(localPath, { withFileTypes: true }) @@ -404,11 +404,11 @@ export const uploadDirectory = async ( return files } -export const downloadTarballDirect = async ( +export async function downloadTarballDirect( url: string, path: string, headers = {} -) => { +) { path = sanitizeKey(path) const response = await fetch(url, { headers }) if (!response.ok) { @@ -418,11 +418,11 @@ export const downloadTarballDirect = async ( await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path)) } -export const downloadTarball = async ( +export async function downloadTarball( url: string, bucketName: string, path: string -) => { +) { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) @@ -438,3 +438,17 @@ export const downloadTarball = async ( // return the temporary path incase there is a use for it return tmpPath } + +export async function getReadStream( + bucketName: string, + path: string +): Promise { + bucketName = sanitizeBucket(bucketName) + path = sanitizeKey(path) + const client = ObjectStore(bucketName) + const params = { + Bucket: bucketName, + Key: path, + } + return client.getObject(params).createReadStream() +} diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index bbf9dd34f5..d2e887dcf8 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -16,7 +16,7 @@ import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" -import { App } from "@budibase/types" +import { App, Ctx } from "@budibase/types" const send = require("koa-send") @@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) { } } -export const toggleBetaUiFeature = async function (ctx: any) { +export const toggleBetaUiFeature = async function (ctx: Ctx) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) { } } -export const serveBuilder = async function (ctx: any) { +export const serveBuilder = async function (ctx: Ctx) { const builderPath = join(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) } -export const uploadFile = async function (ctx: any) { - let files = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] +export const uploadFile = async function (ctx: Ctx) { + const file = ctx.request?.files?.file + let files = file && Array.isArray(file) ? Array.from(file) : [file] const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() @@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) { ctx.body = await Promise.all(uploads) } -export const deleteObjects = async function (ctx: any) { +export const deleteObjects = async function (ctx: Ctx) { ctx.body = await objectStore.deleteFiles( ObjectStoreBuckets.APPS, ctx.request.body.keys ) } -export const serveApp = async function (ctx: any) { +export const serveApp = async function (ctx: Ctx) { const bbHeaderEmbed = ctx.request.get("x-budibase-embed")?.toLowerCase() === "true" @@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) { } } -export const serveBuilderPreview = async function (ctx: any) { +export const serveBuilderPreview = async function (ctx: Ctx) { const db = context.getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentType.APP_METADATA) @@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) { } } -export const serveClientLibrary = async function (ctx: any) { +export const serveClientLibrary = async function (ctx: Ctx) { + const appId = context.getAppId() || (ctx.request.query.appId as string) let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist") - // incase running from TS directly - if (env.isDev() && !fs.existsSync(rootPath)) { - rootPath = join(require.resolve("@budibase/client"), "..") + if (!appId) { + ctx.throw(400, "No app ID provided - cannot fetch client library.") + } + if (env.isProd()) { + ctx.body = await objectStore.getReadStream( + ObjectStoreBuckets.APPS, + objectStore.clientLibraryPath(appId!) + ) + } else if (env.isDev()) { + // incase running from TS directly + const tsPath = join(require.resolve("@budibase/client"), "..") + return send(ctx, "budibase-client.js", { + root: !fs.existsSync(rootPath) ? tsPath : rootPath, + }) + } else { + ctx.throw(500, "Unable to retrieve client library.") } - return send(ctx, "budibase-client.js", { - root: rootPath, - }) } -export const getSignedUploadURL = async function (ctx: any) { +export const getSignedUploadURL = async function (ctx: Ctx) { // Ensure datasource is valid let datasource try { @@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) { const params = { Bucket: bucket, Key: key } signedUrl = s3.getSignedUrl("putObject", params) publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}` - } catch (error) { + } catch (error: any) { ctx.throw(400, error) } }