MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Revert "Client library fetching through server""

This commit is contained in:
Michael Drury 2023-10-24 18:13:36 +01:00 committed by GitHub
parent bbcc2a81d4
commit c6b377c565
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 127 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
packages/backend-core/src/objectStore/buckets/app.ts
View File

@ -1,37 +1,50 @@
import env from "../../environment"
import * as objectStore from "../objectStore"
import * as cloudfront from "../cloudfront"
import qs from "querystring"
import { DEFAULT_TENANT_ID, getTenantId } from "../../context"
export function clientLibraryPath(appId: string) {
return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
}
/**
* In production the client library is stored in the object store, however in development
* we use the symlinked version produced by lerna, located in node modules. We link to this
* via a specific endpoint (under /api/assets/client).
* @param appId In production we need the appId to look up the correct bucket, as the
* version of the client lib may differ between apps.
* @param version The version to retrieve.
* @return The URL to be inserted into appPackage response or server rendered
* app index file.
* Previously we used to serve the client library directly from Cloudfront, however
* due to issues with the domain we were unable to continue doing this - keeping
* incase we are able to switch back to CDN path again in future.
*/
export const clientLibraryUrl = (appId: string, version: string) => {
if (env.isProd()) {
let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
if (env.CLOUDFRONT_CDN) {
// append app version to bust the cache
if (version) {
file += `?v=${version}`
}
// don't need to use presigned for client with cloudfront
// file is public
return cloudfront.getUrl(file)
} else {
return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
export function clientLibraryCDNUrl(appId: string, version: string) {
let file = clientLibraryPath(appId)
if (env.CLOUDFRONT_CDN) {
// append app version to bust the cache
if (version) {
file += `?v=${version}`
}
// don't need to use presigned for client with cloudfront
// file is public
return cloudfront.getUrl(file)
} else {
return `/api/assets/client`
return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
}
}
export const getAppFileUrl = (s3Key: string) => {
export function clientLibraryUrl(appId: string, version: string) {
let tenantId, qsParams: { appId: string; version: string; tenantId?: string }
try {
tenantId = getTenantId()
} finally {
qsParams = {
appId,
version,
}
}
if (tenantId && tenantId !== DEFAULT_TENANT_ID) {
qsParams.tenantId = tenantId
}
return `/api/assets/client?${qs.encode(qsParams)}`
}
export function getAppFileUrl(s3Key: string) {
if (env.CLOUDFRONT_CDN) {
return cloudfront.getPresignedUrl(s3Key)
} else {

16
packages/backend-core/src/objectStore/buckets/plugins.ts
View File

@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
// URLS
export const enrichPluginURLs = (plugins: Plugin[]) => {
export function enrichPluginURLs(plugins: Plugin[]) {
if (!plugins || !plugins.length) {
return []
}
@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => {
})
}
const getPluginJSUrl = (plugin: Plugin) => {
function getPluginJSUrl(plugin: Plugin) {
const s3Key = getPluginJSKey(plugin)
return getPluginUrl(s3Key)
}
const getPluginIconUrl = (plugin: Plugin): string | undefined => {
function getPluginIconUrl(plugin: Plugin): string | undefined {
const s3Key = getPluginIconKey(plugin)
if (!s3Key) {
return
@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => {
return getPluginUrl(s3Key)
}
const getPluginUrl = (s3Key: string) => {
function getPluginUrl(s3Key: string) {
if (env.CLOUDFRONT_CDN) {
return cloudfront.getPresignedUrl(s3Key)
} else {
@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => {
// S3 KEYS
export const getPluginJSKey = (plugin: Plugin) => {
export function getPluginJSKey(plugin: Plugin) {
return getPluginS3Key(plugin, "plugin.min.js")
}
export const getPluginIconKey = (plugin: Plugin) => {
export function getPluginIconKey(plugin: Plugin) {
// stored iconUrl is deprecated - hardcode to icon.svg in this case
const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
if (!iconFileName) {
@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => {
return getPluginS3Key(plugin, iconFileName)
}
const getPluginS3Key = (plugin: Plugin, fileName: string) => {
function getPluginS3Key(plugin: Plugin, fileName: string) {
const s3Key = getPluginS3Dir(plugin.name)
return `${s3Key}/${fileName}`
}
export const getPluginS3Dir = (pluginName: string) => {
export function getPluginS3Dir(pluginName: string) {
let s3Key = `${pluginName}`
if (env.MULTI_TENANCY) {
const tenantId = context.getTenantId()

42
packages/backend-core/src/objectStore/buckets/tests/app.spec.ts
View File

@ -1,5 +1,4 @@
import * as app from "../app"
import { getAppFileUrl } from "../app"
import { testEnv } from "../../../../tests/extra"
describe("app", () => {
@ -7,6 +6,15 @@ describe("app", () => {
testEnv.nodeJest()
})
function baseCheck(url: string, tenantId?: string) {
expect(url).toContain("/api/assets/client")
if (tenantId) {
expect(url).toContain(`tenantId=${tenantId}`)
}
expect(url).toContain("appId=app_123")
expect(url).toContain("version=2.0.0")
}
describe("clientLibraryUrl", () => {
function getClientUrl() {
return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
@ -20,31 +28,19 @@ describe("app", () => {
it("gets url in dev", () => {
testEnv.nodeDev()
const url = getClientUrl()
expect(url).toBe("/api/assets/client")
})
it("gets url with embedded minio", () => {
testEnv.withMinio()
const url = getClientUrl()
expect(url).toBe(
"/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
)
baseCheck(url)
})
it("gets url with custom S3", () => {
testEnv.withS3()
const url = getClientUrl()
expect(url).toBe(
"http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
)
baseCheck(url)
})
it("gets url with cloudfront + s3", () => {
testEnv.withCloudfront()
const url = getClientUrl()
expect(url).toBe(
"http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
)
baseCheck(url)
})
})
@ -57,7 +53,7 @@ describe("app", () => {
testEnv.nodeDev()
await testEnv.withTenant(tenantId => {
const url = getClientUrl()
expect(url).toBe("/api/assets/client")
baseCheck(url, tenantId)
})
})
@ -65,9 +61,7 @@ describe("app", () => {
await testEnv.withTenant(tenantId => {
testEnv.withMinio()
const url = getClientUrl()
expect(url).toBe(
"/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
)
baseCheck(url, tenantId)
})
})
@ -75,9 +69,7 @@ describe("app", () => {
await testEnv.withTenant(tenantId => {
testEnv.withS3()
const url = getClientUrl()
expect(url).toBe(
"http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
)
baseCheck(url, tenantId)
})
})
@ -85,9 +77,7 @@ describe("app", () => {
await testEnv.withTenant(tenantId => {
testEnv.withCloudfront()
const url = getClientUrl()
expect(url).toBe(
"http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
)
baseCheck(url, tenantId)
})
})
})

62
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -1,6 +1,6 @@
const sanitize = require("sanitize-s3-objectkey")
import AWS from "aws-sdk"
import stream from "stream"
import stream, { Readable } from "stream"
import fetch from "node-fetch"
import tar from "tar-fs"
import zlib from "zlib"
@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) {
* @return an S3 object store object, check S3 Nodejs SDK for usage.
* @constructor
*/
export const ObjectStore = (
export function ObjectStore(
bucket: string,
opts: { presigning: boolean } = { presigning: false }
) => {
) {
const config: any = {
s3ForcePathStyle: true,
signatureVersion: "v4",
@ -104,7 +104,7 @@ export const ObjectStore = (
* Given an object store and a bucket name this will make sure the bucket exists,
* if it does not exist then it will create it.
*/
export const makeSureBucketExists = async (client: any, bucketName: string) => {
export async function makeSureBucketExists(client: any, bucketName: string) {
bucketName = sanitizeBucket(bucketName)
try {
await client
@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
* Uploads the contents of a file given the required parameters, useful when
* temp files in use (for example file uploaded as an attachment).
*/
export const upload = async ({
export async function upload({
bucket: bucketName,
filename,
path,
type,
metadata,
}: UploadParams) => {
}: UploadParams) {
const extension = filename.split(".").pop()
const fileBytes = fs.readFileSync(path)
@ -180,12 +180,12 @@ export const upload = async ({
* Similar to the upload function but can be used to send a file stream
* through to the object store.
*/
export const streamUpload = async (
export async function streamUpload(
bucketName: string,
filename: string,
stream: any,
extra = {}
) => {
) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
@ -215,7 +215,7 @@ export const streamUpload = async (
* retrieves the contents of a file from the object store, if it is a known content type it
* will be converted, otherwise it will be returned as a buffer stream.
*/
export const retrieve = async (bucketName: string, filepath: string) => {
export async function retrieve(bucketName: string, filepath: string) {
const objectStore = ObjectStore(bucketName)
const params = {
Bucket: sanitizeBucket(bucketName),
@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => {
}
}
export const listAllObjects = async (bucketName: string, path: string) => {
export async function listAllObjects(bucketName: string, path: string) {
const objectStore = ObjectStore(bucketName)
const list = (params: ListParams = {}) => {
return objectStore
@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => {
/**
* Generate a presigned url with a default TTL of 1 hour
*/
export const getPresignedUrl = (
export function getPresignedUrl(
bucketName: string,
key: string,
durationSeconds: number = 3600
) => {
) {
const objectStore = ObjectStore(bucketName, { presigning: true })
const params = {
Bucket: sanitizeBucket(bucketName),
@ -291,7 +291,7 @@ export const getPresignedUrl = (
/**
* Same as retrieval function but puts to a temporary file.
*/
export const retrieveToTmp = async (bucketName: string, filepath: string) => {
export async function retrieveToTmp(bucketName: string, filepath: string) {
bucketName = sanitizeBucket(bucketName)
filepath = sanitizeKey(filepath)
const data = await retrieve(bucketName, filepath)
@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => {
return outputPath
}
export const retrieveDirectory = async (bucketName: string, path: string) => {
export async function retrieveDirectory(bucketName: string, path: string) {
let writePath = join(budibaseTempDir(), v4())
fs.mkdirSync(writePath)
const objects = await listAllObjects(bucketName, path)
@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => {
/**
* Delete a single file.
*/
export const deleteFile = async (bucketName: string, filepath: string) => {
export async function deleteFile(bucketName: string, filepath: string) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
const params = {
@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
return objectStore.deleteObject(params).promise()
}
export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
export async function deleteFiles(bucketName: string, filepaths: string[]) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
const params = {
@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
/**
* Delete a path, including everything within.
*/
export const deleteFolder = async (
export async function deleteFolder(
bucketName: string,
folder: string
): Promise<any> => {
): Promise<any> {
bucketName = sanitizeBucket(bucketName)
folder = sanitizeKey(folder)
const client = ObjectStore(bucketName)
@ -383,11 +383,11 @@ export const deleteFolder = async (
}
}
export const uploadDirectory = async (
export async function uploadDirectory(
bucketName: string,
localPath: string,
bucketPath: string
) => {
) {
bucketName = sanitizeBucket(bucketName)
let uploads = []
const files = fs.readdirSync(localPath, { withFileTypes: true })
@ -404,11 +404,11 @@ export const uploadDirectory = async (
return files
}
export const downloadTarballDirect = async (
export async function downloadTarballDirect(
url: string,
path: string,
headers = {}
) => {
) {
path = sanitizeKey(path)
const response = await fetch(url, { headers })
if (!response.ok) {
@ -418,11 +418,11 @@ export const downloadTarballDirect = async (
await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
}
export const downloadTarball = async (
export async function downloadTarball(
url: string,
bucketName: string,
path: string
) => {
) {
bucketName = sanitizeBucket(bucketName)
path = sanitizeKey(path)
const response = await fetch(url)
@ -438,3 +438,17 @@ export const downloadTarball = async (
// return the temporary path incase there is a use for it
return tmpPath
}
export async function getReadStream(
bucketName: string,
path: string
): Promise<Readable> {
bucketName = sanitizeBucket(bucketName)
path = sanitizeKey(path)
const client = ObjectStore(bucketName)
const params = {
Bucket: bucketName,
Key: path,
}
return client.getObject(params).createReadStream()
}

49
packages/server/src/api/controllers/static/index.ts
View File

@ -16,7 +16,7 @@ import AWS from "aws-sdk"
import fs from "fs"
import sdk from "../../../sdk"
import * as pro from "@budibase/pro"
import { App } from "@budibase/types"
import { App, Ctx } from "@budibase/types"
const send = require("koa-send")
@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
}
}
export const toggleBetaUiFeature = async function (ctx: any) {
export const toggleBetaUiFeature = async function (ctx: Ctx) {
const cookieName = `beta:${ctx.params.feature}`
if (ctx.cookies.get(cookieName)) {
@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) {
}
}
export const serveBuilder = async function (ctx: any) {
export const serveBuilder = async function (ctx: Ctx) {
const builderPath = join(TOP_LEVEL_PATH, "builder")
await send(ctx, ctx.file, { root: builderPath })
}
export const uploadFile = async function (ctx: any) {
let files =
ctx.request.files.file.length > 1
? Array.from(ctx.request.files.file)
: [ctx.request.files.file]
export const uploadFile = async function (ctx: Ctx) {
const file = ctx.request?.files?.file
let files = file && Array.isArray(file) ? Array.from(file) : [file]
const uploads = files.map(async (file: any) => {
const fileExtension = [...file.name.split(".")].pop()
@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) {
ctx.body = await Promise.all(uploads)
}
export const deleteObjects = async function (ctx: any) {
export const deleteObjects = async function (ctx: Ctx) {
ctx.body = await objectStore.deleteFiles(
ObjectStoreBuckets.APPS,
ctx.request.body.keys
)
}
export const serveApp = async function (ctx: any) {
export const serveApp = async function (ctx: Ctx) {
const bbHeaderEmbed =
ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) {
}
}
export const serveBuilderPreview = async function (ctx: any) {
export const serveBuilderPreview = async function (ctx: Ctx) {
const db = context.getAppDB({ skip_setup: true })
const appInfo = await db.get<App>(DocumentType.APP_METADATA)
@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) {
}
}
export const serveClientLibrary = async function (ctx: any) {
export const serveClientLibrary = async function (ctx: Ctx) {
const appId = context.getAppId() || (ctx.request.query.appId as string)
let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
// incase running from TS directly
if (env.isDev() && !fs.existsSync(rootPath)) {
rootPath = join(require.resolve("@budibase/client"), "..")
if (!appId) {
ctx.throw(400, "No app ID provided - cannot fetch client library.")
}
if (env.isProd()) {
ctx.body = await objectStore.getReadStream(
ObjectStoreBuckets.APPS,
objectStore.clientLibraryPath(appId!)
)
} else if (env.isDev()) {
// incase running from TS directly
const tsPath = join(require.resolve("@budibase/client"), "..")
return send(ctx, "budibase-client.js", {
root: !fs.existsSync(rootPath) ? tsPath : rootPath,
})
} else {
ctx.throw(500, "Unable to retrieve client library.")
}
return send(ctx, "budibase-client.js", {
root: rootPath,
})
}
export const getSignedUploadURL = async function (ctx: any) {
export const getSignedUploadURL = async function (ctx: Ctx) {
// Ensure datasource is valid
let datasource
try {
@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) {
const params = { Bucket: bucket, Key: key }
signedUrl = s3.getSignedUrl("putObject", params)
publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
} catch (error) {
} catch (error: any) {
ctx.throw(400, error)
}
}