From b70ff24dac0190723e6ac76122f7a874f2d2128f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Mar 2021 18:03:58 +0000 Subject: [PATCH] Some further work to make sure all use of the budibaseAppsDir is removed in important places. --- packages/server/scripts/exportAppTemplate.js | 9 +- .../src/api/controllers/static/index.js | 110 +++++++++--------- packages/server/src/api/index.js | 2 - .../server/src/utilities/fileSystem/index.js | 6 + .../src/utilities/fileSystem/utilities.js | 15 ++- 5 files changed, 79 insertions(+), 63 deletions(-) diff --git a/packages/server/scripts/exportAppTemplate.js b/packages/server/scripts/exportAppTemplate.js index c6d738225c..34988cda37 100755 --- a/packages/server/scripts/exportAppTemplate.js +++ b/packages/server/scripts/exportAppTemplate.js @@ -3,7 +3,8 @@ const yargs = require("yargs") const fs = require("fs") const { join } = require("path") const CouchDB = require("../src/db") -const { budibaseAppsDir } = require("../src/utilities/budibaseDir") +// load environment +const env = require("../src/environment") // Script to export a chosen budibase app into a package // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId @@ -25,6 +26,9 @@ yargs }, }, async args => { + if (!env.isDev()) { + throw "Only works in dev" + } const name = args.name, appId = args.appId console.log("Exporting app..") @@ -34,10 +38,11 @@ yargs ) return } - const exportPath = join(budibaseAppsDir(), "templates", "app", name, "db") + const exportPath = join(process.cwd(), name, "db") fs.ensureDirSync(exportPath) const writeStream = fs.createWriteStream(join(exportPath, "dump.text")) // perform couch dump + const instanceDb = new CouchDB(appId) await instanceDb.dump(writeStream, {}) console.log(`Template ${name} exported to ${exportPath}`) diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index 1da98d1940..c866db3561 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -6,10 +6,7 @@ const fetch = require("node-fetch") const uuid = require("uuid") const { prepareUpload } = require("../deploy/utils") const { processString } = require("@budibase/string-templates") -const { - budibaseAppsDir, - budibaseTempDir, -} = require("../../../utilities/budibaseDir") +const { budibaseTempDir } = require("../../../utilities/budibaseDir") const { getDeployedApps } = require("../../../utilities/builder/hosting") const CouchDB = require("../../../db") const setBuilderToken = require("../../../utilities/builder/setBuilderToken") @@ -18,15 +15,36 @@ const env = require("../../../environment") const { OBJ_STORE_DIRECTORY } = require("../../../constants") const fileProcessor = require("../../../utilities/fileSystem/processor") +const BB_CDN = "https://cdn.app.budi.live/assets" + function objectStoreUrl() { if (env.SELF_HOSTED) { // can use a relative url for this as all goes through the proxy (this is hosted in minio) return OBJ_STORE_DIRECTORY } else { - return "https://cdn.app.budi.live/assets" + return BB_CDN } } +function internalObjectStoreUrl() { + if (env.SELF_HOSTED) { + return (env.MINIO_URL + OBJ_STORE_DIRECTORY).replace( + /(https?:\/\/)|(\/)+/g, + "$1$2" + ) + } else { + return BB_CDN + } +} + +async function returnObjectStoreFile(ctx, path) { + const S3_URL = `${internalObjectStoreUrl()}/${path}` + const response = await fetch(S3_URL) + const body = await response.text() + ctx.set("Content-Type", response.headers.get("Content-Type")) + ctx.body = body +} + async function checkForSelfHostedURL(ctx) { // the "appId" component of the URL may actually be a specific self hosted URL let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}` @@ -102,69 +120,47 @@ exports.serveApp = async function(ctx) { } exports.serveAttachment = async function(ctx) { - const appId = ctx.user.appId - const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments") - - // Serve from object store - if (env.isProd()) { - const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file) - const response = await fetch(S3_URL) - const body = await response.text() - ctx.set("Content-Type", response.headers.get("Content-Type")) - ctx.body = body - return - } - - await send(ctx, ctx.file, { root: attachmentsPath }) + await returnObjectStoreFile( + ctx, + join(ctx.user.appId, "attachments", ctx.file) + ) } exports.serveAppAsset = async function(ctx) { - // default to homedir - const appPath = resolve(budibaseAppsDir(), ctx.user.appId, "public") - - await send(ctx, ctx.file, { root: ctx.devPath || appPath }) + if (env.isDev() || env.isTest()) { + return send(ctx, ctx.file, { root: budibaseTempDir() }) + } + await returnObjectStoreFile(ctx, join(ctx.user.appId, "public", ctx.file)) } exports.serveComponentLibrary = async function(ctx) { const appId = ctx.query.appId || ctx.appId - // default to homedir - let componentLibraryPath = resolve( - budibaseAppsDir(), - appId, - "node_modules", - decodeURI(ctx.query.library), - "package", - "dist" - ) - if (env.isDev()) { - componentLibraryPath = join( + if (env.isDev() || env.isTest()) { + const componentLibraryPath = join( budibaseTempDir(), decodeURI(ctx.query.library), "dist" ) - } else { - let componentLib = "componentlibrary" - if (ctx.user.version) { - componentLib += `-${ctx.user.version}` - } else { - componentLib += `-${COMP_LIB_BASE_APP_VERSION}` - } - const S3_URL = encodeURI( - join( - objectStoreUrl(appId), - componentLib, - ctx.query.library, - "dist", - "index.js" - ) - ) - const response = await fetch(S3_URL) - const body = await response.text() - ctx.type = "application/javascript" - ctx.body = body - return + return send(ctx, "/awsDeploy.js", { root: componentLibraryPath }) } - - await send(ctx, "/awsDeploy.js", { root: componentLibraryPath }) + let componentLib = "componentlibrary" + if (ctx.user.version) { + componentLib += `-${ctx.user.version}` + } else { + componentLib += `-${COMP_LIB_BASE_APP_VERSION}` + } + const S3_URL = encodeURI( + join( + objectStoreUrl(appId), + componentLib, + ctx.query.library, + "dist", + "index.js" + ) + ) + const response = await fetch(S3_URL) + const body = await response.text() + ctx.type = "application/javascript" + ctx.body = body } diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 9315c2aaf0..aeceb65039 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -2,7 +2,6 @@ const Router = require("@koa/router") const authenticated = require("../middleware/authenticated") const compress = require("koa-compress") const zlib = require("zlib") -const { budibaseAppsDir } = require("../utilities/budibaseDir") const { mainRoutes, authRoutes, staticRoutes } = require("./routes") const pkg = require("../../package.json") @@ -24,7 +23,6 @@ router ) .use(async (ctx, next) => { ctx.config = { - latestPackagesFolder: budibaseAppsDir(), jwtSecret: env.JWT_SECRET, useAppRootPath: true, } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 9f6ce3fe7e..7ef99eda41 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -8,6 +8,7 @@ const { ObjectStoreBuckets } = require("../../constants") const { upload, retrieve, + retrieveToTmp, streamUpload, deleteFolder, downloadTarball, @@ -38,6 +39,10 @@ exports.init = () => { if (!fs.existsSync(tempDir)) { fs.mkdirSync(tempDir) } + const clientLibPath = join(budibaseTempDir(), "budibase-client.js") + if (env.isTest() && !fs.existsSync(clientLibPath)) { + fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath) + } } /** @@ -224,3 +229,4 @@ exports.cleanup = appIds => { */ exports.upload = upload exports.retrieve = retrieve +exports.retrieveToTmp = retrieveToTmp diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index 7b97790dfe..98f856bd37 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -10,6 +10,7 @@ const fs = require("fs") const { budibaseTempDir } = require("../budibaseDir") const env = require("../../environment") const { ObjectStoreBuckets } = require("../../constants") +const uuid = require("uuid/v4") const streamPipeline = promisify(stream.pipeline) @@ -146,11 +147,11 @@ exports.streamUpload = async (bucket, filename, stream) => { * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -exports.retrieve = async (bucket, filename) => { +exports.retrieve = async (bucket, filepath) => { const objectStore = exports.ObjectStore(bucket) const params = { Bucket: bucket, - Key: sanitize(filename).replace(/\\/g, "/"), + Key: sanitize(filepath).replace(/\\/g, "/"), } const response = await objectStore.getObject(params).promise() // currently these are all strings @@ -161,6 +162,16 @@ exports.retrieve = async (bucket, filename) => { } } +/** + * Same as retrieval function but puts to a temporary file. + */ +exports.retrieveToTmp = async (bucket, filepath) => { + const data = await exports.retrieve(bucket, filepath) + const outputPath = join(budibaseTempDir(), uuid()) + fs.writeFileSync(outputPath, data) + return outputPath +} + exports.deleteFolder = async (bucket, folder) => { const client = exports.ObjectStore(bucket) const listParams = {