From 34918013cb5e0669060e82a8165acde5e58f6ee8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 23 Mar 2021 17:54:02 +0000 Subject: [PATCH] Main work of file system refactor now complete, ready to test more fully - most test cases passing, need to look through them more thoroughly and make sure everything still makes sense. --- .../popovers/EditDatasourcePopover.svelte | 2 +- .../popovers/EditTablePopover.svelte | 4 +- .../server/src/api/controllers/application.js | 4 +- .../server/src/api/controllers/component.js | 63 ++++++------- .../src/api/controllers/deploy/utils.js | 17 +++- .../server/src/api/routes/tests/auth.spec.js | 1 - .../src/api/routes/tests/component.spec.js | 28 ++---- packages/server/src/automations/actions.js | 44 ++------- .../src/tests/utilities/TestConfiguration.js | 10 -- .../server/src/utilities/fileSystem/index.js | 94 ++++++++++++++++++- .../src/utilities/fileSystem/utilities.js | 78 ++++++++++++--- packages/server/src/utilities/index.js | 19 ---- 12 files changed, 224 insertions(+), 140 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte index 7570cd6c5e..8c2ea880ae 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte @@ -27,7 +27,7 @@ notifier.success("Datasource deleted") // navigate to first index page if the source you are deleting is selected if (wasSelectedSource === datasource._id) { - $goto('./datasource') + $goto("./datasource") } hideEditor() } diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte index 60886b5be1..0a186375d8 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte @@ -37,13 +37,13 @@ } async function deleteTable() { - const wasSelectedTable = $backendUiStore.selectedTable + const wasSelectedTable = $backendUiStore.selectedTable await backendUiStore.actions.tables.delete(table) store.actions.screens.delete(templateScreens) await backendUiStore.actions.tables.fetch() notifier.success("Table deleted") if (wasSelectedTable._id === table._id) { - $goto('./table') + $goto("./table") } hideEditor() } diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index e2bb15dbd3..9255932877 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -210,7 +210,9 @@ exports.delete = async function(ctx) { const app = await db.get(ctx.params.appId) const result = await db.destroy() - await deleteApp(ctx.params.appId) + if (env.NODE_ENV !== "jest") { + await deleteApp(ctx.params.appId) + } ctx.status = 200 ctx.message = `Application ${app.name} deleted successfully.` diff --git a/packages/server/src/api/controllers/component.js b/packages/server/src/api/controllers/component.js index 092d154817..67fd6e8897 100644 --- a/packages/server/src/api/controllers/component.js +++ b/packages/server/src/api/controllers/component.js @@ -1,44 +1,41 @@ const CouchDB = require("../../db") -const { resolve, join } = require("../../utilities/centralPath") -const { - budibaseTempDir, - budibaseAppsDir, -} = require("../../utilities/budibaseDir") +const { join } = require("../../utilities/centralPath") +const { budibaseTempDir } = require("../../utilities/budibaseDir") +const fileSystem = require("../../utilities/fileSystem") exports.fetchAppComponentDefinitions = async function(ctx) { const appId = ctx.params.appId || ctx.appId const db = new CouchDB(appId) const app = await db.get(appId) - ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => { - let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules") - - if (ctx.isDev) { - appDirectory = budibaseTempDir() - } - - const componentJson = require(join( - appDirectory, - componentLibrary, - ctx.isDev ? "" : "package", - "manifest.json" - )) - - const result = {} - - // map over the components.json and add the library identifier as a key - // button -> @budibase/standard-components/button - for (let key of Object.keys(componentJson)) { - const fullComponentName = `${componentLibrary}/${key}`.toLowerCase() - result[fullComponentName] = { + let componentManifests = await Promise.all( + app.componentLibraries.map(async library => { + let manifest + if (ctx.isDev) { + manifest = require(join( + budibaseTempDir(), + library, + ctx.isDev ? "" : "package", + "manifest.json" + )) + } else { + manifest = await fileSystem.getComponentLibraryManifest(appId, library) + } + return { + manifest, + library, + } + }) + ) + const definitions = {} + for (let { manifest, library } of componentManifests) { + for (let key of Object.keys(manifest)) { + const fullComponentName = `${library}/${key}`.toLowerCase() + definitions[fullComponentName] = { component: fullComponentName, - ...componentJson[key], + ...manifest[key], } } - - return { - ...acc, - ...result, - } - }, {}) + } + ctx.body = definitions } diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index eb9f85eed8..de608acfb1 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -1,11 +1,26 @@ -const { walkDir } = require("../../../utilities") const { join } = require("../../../utilities/centralPath") +const fs = require("fs") const { budibaseAppsDir } = require("../../../utilities/budibaseDir") const fetch = require("node-fetch") const PouchDB = require("../../../db") const CouchDB = require("pouchdb") const { upload } = require("../../../utilities/fileSystem") +// TODO: everything in this file is to be removed + +function walkDir(dirPath, callback) { + for (let filename of fs.readdirSync(dirPath)) { + const filePath = `${dirPath}/${filename}` + const stat = fs.lstatSync(filePath) + + if (stat.isFile()) { + callback(filePath) + } else { + walkDir(filePath, callback) + } + } +} + exports.fetchCredentials = async function(url, body) { const response = await fetch(url, { method: "POST", diff --git a/packages/server/src/api/routes/tests/auth.spec.js b/packages/server/src/api/routes/tests/auth.spec.js index 0eb0b6d851..13695d596d 100644 --- a/packages/server/src/api/routes/tests/auth.spec.js +++ b/packages/server/src/api/routes/tests/auth.spec.js @@ -1,4 +1,3 @@ -const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const setup = require("./utilities") describe("/authenticate", () => { diff --git a/packages/server/src/api/routes/tests/component.spec.js b/packages/server/src/api/routes/tests/component.spec.js index cabf9f8223..a485939ae4 100644 --- a/packages/server/src/api/routes/tests/component.spec.js +++ b/packages/server/src/api/routes/tests/component.spec.js @@ -1,8 +1,15 @@ const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const setup = require("./utilities") -const fs = require("fs") -const { resolve, join } = require("path") -const { budibaseAppsDir } = require("../../../utilities/budibaseDir") + +jest.mock("../../../utilities/fileSystem/utilities", () => ({ + ...jest.requireActual("../../../utilities/fileSystem/utilities"), + retrieve: () => { + const { join } = require("path") + const library = join("@budibase", "standard-components") + const path = require.resolve(library).split(join("dist", "index.js"))[0] + "manifest.json" + return JSON.stringify(require(path)) + } +})) describe("/component", () => { let request = setup.getRequest() @@ -14,23 +21,8 @@ describe("/component", () => { await config.init() }) - function mock() { - const manifestFile = "manifest.json" - const appId = config.getAppId() - const libraries = [join("@budibase", "standard-components")] - for (let library of libraries) { - let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules", library, "package") - fs.mkdirSync(appDirectory, { recursive: true }) - - const file = require.resolve(library).split(join("dist", "index.js"))[0] + manifestFile - fs.copyFileSync(file, join(appDirectory, manifestFile)) - } - } - describe("fetch definitions", () => { it("should be able to fetch definitions", async () => { - // have to "mock" the files required - mock() const res = await request .get(`/${config.getAppId()}/components/definitions`) .set(config.defaultHeaders()) diff --git a/packages/server/src/automations/actions.js b/packages/server/src/automations/actions.js index ee57f5a109..3859bbc910 100644 --- a/packages/server/src/automations/actions.js +++ b/packages/server/src/automations/actions.js @@ -5,17 +5,12 @@ const deleteRow = require("./steps/deleteRow") const createUser = require("./steps/createUser") const outgoingWebhook = require("./steps/outgoingWebhook") const env = require("../environment") -const download = require("download") -const fetch = require("node-fetch") -const { join } = require("../utilities/centralPath") -const os = require("os") -const fs = require("fs") const Sentry = require("@sentry/node") +const { + automationInit, + getExternalAutomationStep, +} = require("../utilities/fileSystem") -const DEFAULT_BUCKET = - "https://prod-budi-automations.s3-eu-west-1.amazonaws.com" -const DEFAULT_DIRECTORY = ".budibase-automations" -const AUTOMATION_MANIFEST = "manifest.json" const BUILTIN_ACTIONS = { SEND_EMAIL: sendEmail.run, CREATE_ROW: createRow.run, @@ -33,8 +28,6 @@ const BUILTIN_DEFINITIONS = { OUTGOING_WEBHOOK: outgoingWebhook.definition, } -let AUTOMATION_BUCKET = env.AUTOMATION_BUCKET -let AUTOMATION_DIRECTORY = env.AUTOMATION_DIRECTORY let MANIFEST = null /* istanbul ignore next */ @@ -42,15 +35,6 @@ function buildBundleName(pkgName, version) { return `${pkgName}@${version}.min.js` } -/* istanbul ignore next */ -async function downloadPackage(name, version, bundleName) { - await download( - `${AUTOMATION_BUCKET}/${name}/${version}/${bundleName}`, - AUTOMATION_DIRECTORY - ) - return require(join(AUTOMATION_DIRECTORY, bundleName)) -} - /* istanbul ignore next */ module.exports.getAction = async function(actionName) { if (BUILTIN_ACTIONS[actionName] != null) { @@ -66,28 +50,12 @@ module.exports.getAction = async function(actionName) { } const pkg = MANIFEST.packages[actionName] const bundleName = buildBundleName(pkg.stepId, pkg.version) - try { - return require(join(AUTOMATION_DIRECTORY, bundleName)) - } catch (err) { - return downloadPackage(pkg.stepId, pkg.version, bundleName) - } + return getExternalAutomationStep(pkg.stepId, pkg.version, bundleName) } module.exports.init = async function() { - // set defaults - if (!AUTOMATION_DIRECTORY) { - AUTOMATION_DIRECTORY = join(os.homedir(), DEFAULT_DIRECTORY) - } - if (!AUTOMATION_BUCKET) { - AUTOMATION_BUCKET = DEFAULT_BUCKET - } - if (!fs.existsSync(AUTOMATION_DIRECTORY)) { - fs.mkdirSync(AUTOMATION_DIRECTORY, { recursive: true }) - } - // env setup to get async packages try { - let response = await fetch(`${AUTOMATION_BUCKET}/${AUTOMATION_MANIFEST}`) - MANIFEST = await response.json() + MANIFEST = await automationInit() module.exports.DEFINITIONS = MANIFEST && MANIFEST.packages ? Object.assign(MANIFEST.packages, BUILTIN_DEFINITIONS) diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 6cdd468c0e..e759a1fd7f 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -14,9 +14,6 @@ const { } = require("./structures") const controllers = require("./controllers") const supertest = require("supertest") -const fs = require("fs") -const { budibaseAppsDir } = require("../../utilities/budibaseDir") -const { join } = require("path") const EMAIL = "babs@babs.com" const PASSWORD = "babs_password" @@ -66,13 +63,6 @@ class TestConfiguration { if (this.server) { this.server.close() } - const appDir = budibaseAppsDir() - const files = fs.readdirSync(appDir) - for (let file of files) { - if (this.allApps.some(app => file.includes(app._id))) { - fs.rmdirSync(join(appDir, file), { recursive: true }) - } - } } defaultHeaders() { diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index cec04ad699..b19b89f710 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -5,8 +5,21 @@ const { join } = require("path") const uuid = require("uuid/v4") const CouchDB = require("../../db") const { ObjectStoreBuckets } = require("../../constants") -const { upload, streamUpload, deleteFolder, downloadTarball } = require("./utilities") +const { + upload, + retrieve, + streamUpload, + deleteFolder, + downloadTarball, +} = require("./utilities") const { downloadLibraries, newAppPublicPath } = require("./newApp") +const download = require("download") +const env = require("../../environment") +const { homedir } = require("os") + +const DEFAULT_AUTOMATION_BUCKET = + "https://prod-budi-automations.s3-eu-west-1.amazonaws.com" +const DEFAULT_AUTOMATION_DIRECTORY = ".budibase-automations" /** * The single stack system (Cloud and Builder) should not make use of the file system where possible, @@ -21,10 +34,19 @@ const { downloadLibraries, newAppPublicPath } = require("./newApp") * everything required to function is ready. */ exports.checkDevelopmentEnvironment = () => { - if (isDev() && !fs.existsSync(budibaseTempDir())) { - console.error( + if (!isDev()) { + return + } + let error + if (!fs.existsSync(budibaseTempDir())) { + error = "Please run a build before attempting to run server independently to fill 'tmp' directory." - ) + } + if (!fs.existsSync(join(process.cwd(), ".env"))) { + error = "Must run via yarn once to generate environment." + } + if (error) { + console.error(error) process.exit(-1) } } @@ -66,6 +88,13 @@ exports.apiFileReturn = contents => { return fs.createReadStream(path) } +/** + * Takes a copy of the database state for an app to the object store. + * @param {string} appId The ID of the app which is to be backed up. + * @param {string} backupName The name of the backup located in the object store. + * @return The backup has been completed when this promise completes and returns a file stream + * to the temporary backup file (to return via API if required). + */ exports.performBackup = async (appId, backupName) => { const path = join(budibaseTempDir(), backupName) const writeStream = fs.createWriteStream(path) @@ -81,15 +110,31 @@ exports.performBackup = async (appId, backupName) => { return fs.createReadStream(path) } +/** + * Downloads required libraries and creates a new path in the object store. + * @param {string} appId The ID of the app which is being created. + * @return {Promise} once promise completes app resources should be ready in object store. + */ exports.createApp = async appId => { await downloadLibraries(appId) await newAppPublicPath(appId) } +/** + * Removes all of the assets created for an app in the object store. + * @param {string} appId The ID of the app which is being deleted. + * @return {Promise} once promise completes the app resources will be removed from object store. + */ exports.deleteApp = async appId => { await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) } +/** + * Retrieves a template and pipes it to minio as well as making it available temporarily. + * @param {string} type The type of template which is to be retrieved. + * @param name + * @return {Promise<*>} + */ exports.downloadTemplate = async (type, name) => { const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com" @@ -97,6 +142,44 @@ exports.downloadTemplate = async (type, name) => { return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) } +/** + * Retrieves component libraries from object store (or tmp symlink if in local) + */ +exports.getComponentLibraryManifest = async (appId, library) => { + const path = join(appId, "node_modules", library, "package", "manifest.json") + let resp = await retrieve(ObjectStoreBuckets.APPS, path) + if (typeof resp !== "string") { + resp = resp.toString("utf8") + } + return JSON.parse(resp) +} + +exports.automationInit = async () => { + const directory = + env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) + const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }) + } + // env setup to get async packages + let response = await fetch(`${bucket}/manifest.json`) + return response.json() +} + +exports.getExternalAutomationStep = async (name, version, bundleName) => { + const directory = env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) + const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET + try { + return require(join(directory, bundleName)) + } catch (err) { + await download( + `${bucket}/${name}/${version}/${bundleName}`, + directory + ) + return require(join(directory, bundleName)) + } +} + /** * All file reads come through here just to make sure all of them make sense * allows a centralised location to check logic is all good. @@ -106,6 +189,7 @@ exports.readFileSync = (filepath, options = "utf8") => { } /** - * Full function definition provided in the utilities. + * Full function definition for below can be found in the utilities. */ exports.upload = upload +exports.retrieve = retrieve diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index aa58668b45..e637130473 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -10,6 +10,7 @@ const { streamUpload } = require("./utilities") const fs = require("fs") const { budibaseTempDir } = require("../budibaseDir") const env = require("../../environment") +const { ObjectStoreBuckets } = require("../../constants") const streamPipeline = promisify(stream.pipeline) @@ -18,6 +19,29 @@ const CONTENT_TYPE_MAP = { css: "text/css", js: "application/javascript", } +const STRING_CONTENT_TYPES = [ + CONTENT_TYPE_MAP.html, + CONTENT_TYPE_MAP.css, + CONTENT_TYPE_MAP.js, +] + +function publicPolicy(bucketName) { + return { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { + AWS: ["*"], + }, + Action: "s3:GetObject", + Resource: [`arn:aws:s3:::${bucketName}/*`], + }, + ], + } +} + +const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS] /** * Gets a connection to the object store using the S3 SDK. @@ -26,17 +50,23 @@ const CONTENT_TYPE_MAP = { * @constructor */ exports.ObjectStore = bucket => { - return new AWS.S3({ - // TODO: need to deal with endpoint properly - endpoint: env.MINIO_URL, + const config = { s3ForcePathStyle: true, // needed with minio? signatureVersion: "v4", params: { Bucket: bucket, }, - }) + } + if (env.MINIO_URL) { + config.endpoint = env.MINIO_URL + } + return new AWS.S3(config) } +/** + * Given an object store and a bucket name this will make sure the bucket exists, + * if it does not exist then it will create it. + */ exports.makeSureBucketExists = async (client, bucketName) => { try { await client @@ -52,6 +82,16 @@ exports.makeSureBucketExists = async (client, bucketName) => { Bucket: bucketName, }) .promise() + // public buckets are quite hidden in the system, make sure + // no bucket is set accidentally + if (PUBLIC_BUCKETS.includes(bucketName)) { + await client + .putBucketPolicy({ + Bucket: bucketName, + Policy: JSON.stringify(publicPolicy(bucketName)), + }) + .promise() + } } else { throw err } @@ -61,13 +101,6 @@ exports.makeSureBucketExists = async (client, bucketName) => { /** * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). - * @param {string} bucket The name of the bucket to be uploaded to. - * @param {string} filename The name/path of the file in the object store. - * @param {string} path The path to the file (ideally a temporary file). - * @param {string} type If the content type is known can be specified. - * @param {object} metadata If there is metadata for the object it can be passed as well. - * @return {Promise} The file has been uploaded to the object store successfully when - * promise completes. */ exports.upload = async ({ bucket, filename, path, type, metadata }) => { const extension = [...filename.split(".")].pop() @@ -86,6 +119,10 @@ exports.upload = async ({ bucket, filename, path, type, metadata }) => { return objectStore.upload(config).promise() } +/** + * Similar to the upload function but can be used to send a file stream + * through to the object store. + */ exports.streamUpload = async (bucket, filename, stream) => { const objectStore = exports.ObjectStore(bucket) await exports.makeSureBucketExists(objectStore, bucket) @@ -98,6 +135,25 @@ exports.streamUpload = async (bucket, filename, stream) => { return objectStore.upload(params).promise() } +/** + * retrieves the contents of a file from the object store, if it is a known content type it + * will be converted, otherwise it will be returned as a buffer stream. + */ +exports.retrieve = async (bucket, filename) => { + const objectStore = exports.ObjectStore(bucket) + const params = { + Bucket: bucket, + Key: sanitize(filename).replace(/\\/g, "/"), + } + const response = await objectStore.getObject(params).promise() + // currently these are all strings + if (STRING_CONTENT_TYPES.includes(response.ContentType)) { + return response.Body.toString("utf8") + } else { + return response.Body + } +} + exports.deleteFolder = async (bucket, folder) => { const client = exports.ObjectStore(bucket) const listParams = { diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 226955101f..4c7a12398b 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -1,6 +1,5 @@ const env = require("../environment") const { DocumentTypes, SEPARATOR } = require("../db/utils") -const fs = require("fs") const CouchDB = require("../db") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -82,24 +81,6 @@ exports.isClient = ctx => { return ctx.headers["x-budibase-type"] === "client" } -/** - * Recursively walk a directory tree and execute a callback on all files. - * @param {String} dirPath - Directory to traverse - * @param {Function} callback - callback to execute on files - */ -exports.walkDir = (dirPath, callback) => { - for (let filename of fs.readdirSync(dirPath)) { - const filePath = `${dirPath}/${filename}` - const stat = fs.lstatSync(filePath) - - if (stat.isFile()) { - callback(filePath) - } else { - exports.walkDir(filePath, callback) - } - } -} - exports.getLogoUrl = () => { return "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" }