From a5d2e9a3b9719782f179fd0c27d281c1ee2fc4ef Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Dec 2020 17:45:20 +0000 Subject: [PATCH] Some work to not contact lambda or use quotas when deploying locally. --- .../src/api/controllers/deploy/awsDeploy.js | 71 ++----------------- .../src/api/controllers/deploy/selfDeploy.js | 32 ++++++++- .../src/api/controllers/deploy/utils.js | 66 ++++++++++++++++- packages/server/src/environment.js | 2 + 4 files changed, 101 insertions(+), 70 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/awsDeploy.js b/packages/server/src/api/controllers/deploy/awsDeploy.js index 5a36871ae8..433913644a 100644 --- a/packages/server/src/api/controllers/deploy/awsDeploy.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -1,12 +1,7 @@ -const fs = require("fs") -const { join } = require("../../../utilities/centralPath") -const AwsDeploy = require("aws-sdk") +const AWS = require("aws-sdk") const fetch = require("node-fetch") -const { budibaseAppsDir } = require("../../../utilities/budibaseDir") -const PouchDB = require("../../../db") const env = require("../../../environment") -const { prepareUpload } = require("./utils") -const { walkDir } = require("../../../utilities") +const { deployToObjectStore } = require("./utils") /** * Verifies the users API key and @@ -37,7 +32,7 @@ exports.preDeployment = async function(deployment) { // set credentials here, means any time we're verified we're ready to go if (json.credentials) { - AwsDeploy.config.update({ + AWS.config.update({ accessKeyId: json.credentials.AccessKeyId, secretAccessKey: json.credentials.SecretAccessKey, sessionToken: json.credentials.SessionToken, @@ -80,65 +75,11 @@ exports.postDeployment = async function(deployment) { exports.deploy = async function(deployment) { const appId = deployment.getAppId() const { bucket, accountId } = deployment.getVerification() - const s3 = new AwsDeploy.S3({ + const metadata = { accountId } + const s3Client = new AWS.S3({ params: { Bucket: bucket, }, }) - - const appAssetsPath = join(budibaseAppsDir(), appId, "public") - - const appPages = fs.readdirSync(appAssetsPath) - - let uploads = [] - - for (let page of appPages) { - // Upload HTML, CSS and JS for each page of the web app - walkDir(join(appAssetsPath, page), function(filePath) { - const appAssetUpload = prepareUpload({ - file: { - path: filePath, - name: [...filePath.split("/")].pop(), - }, - s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), - s3, - metadata: { accountId }, - }) - uploads.push(appAssetUpload) - }) - } - - // Upload file attachments - const db = new PouchDB(appId) - let fileUploads - try { - fileUploads = await db.get("_local/fileuploads") - } catch (err) { - fileUploads = { _id: "_local/fileuploads", uploads: [] } - } - - for (let file of fileUploads.uploads) { - if (file.uploaded) continue - - const attachmentUpload = prepareUpload({ - file, - s3Key: `assets/${appId}/attachments/${file.processedFileName}`, - s3, - metadata: { accountId }, - }) - - uploads.push(attachmentUpload) - - // mark file as uploaded - file.uploaded = true - } - - db.put(fileUploads) - - try { - return await Promise.all(uploads) - } catch (err) { - console.error("Error uploading budibase app assets to s3", err) - throw err - } + await deployToObjectStore(appId, s3Client, metadata) } diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index ad81b6943a..a18736a5f4 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -1,5 +1,31 @@ -exports.preDeployment = async function(deployment) {} +const env = require("../../../environment") +const AWS = require("aws-sdk") +const { deployToObjectStore } = require("./utils") -exports.postDeployment = async function(deployment) {} +const APP_BUCKET = "app-assets" -exports.deploy = async function(deployment) {} +exports.preDeployment = async function() { + AWS.config.update({ + accessKeyId: env.MINIO_ACCESS_KEY, + secretAccessKey: env.MINIO_SECRET_KEY, + }) +} + +exports.postDeployment = async function() { + // we don't actively need to do anything after deployment in self hosting +} + +exports.deploy = async function(deployment) { + const appId = deployment.getAppId() + var objClient = new AWS.S3({ + endpoint: "http://localhost:9000", + s3ForcePathStyle: true, // needed with minio? + signatureVersion: "v4", + params: { + Bucket: APP_BUCKET, + }, + }) + // no metadata, aws has account ID in metadata + const metadata = {} + await deployToObjectStore(appId, objClient, metadata) +} diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index 75fe3133ba..f0a4b44e1d 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -1,5 +1,9 @@ const fs = require("fs") const sanitize = require("sanitize-s3-objectkey") +const { walkDir } = require("../../../utilities") +const { join } = require("../../../utilities/centralPath") +const { budibaseAppsDir } = require("../../../utilities/budibaseDir") +const PouchDB = require("../../../db") const CONTENT_TYPE_MAP = { html: "text/html", @@ -7,11 +11,11 @@ const CONTENT_TYPE_MAP = { js: "application/javascript", } -exports.prepareUpload = async function({ s3Key, metadata, s3, file }) { +exports.prepareUpload = async function({ s3Key, metadata, client, file }) { const extension = [...file.name.split(".")].pop() const fileBytes = fs.readFileSync(file.path) - const upload = await s3 + const upload = await client .upload({ // windows file paths need to be converted to forward slashes for s3 Key: sanitize(s3Key).replace(/\\/g, "/"), @@ -29,3 +33,61 @@ exports.prepareUpload = async function({ s3Key, metadata, s3, file }) { key: upload.Key, } } + +exports.deployToObjectStore = async function(appId, objectClient, metadata) { + const appAssetsPath = join(budibaseAppsDir(), appId, "public") + + const appPages = fs.readdirSync(appAssetsPath) + + let uploads = [] + + for (let page of appPages) { + // Upload HTML, CSS and JS for each page of the web app + walkDir(join(appAssetsPath, page), function(filePath) { + const appAssetUpload = exports.prepareUpload({ + file: { + path: filePath, + name: [...filePath.split("/")].pop(), + }, + s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), + client: objectClient, + metadata, + }) + uploads.push(appAssetUpload) + }) + } + + // Upload file attachments + const db = new PouchDB(appId) + let fileUploads + try { + fileUploads = await db.get("_local/fileuploads") + } catch (err) { + fileUploads = { _id: "_local/fileuploads", uploads: [] } + } + + for (let file of fileUploads.uploads) { + if (file.uploaded) continue + + const attachmentUpload = exports.prepareUpload({ + file, + s3Key: `assets/${appId}/attachments/${file.processedFileName}`, + client: objectClient, + metadata, + }) + + uploads.push(attachmentUpload) + + // mark file as uploaded + file.uploaded = true + } + + db.put(fileUploads) + + try { + return await Promise.all(uploads) + } catch (err) { + console.error("Error uploading budibase app assets to s3", err) + throw err + } +} diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 5fbf57448a..578645ec66 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -36,6 +36,8 @@ module.exports = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, + MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, + MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, _set(key, value) { process.env[key] = value module.exports[key] = value