From 8655d73248a706bc63de09dbf8a2158aad70d4d9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 18 Dec 2020 12:54:20 +0000 Subject: [PATCH] Update after testing, it is now possible to make a deployment to a self hosted environment. Some work still required, better authentication around MINIO deployment, currently the bucket is set to public read and there is no signing/verification to the upload process, also right now four different URLs are needed for the builder to connect correctly, ideally this shouldn't be the case. --- hosting/docker-compose.yml | 8 ++-- hosting/hosting.properties | 9 ++-- .../builder/src/builderStore/store/hosting.js | 2 +- .../start/BuilderSettingsModal.svelte | 6 ++- packages/server/src/api/controllers/auth.js | 4 +- .../src/api/controllers/deploy/Deployment.js | 2 +- .../src/api/controllers/deploy/index.js | 2 +- .../src/api/controllers/deploy/selfDeploy.js | 19 ++++---- .../src/api/controllers/deploy/utils.js | 1 + .../server/src/api/controllers/hosting.js | 4 +- .../src/api/controllers/static/index.js | 16 ++----- .../static/templates/BudibaseApp.svelte | 4 +- packages/server/src/environment.js | 6 +-- .../server/src/utilities/builder/hosting.js | 33 +++++++++---- packages/worker/src/api/controllers/deploy.js | 48 +++++++++---------- packages/worker/src/environment.js | 4 +- 16 files changed, 89 insertions(+), 79 deletions(-) diff --git a/hosting/docker-compose.yml b/hosting/docker-compose.yml index 7f196d4e3b..65d89d2b0f 100644 --- a/hosting/docker-compose.yml +++ b/hosting/docker-compose.yml @@ -8,14 +8,14 @@ services: ports: - "${APP_PORT}:${APP_PORT}" environment: - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} - MINIO_URL: http://nginx-service:${MINIO_PORT} SELF_HOSTED: 1 COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} LOGO_URL: ${LOGO_URL} PORT: ${APP_PORT} + HOSTING_URL: ${HOSTING_URL} + MINIO_PORT: ${MINIO_PORT} + JWT_SECRET: ${JWT_SECRET} depends_on: - worker-service @@ -29,7 +29,7 @@ services: PORT: ${WORKER_PORT} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} - RAW_MINIO_URL: http://nginx-service:${MINIO_PORT} + RAW_MINIO_URL: http://nginx-service:5001 COUCH_DB_USERNAME: ${COUCH_DB_USER} COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} RAW_COUCH_DB_URL: http://couchdb-service:5984 diff --git a/hosting/hosting.properties b/hosting/hosting.properties index c48d4cc1d8..4afdcc044c 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -4,9 +4,10 @@ COUCH_DB_PASSWORD=budibase COUCH_DB_USER=budibase WORKER_API_KEY=budibase BUDIBASE_ENVIRONMENT=PRODUCTION -HOSTING_URL="http://localhost:4001" +HOSTING_URL="http://localhost" LOGO_URL=https://logoipsum.com/logo/logo-15.svg APP_PORT=4002 -MINIO_PORT=4003 -COUCH_DB_PORT=4004 -WORKER_PORT=4006 +WORKER_PORT=4003 +MINIO_PORT=4004 +COUCH_DB_PORT=4005 +JWT_SECRET=testsecret diff --git a/packages/builder/src/builderStore/store/hosting.js b/packages/builder/src/builderStore/store/hosting.js index f626a738ae..36067773b5 100644 --- a/packages/builder/src/builderStore/store/hosting.js +++ b/packages/builder/src/builderStore/store/hosting.js @@ -17,7 +17,7 @@ export const getHostingStore = () => { const [info, urls] = await Promise.all(responses.map(resp => resp.json())) store.update(state => { state.hostingInfo = info - state.appUrl = urls.appServer + state.appUrl = urls.app return state }) return info diff --git a/packages/builder/src/components/start/BuilderSettingsModal.svelte b/packages/builder/src/components/start/BuilderSettingsModal.svelte index c734a3b297..d101369fba 100644 --- a/packages/builder/src/components/start/BuilderSettingsModal.svelte +++ b/packages/builder/src/components/start/BuilderSettingsModal.svelte @@ -39,8 +39,10 @@

{#if selfhosted} - - + + + + {/if} diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js index e8bc000e03..f9c3ef945f 100644 --- a/packages/server/src/api/controllers/auth.js +++ b/packages/server/src/api/controllers/auth.js @@ -35,8 +35,8 @@ exports.authenticate = async ctx => { roleId: dbUser.roleId, version: app.version, } - // if in cloud add the user api key - if (env.CLOUD) { + // if in cloud add the user api key, unless self hosted + if (env.CLOUD && !env.SELF_HOSTED) { const { apiKey } = await getAPIKey(ctx.user.appId) payload.apiKey = apiKey } diff --git a/packages/server/src/api/controllers/deploy/Deployment.js b/packages/server/src/api/controllers/deploy/Deployment.js index 3b95652c44..4c0f0f2df9 100644 --- a/packages/server/src/api/controllers/deploy/Deployment.js +++ b/packages/server/src/api/controllers/deploy/Deployment.js @@ -37,10 +37,10 @@ class Deployment { if (!verification) { return } + this.verification = verification if (this.verification.quota) { this.quota = this.verification.quota } - this.verification = verification } getVerification() { diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index c92f5f5948..0117fd3735 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -24,7 +24,7 @@ async function checkAllDeployments(deployments) { deployment.status === DeploymentStatus.PENDING && Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS ) { - deployment.status = status + deployment.status = DeploymentStatus.FAILURE deployment.err = "Timed out" updated = true } diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index 766a1c35bc..429c4c9505 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -5,23 +5,26 @@ const { performReplication, fetchCredentials, } = require("./utils") -const { getDeploymentUrl } = require("../../../utilities/builder/hosting") -const { join } = require("path") +const { + getWorkerUrl, + getCouchUrl, + getMinioUrl, +} = require("../../../utilities/builder/hosting") exports.preDeployment = async function() { - const url = join(await getDeploymentUrl(), "api", "deploy") + const url = `${await getWorkerUrl()}/api/deploy` const json = await fetchCredentials(url, { apiKey: env.BUDIBASE_API_KEY, }) // response contains: - // couchDbSession, bucket, objectStoreSession, couchDbUrl, objectStoreUrl + // couchDbSession, bucket, objectStoreSession // set credentials here, means any time we're verified we're ready to go if (json.objectStoreSession) { AWS.config.update({ - accessKeyId: json.objectStoreSession.AccessKeyId, - secretAccessKey: json.objectStoreSession.SecretAccessKey, + accessKeyId: json.objectStoreSession.accessKeyId, + secretAccessKey: json.objectStoreSession.secretAccessKey, }) } @@ -36,7 +39,7 @@ exports.deploy = async function(deployment) { const appId = deployment.getAppId() const verification = deployment.getVerification() const objClient = new AWS.S3({ - endpoint: verification.objectStoreUrl, + endpoint: await getMinioUrl(), s3ForcePathStyle: true, // needed with minio? signatureVersion: "v4", params: { @@ -54,6 +57,6 @@ exports.replicateDb = async function(deployment) { return performReplication( appId, verification.couchDbSession, - verification.couchDbUrl + await getCouchUrl() ) } diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index 8e00a7c5a5..87030a82e7 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -3,6 +3,7 @@ const sanitize = require("sanitize-s3-objectkey") const { walkDir } = require("../../../utilities") const { join } = require("../../../utilities/centralPath") const { budibaseAppsDir } = require("../../../utilities/budibaseDir") +const fetch = require("node-fetch") const PouchDB = require("../../../db") const CouchDB = require("pouchdb") diff --git a/packages/server/src/api/controllers/hosting.js b/packages/server/src/api/controllers/hosting.js index 351290ca23..ce4ff98c69 100644 --- a/packages/server/src/api/controllers/hosting.js +++ b/packages/server/src/api/controllers/hosting.js @@ -3,7 +3,7 @@ const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") const { getHostingInfo, HostingTypes, - getAppServerUrl, + getAppUrl, } = require("../../utilities/builder/hosting") exports.fetchInfo = async ctx => { @@ -30,6 +30,6 @@ exports.fetch = async ctx => { exports.fetchUrls = async ctx => { ctx.body = { - appServer: await getAppServerUrl(ctx.appId), + app: await getAppUrl(ctx.appId), } } diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index 468a896a96..8916758d20 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -17,17 +17,10 @@ const setBuilderToken = require("../../../utilities/builder/setBuilderToken") const fileProcessor = require("../../../utilities/fileProcessor") const env = require("../../../environment") -function appServerUrl(appId) { - if (env.SELF_HOSTED) { - return env.HOSTING_URL - } else { - return `https://${appId}.app.budi.live` - } -} - function objectStoreUrl() { if (env.SELF_HOSTED) { - return env.MINIO_URL + // TODO: need a better way to handle this, probably reverse proxy + return `${env.HOSTING_URL}:${env.MINIO_PORT}/app-assets/assets` } else { return "https://cdn.app.budi.live/assets" } @@ -164,7 +157,7 @@ exports.serveApp = async function(ctx) { title: appInfo.name, production: env.CLOUD, appId: ctx.params.appId, - appServerUrl: appServerUrl(ctx.params.appId), + objectStoreUrl: objectStoreUrl(ctx.params.appId), }) const template = handlebars.compile( @@ -232,8 +225,7 @@ exports.serveComponentLibrary = async function(ctx) { } const S3_URL = encodeURI( join( - appServerUrl(appId), - "assets", + objectStoreUrl(appId), componentLib, ctx.query.library, "dist", diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte index 0729737a44..818b6bbd45 100644 --- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte +++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte @@ -4,11 +4,11 @@ export let appId export let production - export let appServerUrl + export let objectStoreUrl function publicPath(path) { if (production) { - return `${appServerUrl}/assets/${appId}/${path}` + return `${objectStoreUrl}/${appId}/${path}` } return `/assets/${path}` diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 19d3cf1c57..a06d5ea23c 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -37,10 +37,8 @@ module.exports = { DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, // self hosting features - MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, - MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, - MINIO_URL: process.MINIO_URL, - HOSTING_URL: process.HOSTING_URL, + HOSTING_URL: process.env.HOSTING_URL, + MINIO_PORT: process.env.MINIO_PORT, LOGO_URL: process.env.LOGO_URL, _set(key, value) { process.env[key] = value diff --git a/packages/server/src/utilities/builder/hosting.js b/packages/server/src/utilities/builder/hosting.js index d1f8632b5d..0e5e07b26f 100644 --- a/packages/server/src/utilities/builder/hosting.js +++ b/packages/server/src/utilities/builder/hosting.js @@ -1,6 +1,5 @@ const CouchDB = require("../../db") const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") -const { join } = require("path") function getProtocol(hostingInfo) { return hostingInfo.useHttps ? "https://" : "http://" @@ -23,8 +22,10 @@ exports.getHostingInfo = async () => { doc = { _id: HOSTING_DOC, type: exports.HostingTypes.CLOUD, - appServerUrl: "app.budi.live", - deploymentServerUrl: "", + appUrl: "app.budi.live", + workerUrl: "", + minioUrl: "", + couchUrl: "", templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com", useHttps: true, } @@ -32,22 +33,34 @@ exports.getHostingInfo = async () => { return doc } -exports.getAppServerUrl = async appId => { +exports.getAppUrl = async appId => { const hostingInfo = await exports.getHostingInfo() const protocol = getProtocol(hostingInfo) let url if (hostingInfo.type === "cloud") { - url = `${protocol}${appId}.${hostingInfo.appServerUrl}` + url = `${protocol}${appId}.${hostingInfo.appUrl}` } else { - url = `${protocol}${hostingInfo.appServerUrl}` + url = `${protocol}${hostingInfo.appUrl}` } return url } -exports.getDeploymentUrl = async () => { +exports.getWorkerUrl = async () => { const hostingInfo = await exports.getHostingInfo() const protocol = getProtocol(hostingInfo) - return `${protocol}${hostingInfo.deploymentServerUrl}` + return `${protocol}${hostingInfo.workerUrl}` +} + +exports.getMinioUrl = async () => { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + return `${protocol}${hostingInfo.minioUrl}` +} + +exports.getCouchUrl = async () => { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + return `${protocol}${hostingInfo.couchUrl}` } exports.getTemplatesUrl = async (appId, type, name) => { @@ -55,9 +68,9 @@ exports.getTemplatesUrl = async (appId, type, name) => { const protocol = getProtocol(hostingInfo) let path if (type && name) { - path = join("templates", type, `${name}.tar.gz`) + path = `templates/type/${name}.tar.gz` } else { path = "manifest.json" } - return join(`${protocol}${hostingInfo.templatesUrl}`, path) + return `${protocol}${hostingInfo.templatesUrl}/${path}` } diff --git a/packages/worker/src/api/controllers/deploy.js b/packages/worker/src/api/controllers/deploy.js index cdff0dadcb..e2cf3f6a6e 100644 --- a/packages/worker/src/api/controllers/deploy.js +++ b/packages/worker/src/api/controllers/deploy.js @@ -5,11 +5,23 @@ const AWS = require("aws-sdk") const APP_BUCKET = "app-assets" // this doesn't matter in self host const REGION = "eu-west-1" +const PUBLIC_READ_POLICY = { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: "*", + Action: "s3:GetObject", + Resource: `arn:aws:s3:::${APP_BUCKET}/*`, + } + ] +} async function getCouchSession() { // fetch session token for the api user const session = await got.post(`${env.RAW_COUCH_DB_URL}/_session`, { responseType: "json", + credentials: "include", json: { username: env.COUCH_DB_USERNAME, password: env.COUCH_DB_PASSWORD, @@ -38,37 +50,25 @@ async function getMinioSession() { }) // make sure the bucket exists try { - await objClient.headBucket({ Bucket: APP_BUCKET }).promise() + await objClient.headBucket({ + Bucket: APP_BUCKET + }).promise() + await objClient.putBucketPolicy({ + Bucket: APP_BUCKET, + Policy: JSON.stringify(PUBLIC_READ_POLICY), + }).promise() } catch (err) { // bucket doesn't exist create it if (err.statusCode === 404) { - await objClient.createBucket({ Bucket: APP_BUCKET }).promise() + await objClient.createBucket({ + Bucket: APP_BUCKET, + }).promise() } else { throw err } } - // TODO: this doesn't seem to work get an error - // TODO: Generating temporary credentials not allowed for this request. - // TODO: this should work based on minio documentation - // const sts = new AWS.STS({ - // endpoint: env.RAW_MINIO_URL, - // region: REGION, - // s3ForcePathStyle: true, - // }) - // // NOTE: In the following commands RoleArn and RoleSessionName are not meaningful for MinIO - // const params = { - // DurationSeconds: 3600, - // ExternalId: "123ABC", - // Policy: '{"Version":"2012-10-17","Statement":[{"Sid":"Stmt1","Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}]}', - // RoleArn: 'arn:xxx:xxx:xxx:xxxx', - // RoleSessionName: 'anything', - // }; - // const assumedRole = await sts.assumeRole(params).promise(); - // if (!assumedRole) { - // throw "Unable to get access to object store." - // } - // return assumedRole.Credentials // TODO: need to do something better than this + // Ideally want to send back some pre-signed URLs for files that are to be uploaded return { accessKeyId: env.MINIO_ACCESS_KEY, secretAccessKey: env.MINIO_SECRET_KEY, @@ -80,7 +80,5 @@ exports.deploy = async ctx => { couchDbSession: await getCouchSession(), bucket: APP_BUCKET, objectStoreSession: await getMinioSession(), - couchDbUrl: env.RAW_COUCH_DB_URL, - objectStoreUrl: env.RAW_MINIO_URL, } } diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index fee1d3be04..ce0f174eb9 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -4,10 +4,12 @@ module.exports = { PORT: process.env.PORT, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, - RAW_MINIO_URL: process.env.RAW_MINIO_URL, COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, RAW_COUCH_DB_URL: process.env.RAW_COUCH_DB_URL, + RAW_MINIO_URL: process.env.RAW_MINIO_URL, + COUCH_DB_PORT: process.env.COUCH_DB_PORT, + MINIO_PORT: process.env.MINIO_PORT, _set(key, value) { process.env[key] = value module.exports[key] = value