From f5a9c68c699841e1aa72ba55d2cb6d9260deb4d3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Dec 2020 18:31:48 +0000 Subject: [PATCH] WIP - starting to work on deployment and hosting so that they don't use static URLs anymore to reach assets, instead using environment variables to determine what to use. --- hosting/docker-compose.yml | 1 + hosting/hosting.properties | 1 + .../server/src/api/controllers/application.js | 3 + .../src/api/controllers/deploy/selfDeploy.js | 3 +- .../server/src/api/controllers/hosting.js | 39 +++++++++++++ .../src/api/controllers/static/index.js | 29 +++++++++- .../static/templates/BudibaseApp.svelte | 5 +- packages/server/src/api/routes/hosting.js | 13 +++++ packages/server/src/constants/index.js | 2 + packages/server/src/environment.js | 2 + .../server/src/utilities/builder/hosting.js | 57 +++++++++++++++++++ 11 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/api/controllers/hosting.js create mode 100644 packages/server/src/api/routes/hosting.js create mode 100644 packages/server/src/utilities/builder/hosting.js diff --git a/hosting/docker-compose.yml b/hosting/docker-compose.yml index 13102a28d1..72a055e91a 100644 --- a/hosting/docker-compose.yml +++ b/hosting/docker-compose.yml @@ -10,6 +10,7 @@ services: environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_URL: http://minio-service:9000 SELF_HOSTED: 1 COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 8ffe9f1473..9e4dfe1dfc 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -3,4 +3,5 @@ MINIO_SECRET_KEY=budibase COUCH_DB_PASSWORD=budibase COUCH_DB_USER=budibase BUDIBASE_ENVIRONMENT=PRODUCTION +HOSTING_URL="http://localhost:4001" LOGO_URL=https://logoipsum.com/logo/logo-15.svg diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 293c72d7c3..1d259ac425 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -150,6 +150,9 @@ exports.create = async function(ctx) { name: ctx.request.body.name, template: ctx.request.body.template, instance: instance, + deployment: { + type: "cloud", + }, } const instanceDb = new CouchDB(appId) await instanceDb.put(newApplication) diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index 0b26af4e36..e20462a236 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -20,7 +20,7 @@ exports.postDeployment = async function() { exports.deploy = async function(deployment) { const appId = deployment.getAppId() var objClient = new AWS.S3({ - endpoint: "http://localhost:9000", + endpoint: env.MINIO_URL, s3ForcePathStyle: true, // needed with minio? signatureVersion: "v4", params: { @@ -46,6 +46,7 @@ exports.deploy = async function(deployment) { exports.replicateDb = async function(deployment) { const appId = deployment.getAppId() const localDb = new PouchDB(appId) + const remoteDb = new CouchDB(`${env.COUCH_DB_URL}/${appId}`) return performReplication(localDb, remoteDb) } diff --git a/packages/server/src/api/controllers/hosting.js b/packages/server/src/api/controllers/hosting.js new file mode 100644 index 0000000000..4e95405738 --- /dev/null +++ b/packages/server/src/api/controllers/hosting.js @@ -0,0 +1,39 @@ +const CouchDB = require("../../db") +const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") +const { + getHostingInfo, + HostingTypes, + getAppServerUrl, +} = require("../../utilities/builder/hosting") + +exports.fetchInfo = async ctx => { + ctx.body = { + types: Object.values(HostingTypes), + } +} + +exports.save = async ctx => { + const db = new CouchDB(BUILDER_CONFIG_DB) + const { type, appServerUrl, objectStoreUrl, useHttps } = ctx.request.body + if (type === HostingTypes.CLOUD) { + ctx.throw(400, "Cannot update Cloud hosting information") + } + await db.put({ + _id: HOSTING_DOC, + type, + appServerUrl, + objectStoreUrl, + useHttps, + }) + ctx.body = "Hosting information saved successfully" +} + +exports.fetch = async ctx => { + ctx.body = await getHostingInfo() +} + +exports.fetchUrls = async ctx => { + ctx.body = { + appServer: getAppServerUrl(ctx.appId), + } +} diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index f884725b29..468a896a96 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -17,6 +17,22 @@ const setBuilderToken = require("../../../utilities/builder/setBuilderToken") const fileProcessor = require("../../../utilities/fileProcessor") const env = require("../../../environment") +function appServerUrl(appId) { + if (env.SELF_HOSTED) { + return env.HOSTING_URL + } else { + return `https://${appId}.app.budi.live` + } +} + +function objectStoreUrl() { + if (env.SELF_HOSTED) { + return env.MINIO_URL + } else { + return "https://cdn.app.budi.live/assets" + } +} + // this was the version before we started versioning the component library const COMP_LIB_BASE_APP_VERSION = "0.2.5" @@ -148,6 +164,7 @@ exports.serveApp = async function(ctx) { title: appInfo.name, production: env.CLOUD, appId: ctx.params.appId, + appServerUrl: appServerUrl(ctx.params.appId), }) const template = handlebars.compile( @@ -166,8 +183,9 @@ exports.serveAttachment = async function(ctx) { const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments") // Serve from CloudFront + // TODO: need to replace this with link to self hosted object store if (env.CLOUD) { - const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}` + const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file) const response = await fetch(S3_URL) const body = await response.text() ctx.set("Content-Type", response.headers.get("Content-Type")) @@ -213,7 +231,14 @@ exports.serveComponentLibrary = async function(ctx) { componentLib += `-${COMP_LIB_BASE_APP_VERSION}` } const S3_URL = encodeURI( - `https://${appId}.app.budi.live/assets/${componentLib}/${ctx.query.library}/dist/index.js` + join( + appServerUrl(appId), + "assets", + componentLib, + ctx.query.library, + "dist", + "index.js" + ) ) const response = await fetch(S3_URL) const body = await response.text() diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte index 3641c26e93..0729737a44 100644 --- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte +++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte @@ -4,12 +4,11 @@ export let appId export let production - - export const PRODUCTION_ASSETS_URL = `https://${appId}.app.budi.live` + export let appServerUrl function publicPath(path) { if (production) { - return `${PRODUCTION_ASSETS_URL}/assets/${appId}/${path}` + return `${appServerUrl}/assets/${appId}/${path}` } return `/assets/${path}` diff --git a/packages/server/src/api/routes/hosting.js b/packages/server/src/api/routes/hosting.js new file mode 100644 index 0000000000..b8d4256bcd --- /dev/null +++ b/packages/server/src/api/routes/hosting.js @@ -0,0 +1,13 @@ +const Router = require("@koa/router") +const controller = require("../controllers/hosting") +const authorized = require("../../middleware/authorized") +const { BUILDER } = require("../../utilities/security/permissions") + +const router = Router() + +router + .fetch("/api/hosting/info", authorized(BUILDER), controller.fetchInfo) + .fetch("/api/hosting", authorized(BUILDER), controller.fetch) + .post("/api/hosting", authorized(BUILDER), controller.save) + +module.exports = router diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 0ae805c064..5a97a62af6 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -41,3 +41,5 @@ const USERS_TABLE_SCHEMA = { exports.AuthTypes = AuthTypes exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA +exports.BUILDER_CONFIG_DB = "builder-config-db" +exports.HOSTING_DOC = "hosting-doc" diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index cca3c4fbac..19d3cf1c57 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -39,6 +39,8 @@ module.exports = { // self hosting features MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, + MINIO_URL: process.MINIO_URL, + HOSTING_URL: process.HOSTING_URL, LOGO_URL: process.env.LOGO_URL, _set(key, value) { process.env[key] = value diff --git a/packages/server/src/utilities/builder/hosting.js b/packages/server/src/utilities/builder/hosting.js new file mode 100644 index 0000000000..97af87d3e2 --- /dev/null +++ b/packages/server/src/utilities/builder/hosting.js @@ -0,0 +1,57 @@ +const CouchDB = require("../../db") +const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") +const { join } = require("path") + +function getProtocol(hostingInfo) { + return hostingInfo.useHttps ? "https://" : "http://" +} + +exports.HostingTypes = { + CLOUD: "cloud", + SELF: "self", +} + +exports.getHostingInfo = async () => { + const db = new CouchDB(BUILDER_CONFIG_DB) + let doc + try { + doc = await db.get(HOSTING_DOC) + } catch (err) { + // don't write this doc, want to be able to update these default props + // for our servers with a new release without needing to worry about state of + // PouchDB in peoples installations + doc = { + _id: HOSTING_DOC, + type: exports.HostingTypes.CLOUD, + appServerUrl: "app.budi.live", + objectStoreUrl: "cdn.app.budi.live", + templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com", + useHttps: true, + } + } + return doc +} + +exports.getAppServerUrl = async (appId) => { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + let url + if (hostingInfo.type === "cloud") { + url = `${protocol}${appId}.${hostingInfo.appServerUrl}` + } else { + url = `${protocol}${hostingInfo.appServerUrl}` + } + return url +} + +exports.getTemplatesUrl = async (appId, type, name) => { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + let path + if (type && name) { + path = join("templates", type, `${name}.tar.gz`) + } else { + path = "manifest.json" + } + return join(`${protocol}${hostingInfo.templatesUrl}`, path) +}