Updating worker to support using a self host key, a basic level of security, stopping builder from asking for API key if currently configured for self hosting, made the default values for self hosting make sense for a basic local installation, this should be final.

hosting/docker-compose.yaml

@@ -14,8 +14,8 @@ services:
       LOGO_URL: ${LOGO_URL}
       PORT: 4002
       HOSTING_URL: ${HOSTING_URL}
-      MINIO_PORT: ${MINIO_PORT}
       JWT_SECRET: ${JWT_SECRET}
+      PROXY_PORT: ${MAIN_PORT}
     depends_on:
       - worker-service
 
@@ -33,6 +33,7 @@ services:
       COUCH_DB_USERNAME: ${COUCH_DB_USER}
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       RAW_COUCH_DB_URL: http://couchdb-service:5984
+      SELF_HOST_KEY: ${HOSTING_KEY}
     depends_on:
       - minio-service
       - couch-init
@@ -40,7 +41,7 @@ services:
   minio-service:
     image: minio/minio
     volumes:
-      - data1:/data
+      - minio_data:/data
     ports:
       - "${MINIO_PORT}:9000"
     environment:
@@ -89,4 +90,5 @@ services:
 volumes:
   couchdb_data:
     driver: local
-  data1:
+  minio_data:
+    driver: local

hosting/envoy.yaml

@@ -33,8 +33,8 @@ static_resources:
 
               - match: { prefix: "/db/" }
                 route:
-                  prefix_rewrite: "/" 
                   cluster: couchdb-service
+                  prefix_rewrite: "/"
 
               # minio is on the default route because this works
               # best, minio + AWS SDK doesn't handle path proxy

hosting/hosting.properties

@@ -1,14 +1,27 @@
+# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
+MAIN_PORT=10000
+
+# Use this password when configuring your self hosting settings
+# This should be updated
+HOSTING_KEY=budibase
+
+# This section contains customisation options
+HOSTING_URL=http://localhost
+LOGO_URL=https://logoipsum.com/logo/logo-15.svg
+HOSTING_URL=http://localhost
+
+# This section contains all secrets pertaining to the system
+# These should be updated
+JWT_SECRET=testsecret
 MINIO_ACCESS_KEY=budibase
 MINIO_SECRET_KEY=budibase
 COUCH_DB_PASSWORD=budibase
 COUCH_DB_USER=budibase
 WORKER_API_KEY=budibase
-BUDIBASE_ENVIRONMENT=PRODUCTION
-HOSTING_URL=http://localhost
-LOGO_URL=https://logoipsum.com/logo/logo-15.svg
-MAIN_PORT=10000
+
+# This section contains variables that do not need to be altered under normal circumstances
 APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
-JWT_SECRET=testsecret
+BUDIBASE_ENVIRONMENT=PRODUCTION

hosting/start.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-docker-compose --env-file hosting.properties up
+docker-compose --env-file hosting.properties up --build

packages/builder/src/components/start/BuilderSettingsModal.svelte

@@ -6,14 +6,18 @@
   import analytics from "analytics"
   import { onMount } from "svelte"
 
-  let selfhosted = false
   let hostingInfo
+  let selfhosted = false
 
   async function save() {
-    if (!selfhosted) {
-      return
-    }
     hostingInfo.type = selfhosted ? "self" : "cloud"
+    if (!selfhosted && hostingInfo._rev) {
+      hostingInfo = {
+        type: hostingInfo.type,
+        _id: hostingInfo._id,
+        _rev: hostingInfo._rev,
+      }
+    }
     try {
       await hostingStore.actions.save(hostingInfo)
       notifier.success(`Settings saved.`)
@@ -22,6 +26,15 @@
     }
   }
 
+  function updateSelfHosting(event) {
+
+    if (hostingInfo.type === "cloud" && event.target.checked) {
+      hostingInfo.hostingUrl = "localhost:10000"
+      hostingInfo.useHttps = false
+      hostingInfo.selfHostKey = "budibase"
+    }
+  }
+
   onMount(async () => {
     hostingInfo = await hostingStore.actions.fetch()
     selfhosted = hostingInfo.type === "self"
@@ -31,8 +44,7 @@
 <ModalContent
   title="Builder settings"
   confirmText="Save"
-  onConfirm={save}
-  showConfirmButton={selfhosted}>
+  onConfirm={save}>
   <h5>Theme</h5>
   <ThemeEditor />
   <h5>Hosting</h5>
@@ -40,9 +52,10 @@
     This section contains settings that relate to the deployment and hosting of
     apps made in this builder.
   </p>
-  <Toggle thin text="Self hosted" bind:checked={selfhosted} />
+  <Toggle thin text="Self hosted" on:change={updateSelfHosting} bind:checked={selfhosted} />
   {#if selfhosted}
-    <Input bind:value={hostingInfo.hostingUrl} label="Apps URL" />
+    <Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" />
+    <Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" />
     <Toggle thin text="HTTPS" bind:checked={hostingInfo.useHttps} />
   {/if}
 </ModalContent>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { writable } from "svelte/store"
-  import { store, automationStore, backendUiStore } from "builderStore"
+  import { store, automationStore, backendUiStore, hostingStore } from "builderStore"
   import { string, object } from "yup"
   import api, { get } from "builderStore/api"
   import Form from "@svelteschool/svelte-forms"
@@ -12,6 +12,7 @@
   import { fade } from "svelte/transition"
   import { post } from "builderStore/api"
   import analytics from "analytics"
+  import {onMount} from "svelte"
 
   //Move this to context="module" once svelte-forms is updated so that it can bind to stores correctly
   const createAppStore = writable({ currentStep: 0, values: {} })
@@ -62,20 +63,27 @@
     },
   ]
 
-  let steps = [
-    {
-      component: API,
-      errors,
-    },
-    {
+  let steps = []
+
+  onMount(async () => {
+    let hostingInfo = await hostingStore.actions.fetch()
+    steps = []
+    // only validate API key for Cloud
+    if (hostingInfo.type === "cloud") {
+      steps.push({
+        component: API,
+        errors,
+      })
+    }
+    steps.push({
       component: Info,
       errors,
-    },
-    {
+    })
+    steps.push({
       component: User,
       errors,
-    },
-  ]
+    })
+  })
 
   if (hasKey) {
     validationSchemas.shift()

packages/server/src/api/controllers/deploy/index.js

@@ -84,7 +84,10 @@ async function deployApp(deployment) {
   } catch (err) {
     deployment.setStatus(DeploymentStatus.FAILURE, err.message)
     await storeLocalDeploymentHistory(deployment)
-    throw new Error(`Deployment Failed: ${err.message}`)
+    throw {
+      ...err,
+      message: `Deployment Failed: ${err.message}`,
+    }
   }
 }
 

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -1,4 +1,3 @@
-const env = require("../../../environment")
 const AWS = require("aws-sdk")
 const {
   deployToObjectStore,
@@ -9,26 +8,34 @@ const {
   getWorkerUrl,
   getCouchUrl,
   getMinioUrl,
+  getSelfHostKey,
 } = require("../../../utilities/builder/hosting")
 
 exports.preDeployment = async function() {
   const url = `${await getWorkerUrl()}/api/deploy`
-  const json = await fetchCredentials(url, {
-    apiKey: env.BUDIBASE_API_KEY,
-  })
-
-  // response contains:
-  // couchDbSession, bucket, objectStoreSession
-
-  // set credentials here, means any time we're verified we're ready to go
-  if (json.objectStoreSession) {
-    AWS.config.update({
-      accessKeyId: json.objectStoreSession.accessKeyId,
-      secretAccessKey: json.objectStoreSession.secretAccessKey,
+  try {
+    const json = await fetchCredentials(url, {
+      selfHostKey: await getSelfHostKey(),
     })
-  }
 
-  return json
+    // response contains:
+    // couchDbSession, bucket, objectStoreSession
+
+    // set credentials here, means any time we're verified we're ready to go
+    if (json.objectStoreSession) {
+      AWS.config.update({
+        accessKeyId: json.objectStoreSession.accessKeyId,
+        secretAccessKey: json.objectStoreSession.secretAccessKey,
+      })
+    }
+
+    return json
+  } catch (err) {
+    throw {
+      message: "Unauthorised to deploy, check self hosting key",
+      status: 401,
+    }
+  }
 }
 
 exports.postDeployment = async function() {

packages/server/src/api/controllers/deploy/utils.js

@@ -17,6 +17,7 @@ exports.fetchCredentials = async function(url, body) {
   const response = await fetch(url, {
     method: "POST",
     body: JSON.stringify(body),
+    headers: { "Content-Type": "application/json" },
   })
 
   const json = await response.json()
@@ -26,7 +27,7 @@ exports.fetchCredentials = async function(url, body) {
 
   if (response.status !== 200) {
     throw new Error(
-      `Error fetching temporary credentials for api key: ${body.apiKey}`
+      `Error fetching temporary credentials: ${JSON.stringify(json)}`
     )
   }
 

packages/server/src/api/controllers/hosting.js

@@ -15,13 +15,17 @@ exports.fetchInfo = async ctx => {
 exports.save = async ctx => {
   const db = new CouchDB(BUILDER_CONFIG_DB)
   const { type } = ctx.request.body
-  if (type === HostingTypes.CLOUD) {
-    ctx.throw(400, "Cannot update Cloud hosting information")
+  if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
+    ctx.body = await db.remove({
+      ...ctx.request.body,
+      _id: HOSTING_DOC,
+    })
+  } else {
+    ctx.body = await db.put({
+      ...ctx.request.body,
+      _id: HOSTING_DOC,
+    })
   }
-  ctx.body = await db.put({
-    ...ctx.request.body,
-    _id: HOSTING_DOC,
-  })
 }
 
 exports.fetch = async ctx => {

packages/server/src/api/controllers/static/index.js

@@ -20,7 +20,7 @@ const env = require("../../../environment")
 function objectStoreUrl() {
   if (env.SELF_HOSTED) {
     // TODO: need a better way to handle this, probably reverse proxy
-    return `${env.HOSTING_URL}:${env.MINIO_PORT}/app-assets/assets`
+    return `${env.HOSTING_URL}:${env.PROXY_PORT}/app-assets/assets`
   } else {
     return "https://cdn.app.budi.live/assets"
   }

packages/server/src/environment.js

@@ -38,7 +38,7 @@ module.exports = {
   LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
   // self hosting features
   HOSTING_URL: process.env.HOSTING_URL,
-  MINIO_PORT: process.env.MINIO_PORT,
+  PROXY_PORT: process.env.PROXY_PORT,
   LOGO_URL: process.env.LOGO_URL,
   _set(key, value) {
     process.env[key] = value

packages/server/src/utilities/builder/hosting.js

@@ -1,6 +1,8 @@
 const CouchDB = require("../../db")
 const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
 
+const PROD_HOSTING_URL = "app.budi.live"
+
 function getProtocol(hostingInfo) {
   return hostingInfo.useHttps ? "https://" : "http://"
 }
@@ -30,7 +32,8 @@ exports.getHostingInfo = async () => {
     doc = {
       _id: HOSTING_DOC,
       type: exports.HostingTypes.CLOUD,
-      hostingUrl: "app.budi.live",
+      hostingUrl: PROD_HOSTING_URL,
+      selfHostKey: "",
       templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
       useHttps: true,
     }
@@ -62,6 +65,11 @@ exports.getCouchUrl = async () => {
   return getURLWithPath("/db")
 }
 
+exports.getSelfHostKey = async () => {
+  const hostingInfo = await exports.getHostingInfo()
+  return hostingInfo.selfHostKey
+}
+
 exports.getTemplatesUrl = async (appId, type, name) => {
   const hostingInfo = await exports.getHostingInfo()
   const protocol = getProtocol(hostingInfo)

packages/worker/src/api/controllers/deploy.js

@@ -10,9 +10,11 @@ const PUBLIC_READ_POLICY = {
   Statement: [
     {
       Effect: "Allow",
-      Principal: "*",
+      Principal: {
+        AWS: ["*"]
+      },
       Action: "s3:GetObject",
-      Resource: `arn:aws:s3:::${APP_BUCKET}/*`,
+      Resource: [`arn:aws:s3:::${APP_BUCKET}/*`],
     },
   ],
 }
@@ -63,19 +65,18 @@ async function getMinioSession() {
           Bucket: APP_BUCKET,
         })
         .promise()
-    } else if (err.statusCode === 403) {
-      await objClient
-        .putBucketPolicy({
-          Bucket: APP_BUCKET,
-          Policy: JSON.stringify(PUBLIC_READ_POLICY),
-        })
-        .promise()
     }
     else {
       throw err
     }
   }
-  // TODO: need to do something better than this
+  // always make sure policy is correct
+  await objClient
+    .putBucketPolicy({
+      Bucket: APP_BUCKET,
+      Policy: JSON.stringify(PUBLIC_READ_POLICY),
+    })
+    .promise()
   // Ideally want to send back some pre-signed URLs for files that are to be uploaded
   return {
     accessKeyId: env.MINIO_ACCESS_KEY,

packages/worker/src/environment.js

@@ -10,6 +10,7 @@ module.exports = {
   RAW_MINIO_URL: process.env.RAW_MINIO_URL,
   COUCH_DB_PORT: process.env.COUCH_DB_PORT,
   MINIO_PORT: process.env.MINIO_PORT,
+  SELF_HOST_KEY: process.env.SELF_HOST_KEY,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

packages/worker/src/middleware/check-key.js

@@ -1,4 +1,9 @@
+const env = require("../environment")
+
 module.exports = async (ctx, next) => {
-  // TODO: need to check the API key provided in the header
-  await next()
+  if (!ctx.request.body.selfHostKey || env.SELF_HOST_KEY !== ctx.request.body.selfHostKey) {
+    ctx.throw(401, "Deployment unauthorised")
+  } else {
+    await next()
+  }
 }