Merge branch 'master' of github.com:Budibase/budibase into form-builder

.github/workflows/release.yml

@@ -61,3 +61,12 @@ jobs:
             # macOS notarization API key
             API_KEY_ID: ${{ secrets.api_key_id }}
             API_KEY_ISSUER_ID: ${{ secrets.api_key_issuer_id }}
+
+      - name: Build/release Docker images
+        # only run the docker image build on linux, easiest way
+        if: startsWith(matrix.os, 'ubuntu')
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+        run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+        run: yarn build:docker

.gitignore

@@ -62,6 +62,7 @@ typings/
 
 # dotenv environment variables file
 .env
+!hosting/.env
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache

hosting/.env

@@ -0,0 +1 @@
+hosting.properties

hosting/build/docker-compose.yaml

@@ -0,0 +1,16 @@
+version: "3"
+
+services:
+  app-service:
+    build: ./server
+    volumes: 
+      - ./server:/app
+    environment:
+      SELF_HOSTED: 1 
+      PORT: 4002
+
+  worker-service:
+    build: ./worker
+    environment:
+      SELF_HOSTED: 1,
+      PORT: 4003

hosting/build/server

@@ -0,0 +1 @@
+../../packages/server/

hosting/build/worker

@@ -0,0 +1 @@
+../../packages/worker/

hosting/docker-compose.yaml

@@ -0,0 +1,89 @@
+version: "3"
+
+services:
+  app-service:
+    image: budibase/budibase-apps
+    ports:
+      - "${APP_PORT}:4002"
+    environment:
+      SELF_HOSTED: 1 
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      JWT_SECRET: ${JWT_SECRET}
+    depends_on:
+      - worker-service
+
+  worker-service:
+    image: budibase/budibase-worker
+    ports:
+      - "${WORKER_PORT}:4003"
+    environment:
+      SELF_HOSTED: 1,
+      DEPLOYMENT_API_KEY: ${WORKER_API_KEY}
+      PORT: 4003
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      RAW_MINIO_URL: http://minio-service:9000
+      COUCH_DB_USERNAME: ${COUCH_DB_USER}
+      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
+      RAW_COUCH_DB_URL: http://couchdb-service:5984
+      SELF_HOST_KEY: ${HOSTING_KEY}
+    depends_on:
+      - minio-service
+      - couch-init
+
+  minio-service:
+    image: minio/minio
+    volumes:
+      - minio_data:/data
+    ports:
+      - "${MINIO_PORT}:9000"
+    environment:
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+    command: server /data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  proxy-service:
+    image: envoyproxy/envoy:v1.16-latest
+    volumes:
+      - ./envoy.yaml:/etc/envoy/envoy.yaml
+    ports:
+      - "${MAIN_PORT}:10000"
+      - "9901:9901"
+    depends_on:
+      - minio-service
+      - worker-service
+      - app-service
+      - couchdb-service
+
+  couchdb-service:
+    image: apache/couchdb:3.0
+    environment:
+      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
+      - COUCHDB_USER=${COUCH_DB_USER}
+    ports:
+      - "${COUCH_DB_PORT}:5984"
+      - "4369:4369"
+      - "9100:9100"
+    volumes:
+      - couchdb_data:/couchdb
+
+  couch-init:
+    image: curlimages/curl
+    environment:
+      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
+    depends_on:
+      - couchdb-service
+    command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
+
+volumes:
+  couchdb_data:
+    driver: local
+  minio_data:
+    driver: local

hosting/envoy.yaml

@@ -0,0 +1,104 @@
+static_resources:
+  listeners:
+  - name: main_listener
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 10000 }
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress
+          codec_type: auto
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_services
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/app/" }
+                route:
+                  cluster: app-service
+                  prefix_rewrite: "/"
+              
+              # special case for when API requests are made, can just forward, not to minio    
+              - match: { prefix: "/api/" }
+                route:
+                  cluster: app-service
+
+              - match: { prefix: "/worker/" }
+                route: 
+                  cluster: worker-service
+                  prefix_rewrite: "/"
+
+              - match: { prefix: "/db/" }
+                route:
+                  cluster: couchdb-service
+                  prefix_rewrite: "/"
+
+              # minio is on the default route because this works
+              # best, minio + AWS SDK doesn't handle path proxy
+              - match: { prefix: "/" }
+                route:
+                  cluster: minio-service
+
+          http_filters:
+          - name: envoy.filters.http.router
+
+  clusters:
+  - name: app-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: app-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address: 
+                address: app-service
+                port_value: 4002
+
+  - name: minio-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: minio-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: minio-service
+                port_value: 9000
+
+  - name: worker-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: worker-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address: 
+                address: worker-service
+                port_value: 4003
+
+  - name: couchdb-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: couchdb-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: couchdb-service
+                port_value: 5984
+

hosting/hosting.properties

@@ -0,0 +1,22 @@
+# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
+MAIN_PORT=10000
+
+# Use this password when configuring your self hosting settings
+# This should be updated
+HOSTING_KEY=budibase
+
+# This section contains all secrets pertaining to the system
+# These should be updated
+JWT_SECRET=testsecret
+MINIO_ACCESS_KEY=budibase
+MINIO_SECRET_KEY=budibase
+COUCH_DB_PASSWORD=budibase
+COUCH_DB_USER=budibase
+WORKER_API_KEY=budibase
+
+# This section contains variables that do not need to be altered under normal circumstances
+APP_PORT=4002
+WORKER_PORT=4003
+MINIO_PORT=4004
+COUCH_DB_PORT=4005
+BUDIBASE_ENVIRONMENT=PRODUCTION

hosting/scripts/linux/install-docker-compose.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

hosting/scripts/linux/install-docker.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+echo "**** WARNING - not for production environments ****"
+# warning this is a convience script, for production installations install docker 
+# properly for your environment!
+curl -fsSL https://get.docker.com -o get-docker.sh
+sudo sh get-docker.sh

hosting/scripts/linux/release-to-docker-hub.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+pushd ../../build
+docker-compose build --force app-service
+docker-compose build --force worker-service
+docker tag build_app-service budibase/budibase-apps:latest
+docker push budibase/budibase-apps
+docker tag build_worker-service budibase/budibase-worker:latest
+docker push budibase/budibase-worker
+popd

hosting/start.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker-compose --env-file hosting.properties up

hosting/utils/testing.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+function dockerInstalled {
+    echo "Checking docker installation..."
+    if [ ! -x "$(command -v docker)" ]; then
+        echo "Please install docker to continue"
+	exit -1
+    fi
+}
+
+dockerInstalled
+
+source "${BASH_SOURCE%/*}/hosting.properties"
+
+opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key"
+if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then
+    opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old"
+fi
+
+docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data

package.json

@@ -32,7 +32,8 @@
     "lint:fix": "eslint --fix packages",
     "format": "prettier --write \"{,!(node_modules)/**/}*.{js,jsx,svelte}\"",
     "test:e2e": "lerna run cy:test",
-    "test:e2e:ci": "lerna run cy:ci"
+    "test:e2e:ci": "lerna run cy:ci",
+    "build:docker": "cd hosting/scripts/linux/ && ./release-to-docker-hub.sh && cd -"
   },
   "dependencies": {
     "@fortawesome/fontawesome": "^1.1.8"

packages/builder/src/builderStore/index.js

@@ -1,6 +1,8 @@
 import { getFrontendStore } from "./store/frontend"
 import { getBackendUiStore } from "./store/backend"
-import { getAutomationStore } from "./store/automation/"
+import { getAutomationStore } from "./store/automation"
+import { getHostingStore } from "./store/hosting"
+
 import { getThemeStore } from "./store/theme"
 import { derived, writable } from "svelte/store"
 import analytics from "analytics"
@@ -11,6 +13,7 @@ export const store = getFrontendStore()
 export const backendUiStore = getBackendUiStore()
 export const automationStore = getAutomationStore()
 export const themeStore = getThemeStore()
+export const hostingStore = getHostingStore()
 
 export const currentAsset = derived(store, $store => {
   const type = $store.currentFrontEndType

packages/builder/src/builderStore/store/frontend.js

@@ -3,6 +3,7 @@ import { cloneDeep } from "lodash/fp"
 import {
   allScreens,
   backendUiStore,
+  hostingStore,
   currentAsset,
   mainLayout,
   selectedComponent,
@@ -56,6 +57,7 @@ export const getFrontendStore = () => {
         hasAppPackage: true,
         appInstance: pkg.application.instance,
       }))
+      await hostingStore.actions.fetch()
       await backendUiStore.actions.database.select(pkg.application.instance)
     },
     routing: {

packages/builder/src/builderStore/store/hosting.js

@@ -0,0 +1,38 @@
+import { writable } from "svelte/store"
+import api from "../api"
+
+const INITIAL_BACKEND_UI_STATE = {
+  hostingInfo: {},
+  appUrl: "",
+}
+
+export const getHostingStore = () => {
+  const store = writable({ ...INITIAL_BACKEND_UI_STATE })
+  store.actions = {
+    fetch: async () => {
+      const responses = await Promise.all([
+        api.get("/api/hosting/"),
+        api.get("/api/hosting/urls"),
+      ])
+      const [info, urls] = await Promise.all(responses.map(resp => resp.json()))
+      store.update(state => {
+        state.hostingInfo = info
+        state.appUrl = urls.app
+        return state
+      })
+      return info
+    },
+    save: async hostingInfo => {
+      const response = await api.post("/api/hosting", hostingInfo)
+      const revision = (await response.json()).rev
+      store.update(state => {
+        state.hostingInfo = {
+          ...hostingInfo,
+          _rev: revision,
+        }
+        return state
+      })
+    },
+  }
+  return store
+}

packages/builder/src/components/automation/Shared/WebhookDisplay.svelte

@@ -1,16 +1,17 @@
 <script>
   import { notifier } from "builderStore/store/notifications"
   import { Input } from "@budibase/bbui"
-  import { store } from "builderStore"
+  import { store, hostingStore } from "builderStore"
 
   export let value
   export let production = false
 
   $: appId = $store.appId
+  $: appUrl = $hostingStore.appUrl
 
   function fullWebhookURL(uri) {
     if (production) {
-      return `https://${appId}.app.budi.live/${uri}`
+      return `${appUrl}/${uri}`
     } else {
       return `http://localhost:4001/${uri}`
     }

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -6,6 +6,7 @@
   import api from "builderStore/api"
   import { notifier } from "builderStore/store/notifications"
   import CreateWebhookDeploymentModal from "./CreateWebhookDeploymentModal.svelte"
+  import { hostingStore } from "builderStore"
 
   const DeploymentStatus = {
     SUCCESS: "SUCCESS",
@@ -35,7 +36,7 @@
   let errorReason
   let poll
   let deployments = []
-  let deploymentUrl = `https://${appId}.app.budi.live/${appId}`
+  let deploymentUrl = `${$hostingStore.appUrl}/${appId}`
 
   const formatDate = (date, format) =>
     Intl.DateTimeFormat("en-GB", DATE_OPTIONS[format]).format(date)
@@ -95,9 +96,7 @@
       <h4>Deployment History</h4>
       <div class="deploy-div">
         {#if deployments.some(deployment => deployment.status === DeploymentStatus.SUCCESS)}
-          <a target="_blank" href={`https://${appId}.app.budi.live/${appId}`}>
+          <a target="_blank" href={deploymentUrl}> View Your Deployed App → </a>
-            View Your Deployed App →
-          </a>
           <Button primary on:click={() => modal.show()}>View webhooks</Button>
         {/if}
       </div>

packages/builder/src/components/settings/ThemeEditor.svelte

@@ -1,82 +1,46 @@
 <script>
   import { themeStore } from "builderStore"
-  import { Label, DropdownMenu, Toggle, Button, Slider } from "@budibase/bbui"
+  import { Label, Toggle, Button, Slider } from "@budibase/bbui"
 
-  let anchor
-  let popover
   let showAdvanced = false
 </script>
 
-<div class="topnavitemright" on:click={popover.show} bind:this={anchor}>
+<div class="content">
-  <i class="ri-paint-fill" />
+  <div>
-</div>
+    <Toggle thin text="Dark theme" bind:checked={$themeStore.darkMode} />
-<div class="dropdown">
+  </div>
-  <DropdownMenu bind:this={popover} {anchor} align="right">
+  {#if $themeStore.darkMode && !showAdvanced}
-    <div class="content">
+    <div class="button">
-      <div>
+      <Button text on:click={() => (showAdvanced = true)}>Customise</Button>
-        <Label extraSmall grey>Theme</Label>
-        <Toggle thin text="Dark theme" bind:checked={$themeStore.darkMode} />
-      </div>
-      {#if $themeStore.darkMode && !showAdvanced}
-        <div class="button">
-          <Button text on:click={() => (showAdvanced = true)}>Customise</Button>
-        </div>
-      {/if}
-      {#if $themeStore.darkMode && showAdvanced}
-        <Slider
-          label="Hue"
-          bind:value={$themeStore.hue}
-          min="0"
-          max="360"
-          showValue />
-        <Slider
-          label="Saturation"
-          bind:value={$themeStore.saturation}
-          min="0"
-          max="100"
-          showValue />
-        <Slider
-          label="Lightness"
-          bind:value={$themeStore.lightness}
-          min="0"
-          max="32"
-          showValue />
-        <div class="button">
-          <Button text on:click={themeStore.reset}>Reset</Button>
-        </div>
-      {/if}
     </div>
-  </DropdownMenu>
+  {/if}
+  {#if $themeStore.darkMode && showAdvanced}
+    <Slider
+      label="Hue"
+      bind:value={$themeStore.hue}
+      min="0"
+      max="360"
+      showValue />
+    <Slider
+      label="Saturation"
+      bind:value={$themeStore.saturation}
+      min="0"
+      max="100"
+      showValue />
+    <Slider
+      label="Lightness"
+      bind:value={$themeStore.lightness}
+      min="0"
+      max="32"
+      showValue />
+    <div class="button">
+      <Button text on:click={themeStore.reset}>Reset</Button>
+    </div>
+  {/if}
 </div>
 
 <style>
-  .dropdown {
-    z-index: 2;
-  }
-
-  i {
-    font-size: 18px;
-    color: var(--grey-7);
-  }
-  .topnavitemright {
-    cursor: pointer;
-    color: var(--grey-7);
-    margin: 0 12px 0 0;
-    font-weight: 500;
-    font-size: 1rem;
-    display: flex;
-    flex-direction: row;
-    justify-content: center;
-    align-items: center;
-    height: 24px;
-    width: 24px;
-  }
-  .topnavitemright:hover i {
-    color: var(--ink);
-  }
-
   .content {
-    padding: var(--spacing-xl);
     display: flex;
     flex-direction: column;
     justify-content: flex-start;
@@ -84,11 +48,6 @@
     gap: var(--spacing-l);
   }
 
-  h5 {
-    margin: 0;
-    font-weight: 500;
-  }
-
   .button {
     align-self: flex-start;
   }

packages/builder/src/components/settings/ThemeEditorDropdown.svelte

@@ -0,0 +1,55 @@
+<script>
+  import { Label, DropdownMenu } from "@budibase/bbui"
+  import ThemeEditor from "./ThemeEditor.svelte"
+
+  let anchor
+  let popover
+</script>
+
+<div class="topnavitemright" on:click={popover.show} bind:this={anchor}>
+  <i class="ri-paint-fill" />
+</div>
+<div class="dropdown">
+  <DropdownMenu bind:this={popover} {anchor} align="right">
+    <div class="content">
+      <Label extraSmall grey>Theme</Label>
+      <ThemeEditor />
+    </div>
+  </DropdownMenu>
+</div>
+
+<style>
+  .dropdown {
+    z-index: 2;
+  }
+
+  i {
+    font-size: 18px;
+    color: var(--grey-7);
+  }
+  .topnavitemright {
+    cursor: pointer;
+    color: var(--grey-7);
+    margin: 0 12px 0 0;
+    font-weight: 500;
+    font-size: 1rem;
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
+    align-items: center;
+    height: 24px;
+    width: 24px;
+  }
+  .topnavitemright:hover i {
+    color: var(--ink);
+  }
+
+  h5 {
+    margin: 0;
+    font-weight: 500;
+  }
+
+  .content {
+    padding: var(--spacing-xl);
+  }
+</style>

packages/builder/src/components/start/BuilderSettingsButton.svelte

@@ -0,0 +1,32 @@
+<script>
+  import { TextButton as Button, Modal } from "@budibase/bbui"
+  import BuilderSettingsModal from "./BuilderSettingsModal.svelte"
+
+  let modal
+</script>
+
+<div>
+  <Button text on:click={modal.show}>
+    <i class="ri-settings-3-fill" />
+    <p>Settings</p>
+  </Button>
+</div>
+<Modal bind:this={modal} width="30%">
+  <BuilderSettingsModal />
+</Modal>
+
+<style>
+  div i {
+    font-size: 26px;
+    color: var(--grey-7);
+    margin-left: 12px;
+  }
+
+  div p {
+    font-family: var(--font-sans);
+    font-size: var(--font-size-s);
+    color: var(--ink);
+    font-weight: 400;
+    margin: 0 0 0 12px;
+  }
+</style>

packages/builder/src/components/start/BuilderSettingsModal.svelte

@@ -0,0 +1,72 @@
+<script>
+  import { notifier } from "builderStore/store/notifications"
+  import { hostingStore } from "builderStore"
+  import { Input, ModalContent, Toggle } from "@budibase/bbui"
+  import ThemeEditor from "components/settings/ThemeEditor.svelte"
+  import analytics from "analytics"
+  import { onMount } from "svelte"
+
+  let hostingInfo
+  let selfhosted = false
+
+  async function save() {
+    hostingInfo.type = selfhosted ? "self" : "cloud"
+    if (!selfhosted && hostingInfo._rev) {
+      hostingInfo = {
+        type: hostingInfo.type,
+        _id: hostingInfo._id,
+        _rev: hostingInfo._rev,
+      }
+    }
+    try {
+      await hostingStore.actions.save(hostingInfo)
+      notifier.success(`Settings saved.`)
+    } catch (err) {
+      notifier.danger(`Failed to update builder settings.`)
+    }
+  }
+
+  function updateSelfHosting(event) {
+    if (hostingInfo.type === "cloud" && event.target.checked) {
+      hostingInfo.hostingUrl = "localhost:10000"
+      hostingInfo.useHttps = false
+      hostingInfo.selfHostKey = "budibase"
+    }
+  }
+
+  onMount(async () => {
+    hostingInfo = await hostingStore.actions.fetch()
+    selfhosted = hostingInfo.type === "self"
+  })
+</script>
+
+<ModalContent title="Builder settings" confirmText="Save" onConfirm={save}>
+  <h5>Theme</h5>
+  <ThemeEditor />
+  <h5>Hosting</h5>
+  <p>
+    This section contains settings that relate to the deployment and hosting of
+    apps made in this builder.
+  </p>
+  <Toggle
+    thin
+    text="Self hosted"
+    on:change={updateSelfHosting}
+    bind:checked={selfhosted} />
+  {#if selfhosted}
+    <Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" />
+    <Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" />
+    <Toggle thin text="HTTPS" bind:checked={hostingInfo.useHttps} />
+  {/if}
+</ModalContent>
+
+<style>
+  h5 {
+    margin: 0;
+    font-size: 14px;
+  }
+  p {
+    margin: 0;
+    font-size: 12px;
+  }
+</style>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -1,6 +1,11 @@
 <script>
   import { writable } from "svelte/store"
-  import { store, automationStore, backendUiStore } from "builderStore"
+  import {
+    store,
+    automationStore,
+    backendUiStore,
+    hostingStore,
+  } from "builderStore"
   import { string, object } from "yup"
   import api, { get } from "builderStore/api"
   import Form from "@svelteschool/svelte-forms"
@@ -12,6 +17,7 @@
   import { fade } from "svelte/transition"
   import { post } from "builderStore/api"
   import analytics from "analytics"
+  import { onMount } from "svelte"
 
   //Move this to context="module" once svelte-forms is updated so that it can bind to stores correctly
   const createAppStore = writable({ currentStep: 0, values: {} })
@@ -23,6 +29,7 @@
   let lastApiKey
   let fetchApiKeyPromise
   const validateApiKey = async apiKey => {
+    if (isApiKeyValid) return true
     if (!apiKey) return false
 
     // make sure we only fetch once, unless API Key is changed
@@ -39,43 +46,46 @@
     return isApiKeyValid
   }
 
+  const apiValidation = {
+    apiKey: string()
+      .required("Please enter your API key.")
+      .test("valid-apikey", "This API key is invalid", validateApiKey),
+  }
+  const infoValidation = {
+    applicationName: string().required("Your application must have a name."),
+  }
+  const userValidation = {
+    email: string()
+      .email()
+      .required("Your application needs a first user."),
+    password: string().required("Please enter a password for your first user."),
+    roleId: string().required("You need to select a role for your user."),
+  }
+
   let submitting = false
   let errors = {}
   let validationErrors = {}
-  let validationSchemas = [
+  let validationSchemas = [apiValidation, infoValidation, userValidation]
-    {
-      apiKey: string()
-        .required("Please enter your API key.")
-        .test("valid-apikey", "This API key is invalid", validateApiKey),
-    },
-    {
-      applicationName: string().required("Your application must have a name."),
-    },
-    {
-      email: string()
-        .email()
-        .required("Your application needs a first user."),
-      password: string().required(
-        "Please enter a password for your first user."
-      ),
-      roleId: string().required("You need to select a role for your user."),
-    },
-  ]
 
-  let steps = [
+  function buildStep(component) {
-    {
+    return {
-      component: API,
+      component,
       errors,
-    },
+    }
-    {
+  }
-      component: Info,
+
-      errors,
+  // steps need to be initialized for cypress from the get go
-    },
+  let steps = [buildStep(API), buildStep(Info), buildStep(User)]
-    {
+
-      component: User,
+  onMount(async () => {
-      errors,
+    let hostingInfo = await hostingStore.actions.fetch()
-    },
+    // re-init the steps based on whether self hosting or cloud hosted
-  ]
+    if (hostingInfo.type === "self") {
+      isApiKeyValid = true
+      steps = [buildStep(Info), buildStep(User)]
+      validationSchemas = [infoValidation, userValidation]
+    }
+  })
 
   if (hasKey) {
     validationSchemas.shift()

packages/builder/src/components/userInterface/AppPreview/CurrentItemPreview.svelte

@@ -30,6 +30,7 @@
   }
   $: selectedComponentId = $store.selectedComponentId ?? ""
   $: previewData = {
+    appId: $store.appId,
     layout,
     screen,
     selectedComponentId,

packages/builder/src/components/userInterface/AppPreview/iframeTemplate.html

@@ -22,10 +22,11 @@
         }
 
         // Extract data from message
-        const { selectedComponentId, layout, screen, previewType } = JSON.parse(event.data)
+        const { selectedComponentId, layout, screen, previewType, appId } = JSON.parse(event.data)
 
         // Set some flags so the app knows we're in the builder
         window["##BUDIBASE_IN_BUILDER##"] = true
+        window["##BUDIBASE_APP_ID##"] = appId
         window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
         window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
         window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId

packages/builder/src/pages/[application]/_reset.svelte

@@ -2,7 +2,7 @@
   import { store, automationStore, backendUiStore } from "builderStore"
   import { Button } from "@budibase/bbui"
   import SettingsLink from "components/settings/Link.svelte"
-  import ThemeEditor from "components/settings/ThemeEditor.svelte"
+  import ThemeEditorDropdown from "components/settings/ThemeEditorDropdown.svelte"
   import FeedbackNavLink from "components/userInterface/Feedback/FeedbackNavLink.svelte"
   import { get } from "builderStore/api"
   import { isActive, goto, layout } from "@sveltech/routify"
@@ -67,7 +67,7 @@
       {/each}
     </div>
     <div class="toprightnav">
-      <ThemeEditor />
+      <ThemeEditorDropdown />
       <FeedbackNavLink />
       <div class="topnavitemright">
         <a

packages/builder/src/pages/_layout.svelte

@@ -7,6 +7,9 @@
     CommunityIcon,
     BugIcon,
   } from "components/common/Icons"
+  import BuilderSettingsButton from "components/start/BuilderSettingsButton.svelte"
+
+  let modal
 </script>
 
 <div class="root">
@@ -16,27 +19,30 @@
     </div>
 
     <div class="nav-section">
-      <Link icon={AppsIcon} title="Apps" href="/" active />
+      <div class="nav-top">
-      <Link
+        <Link icon={AppsIcon} title="Apps" href="/" active />
-        icon={HostingIcon}
+        <Link
-        title="Hosting"
+          icon={HostingIcon}
-        href="https://portal.budi.live/" />
+          title="Hosting"
-      <Link
+          href="https://portal.budi.live/" />
-        icon={DocumentationIcon}
+        <Link
-        title="Documentation"
+          icon={DocumentationIcon}
-        href="https://docs.budibase.com/" />
+          title="Documentation"
-      <Link
+          href="https://docs.budibase.com/" />
-        icon={CommunityIcon}
+        <Link
-        title="Community"
+          icon={CommunityIcon}
-        href="https://github.com/Budibase/budibase/discussions" />
+          title="Community"
-
+          href="https://github.com/Budibase/budibase/discussions" />
-      <Link
+        <Link
-        icon={BugIcon}
+          icon={BugIcon}
-        title="Raise an issue"
+          title="Raise an issue"
-        href="https://github.com/Budibase/budibase/issues/new/choose" />
+          href="https://github.com/Budibase/budibase/issues/new/choose" />
+      </div>
+      <div class="nav-bottom">
+        <BuilderSettingsButton />
+      </div>
     </div>
   </div>
-
   <div class="main">
     <slot />
   </div>
@@ -76,8 +82,10 @@
   }
 
   .nav-section {
-    margin: 20px 0px;
+    margin: 20px 0 0 0;
     display: flex;
     flex-direction: column;
+    justify-content: space-between;
+    height: 100%;
   }
 </style>

packages/client/rollup.config.js

@@ -10,7 +10,7 @@ export default {
   output: [
     {
       sourcemap: true,
-      format: "esm",
+      format: "iife",
       file: `./dist/budibase-client.js`,
     },
   ],

packages/client/src/api/api.js

@@ -1,17 +1,8 @@
-import { getAppId } from "../utils/getAppId"
-
 /**
  * API cache for cached request responses.
  */
 let cache = {}
 
-/**
- * Makes a fully formatted URL based on the SDK configuration.
- */
-const makeFullURL = path => {
-  return `/${path}`.replace("//", "/")
-}
-
 /**
  * Handler for API errors.
  */
@@ -29,7 +20,7 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
     let headers = {
       Accept: "application/json",
       "Content-Type": "application/json",
-      "x-budibase-app-id": getAppId(),
+      "x-budibase-app-id": window["##BUDIBASE_APP_ID##"],
     }
     if (!window["##BUDIBASE_IN_BUILDER##"]) {
       headers["x-budibase-type"] = "client"
@@ -82,8 +73,8 @@ const makeCachedApiCall = async params => {
  */
 const requestApiCall = method => async params => {
   const { url, cache = false } = params
-  const fullURL = makeFullURL(url)
+  const fixedUrl = `/${url}`.replace("//", "/")
-  const enrichedParams = { ...params, method, url: fullURL }
+  const enrichedParams = { ...params, method, fixedUrl }
   return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams)
 }
 

packages/client/src/index.js

@@ -7,6 +7,7 @@ const loadBudibase = () => {
   // Update builder store with any builder flags
   builderStore.set({
     inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
+    appId: window["##BUDIBASE_APP_ID##"],
     layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],

packages/client/src/sdk.js

@@ -2,7 +2,6 @@ import * as API from "./api"
 import { authStore, routeStore, screenStore, bindingStore } from "./store"
 import { styleable } from "./utils/styleable"
 import { linkable } from "./utils/linkable"
-import { getAppId } from "./utils/getAppId"
 import DataProvider from "./components/DataProvider.svelte"
 
 export default {
@@ -12,7 +11,6 @@ export default {
   screenStore,
   styleable,
   linkable,
-  getAppId,
   DataProvider,
   setBindableValue: bindingStore.actions.setBindableValue,
 }

packages/client/src/store/auth.js

@@ -1,8 +1,8 @@
 import * as API from "../api"
-import { getAppId } from "../utils/getAppId"
+import { writable, get } from "svelte/store"
-import { writable } from "svelte/store"
 import { initialise } from "./initialise"
 import { routeStore } from "./routes"
+import { builderStore } from "./builder"
 
 const createAuthStore = () => {
   const store = writable("")
@@ -25,7 +25,7 @@ const createAuthStore = () => {
   }
   const logOut = async () => {
     store.set("")
-    const appId = getAppId()
+    const appId = get(builderStore).appId
     if (appId) {
       for (let environment of ["local", "cloud"]) {
         window.document.cookie = `budibase:${appId}:${environment}=; Path=/; Expires=Thu, 01 Jan 1970 00:00:01 GMT;`

packages/client/src/store/builder.js

@@ -3,6 +3,7 @@ import { writable } from "svelte/store"
 const createBuilderStore = () => {
   const initialState = {
     inBuilder: false,
+    appId: null,
     layout: null,
     screen: null,
     selectedComponentId: null,

packages/client/src/store/screens.js

@@ -1,8 +1,7 @@
-import { writable, derived } from "svelte/store"
+import { writable, derived, get } from "svelte/store"
 import { routeStore } from "./routes"
 import { builderStore } from "./builder"
 import * as API from "../api"
-import { getAppId } from "../utils/getAppId"
 
 const createScreenStore = () => {
   const config = writable({
@@ -40,7 +39,7 @@ const createScreenStore = () => {
   )
 
   const fetchScreens = async () => {
-    const appDefinition = await API.fetchAppDefinition(getAppId())
+    const appDefinition = await API.fetchAppDefinition(get(builderStore).appId)
     config.set({
       screens: appDefinition.screens,
       layouts: appDefinition.layouts,

packages/client/src/utils/getAppId.js

@@ -1,47 +0,0 @@
-const COOKIE_SEPARATOR = ";"
-const APP_PREFIX = "app_"
-const KEY_VALUE_SPLIT = "="
-
-function confirmAppId(possibleAppId) {
-  return possibleAppId && possibleAppId.startsWith(APP_PREFIX)
-    ? possibleAppId
-    : undefined
-}
-
-function tryGetFromCookie({ cookies }) {
-  if (!cookies) {
-    return undefined
-  }
-  const cookie = cookies
-    .split(COOKIE_SEPARATOR)
-    .find(cookie => cookie.trim().startsWith("budibase:currentapp"))
-  let appId
-  if (cookie && cookie.split(KEY_VALUE_SPLIT).length === 2) {
-    appId = cookie.split("=")[1]
-  }
-  return confirmAppId(appId)
-}
-
-function tryGetFromPath() {
-  const appId = location.pathname.split("/")[1]
-  return confirmAppId(appId)
-}
-
-function tryGetFromSubdomain() {
-  const parts = window.location.host.split(".")
-  const appId = parts[1] ? parts[0] : undefined
-  return confirmAppId(appId)
-}
-
-export const getAppId = (cookies = window.document.cookie) => {
-  const functions = [tryGetFromSubdomain, tryGetFromPath, tryGetFromCookie]
-  // try getting the app Id in order
-  let appId
-  for (let func of functions) {
-    appId = func({ cookies })
-    if (appId) {
-      break
-    }
-  }
-  return appId
-}

packages/server/Dockerfile

@@ -4,12 +4,16 @@ WORKDIR /app
 
 ENV CLOUD=1
 ENV COUCH_DB_URL=https://couchdb.budi.live:5984
-env BUDIBASE_ENVIRONMENT=PRODUCTION
+ENV BUDIBASE_ENVIRONMENT=PRODUCTION
 
 # copy files and install dependencies
 COPY . ./
-RUN yarn 
+RUN yarn
 
 EXPOSE 4001
 
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
 CMD ["yarn", "run:docker"]

packages/server/scripts/createApiKeyAndAppId.js

@@ -1,56 +0,0 @@
-// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
-const dynamoClient = require("../src/db/dynamoClient")
-const env = require("../src/environment")
-
-if (process.argv[2] == null || process.argv[3] == null) {
-  console.error(
-    "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
-  )
-  process.exit(-1)
-}
-
-const FAKE_STRING = "fakestring"
-
-// set fake credentials for local dynamo to actually work
-env._set("AWS_ACCESS_KEY_ID", "KEY_ID")
-env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY")
-dynamoClient.init("http://localhost:8333")
-
-async function run() {
-  await dynamoClient.apiKeyTable.put({
-    item: {
-      pk: process.argv[2],
-      accountId: FAKE_STRING,
-      trackingId: FAKE_STRING,
-      quotaReset: Date.now() + 2592000000,
-      usageQuota: {
-        automationRuns: 0,
-        rows: 0,
-        storage: 0,
-        users: 0,
-        views: 0,
-      },
-      usageLimits: {
-        automationRuns: 10,
-        rows: 10,
-        storage: 1000,
-        users: 10,
-        views: 10,
-      },
-    },
-  })
-  await dynamoClient.apiKeyTable.put({
-    item: {
-      pk: process.argv[3],
-      apiKey: process.argv[2],
-    },
-  })
-}
-
-run()
-  .then(() => {
-    console.log("Rows should have been created.")
-  })
-  .catch(err => {
-    console.error("Cannot create rows - " + err)
-  })

packages/server/src/api/controllers/apikeys.js

@@ -3,13 +3,20 @@ const { join } = require("../../utilities/centralPath")
 const readline = require("readline")
 const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const env = require("../../environment")
+const selfhost = require("../../selfhost")
 const ENV_FILE_PATH = "/.env"
 
 exports.fetch = async function(ctx) {
   ctx.status = 200
-  ctx.body = {
+  if (env.SELF_HOSTED) {
-    budibase: env.BUDIBASE_API_KEY,
+    ctx.body = {
-    userId: env.USERID_API_KEY,
+      selfhost: await selfhost.getSelfHostAPIKey(),
+    }
+  } else {
+    ctx.body = {
+      budibase: env.BUDIBASE_API_KEY,
+      userId: env.USERID_API_KEY,
+    }
   }
 }
 

packages/server/src/api/controllers/application.js

@@ -150,6 +150,9 @@ exports.create = async function(ctx) {
     name: ctx.request.body.name,
     template: ctx.request.body.template,
     instance: instance,
+    deployment: {
+      type: "cloud",
+    },
   }
   const instanceDb = new CouchDB(appId)
   await instanceDb.put(newApplication)

packages/server/src/api/controllers/auth.js

@@ -35,8 +35,8 @@ exports.authenticate = async ctx => {
       roleId: dbUser.roleId,
       version: app.version,
     }
-    // if in cloud add the user api key
+    // if in cloud add the user api key, unless self hosted
-    if (env.CLOUD) {
+    if (env.CLOUD && !env.SELF_HOSTED) {
       const { apiKey } = await getAPIKey(ctx.user.appId)
       payload.apiKey = apiKey
     }

packages/server/src/api/controllers/deploy/Deployment.js

@@ -0,0 +1,88 @@
+const { getAppQuota } = require("./quota")
+const env = require("../../../environment")
+const newid = require("../../../db/newid")
+
+/**
+ * This is used to pass around information about the deployment that is occurring
+ */
+class Deployment {
+  constructor(appId, id = null) {
+    this.appId = appId
+    this._id = id || newid()
+  }
+
+  // purely so that we can do quota stuff outside the main deployment context
+  async init() {
+    if (!env.SELF_HOSTED) {
+      this.setQuota(await getAppQuota(this.appId))
+    }
+  }
+
+  setQuota(quota) {
+    if (!quota) {
+      return
+    }
+    this.quota = quota
+  }
+
+  getQuota() {
+    return this.quota
+  }
+
+  getAppId() {
+    return this.appId
+  }
+
+  setVerification(verification) {
+    if (!verification) {
+      return
+    }
+    this.verification = verification
+    if (this.verification.quota) {
+      this.quota = this.verification.quota
+    }
+  }
+
+  getVerification() {
+    return this.verification
+  }
+
+  setStatus(status, err = null) {
+    this.status = status
+    if (err) {
+      this.err = err
+    }
+  }
+
+  fromJSON(json) {
+    if (json.verification) {
+      this.setVerification(json.verification)
+    }
+    if (json.quota) {
+      this.setQuota(json.quota)
+    }
+    if (json.status) {
+      this.setStatus(json.status, json.err)
+    }
+  }
+
+  getJSON() {
+    const obj = {
+      _id: this._id,
+      appId: this.appId,
+      status: this.status,
+    }
+    if (this.err) {
+      obj.err = this.err
+    }
+    if (this.verification && this.verification.cfDistribution) {
+      obj.cfDistribution = this.verification.cfDistribution
+    }
+    if (this.quota) {
+      obj.quota = this.quota
+    }
+    return obj
+  }
+}
+
+module.exports = Deployment

packages/server/src/api/controllers/deploy/aws.js

@@ -1,189 +0,0 @@
-const fs = require("fs")
-const { join } = require("../../../utilities/centralPath")
-const AWS = require("aws-sdk")
-const fetch = require("node-fetch")
-const sanitize = require("sanitize-s3-objectkey")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const PouchDB = require("../../../db")
-const env = require("../../../environment")
-
-/**
- * Finalises the deployment, updating the quota for the user API key
- * The verification process returns the levels to update to.
- * Calls the "deployment-success" lambda.
- * @param {object} quota The usage quota levels returned from the verifyDeploy
- * @returns {Promise<object>} The usage has been updated against the user API key.
- */
-exports.updateDeploymentQuota = async function(quota) {
-  const DEPLOYMENT_SUCCESS_URL =
-    env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
-
-  const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      apiKey: env.BUDIBASE_API_KEY,
-      quota,
-    }),
-    headers: {
-      "Content-Type": "application/json",
-      Accept: "application/json",
-    },
-  })
-
-  if (response.status !== 200) {
-    throw new Error(`Error updating deployment quota for API Key`)
-  }
-
-  return await response.json()
-}
-
-/**
- * Verifies the users API key and
- * Verifies that the deployment fits within the quota of the user
- * Links to the "check-api-key" lambda.
- * @param {String} appId - appId being deployed
- * @param {String} appId - appId being deployed
- * @param {quota} quota - current quota being changed with this application
- */
-exports.verifyDeployment = async function({ appId, quota }) {
-  const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      apiKey: env.BUDIBASE_API_KEY,
-      appId,
-      quota,
-    }),
-  })
-
-  const json = await response.json()
-  if (json.errors) {
-    throw new Error(json.errors)
-  }
-
-  if (response.status !== 200) {
-    throw new Error(
-      `Error fetching temporary credentials for api key: ${env.BUDIBASE_API_KEY}`
-    )
-  }
-
-  // set credentials here, means any time we're verified we're ready to go
-  if (json.credentials) {
-    AWS.config.update({
-      accessKeyId: json.credentials.AccessKeyId,
-      secretAccessKey: json.credentials.SecretAccessKey,
-      sessionToken: json.credentials.SessionToken,
-    })
-  }
-
-  return json
-}
-
-const CONTENT_TYPE_MAP = {
-  html: "text/html",
-  css: "text/css",
-  js: "application/javascript",
-}
-
-/**
- * Recursively walk a directory tree and execute a callback on all files.
- * @param {String} dirPath - Directory to traverse
- * @param {Function} callback - callback to execute on files
- */
-function walkDir(dirPath, callback) {
-  for (let filename of fs.readdirSync(dirPath)) {
-    const filePath = `${dirPath}/${filename}`
-    const stat = fs.lstatSync(filePath)
-
-    if (stat.isFile()) {
-      callback(filePath)
-    } else {
-      walkDir(filePath, callback)
-    }
-  }
-}
-
-async function prepareUploadForS3({ s3Key, metadata, s3, file }) {
-  const extension = [...file.name.split(".")].pop()
-  const fileBytes = fs.readFileSync(file.path)
-
-  const upload = await s3
-    .upload({
-      // windows filepaths need to be converted to forward slashes for s3
-      Key: sanitize(s3Key).replace(/\\/g, "/"),
-      Body: fileBytes,
-      ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
-      Metadata: metadata,
-    })
-    .promise()
-
-  return {
-    size: file.size,
-    name: file.name,
-    extension,
-    url: upload.Location,
-    key: upload.Key,
-  }
-}
-
-exports.prepareUploadForS3 = prepareUploadForS3
-
-exports.uploadAppAssets = async function({ appId, bucket, accountId }) {
-  const s3 = new AWS.S3({
-    params: {
-      Bucket: bucket,
-    },
-  })
-
-  const appAssetsPath = join(budibaseAppsDir(), appId, "public")
-
-  let uploads = []
-
-  // Upload HTML and JS of the web app
-  walkDir(appAssetsPath, function(filePath) {
-    const filePathParts = filePath.split("/")
-    const appAssetUpload = prepareUploadForS3({
-      file: {
-        path: filePath,
-        name: filePathParts.pop(),
-      },
-      s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
-      s3,
-      metadata: { accountId },
-    })
-    uploads.push(appAssetUpload)
-  })
-
-  // Upload file attachments
-  const db = new PouchDB(appId)
-  let fileUploads
-  try {
-    fileUploads = await db.get("_local/fileuploads")
-  } catch (err) {
-    fileUploads = { _id: "_local/fileuploads", uploads: [] }
-  }
-
-  for (let file of fileUploads.uploads) {
-    if (file.uploaded) continue
-
-    const attachmentUpload = prepareUploadForS3({
-      file,
-      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-      s3,
-      metadata: { accountId },
-    })
-
-    uploads.push(attachmentUpload)
-
-    // mark file as uploaded
-    file.uploaded = true
-  }
-
-  db.put(fileUploads)
-
-  try {
-    return await Promise.all(uploads)
-  } catch (err) {
-    console.error("Error uploading budibase app assets to s3", err)
-    throw err
-  }
-}

packages/server/src/api/controllers/deploy/awsDeploy.js

@@ -0,0 +1,85 @@
+const AWS = require("aws-sdk")
+const fetch = require("node-fetch")
+const env = require("../../../environment")
+const {
+  deployToObjectStore,
+  performReplication,
+  fetchCredentials,
+} = require("./utils")
+
+/**
+ * Verifies the users API key and
+ * Verifies that the deployment fits within the quota of the user
+ * Links to the "check-api-key" lambda.
+ * @param {object} deployment - information about the active deployment, including the appId and quota.
+ */
+exports.preDeployment = async function(deployment) {
+  const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, {
+    apiKey: env.BUDIBASE_API_KEY,
+    appId: deployment.getAppId(),
+    quota: deployment.getQuota(),
+  })
+
+  // set credentials here, means any time we're verified we're ready to go
+  if (json.credentials) {
+    AWS.config.update({
+      accessKeyId: json.credentials.AccessKeyId,
+      secretAccessKey: json.credentials.SecretAccessKey,
+      sessionToken: json.credentials.SessionToken,
+    })
+  }
+
+  return json
+}
+
+/**
+ * Finalises the deployment, updating the quota for the user API key
+ * The verification process returns the levels to update to.
+ * Calls the "deployment-success" lambda.
+ * @param {object} deployment information about the active deployment, including the quota info.
+ * @returns {Promise<object>} The usage has been updated against the user API key.
+ */
+exports.postDeployment = async function(deployment) {
+  const DEPLOYMENT_SUCCESS_URL =
+    env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
+
+  const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      apiKey: env.BUDIBASE_API_KEY,
+      quota: deployment.getQuota(),
+    }),
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error updating deployment quota for API Key`)
+  }
+
+  return await response.json()
+}
+
+exports.deploy = async function(deployment) {
+  const appId = deployment.getAppId()
+  const { bucket, accountId } = deployment.getVerification()
+  const metadata = { accountId }
+  const s3Client = new AWS.S3({
+    params: {
+      Bucket: bucket,
+    },
+  })
+  await deployToObjectStore(appId, s3Client, metadata)
+}
+
+exports.replicateDb = async function(deployment) {
+  const appId = deployment.getAppId()
+  const verification = deployment.getVerification()
+  return performReplication(
+    appId,
+    verification.couchDbSession,
+    env.DEPLOYMENT_DB_URL
+  )
+}

packages/server/src/api/controllers/deploy/index.js

@@ -1,23 +1,20 @@
-const CouchDB = require("pouchdb")
 const PouchDB = require("../../../db")
+const Deployment = require("./Deployment")
 const {
-  uploadAppAssets,
+  getHostingInfo,
-  verifyDeployment,
+  HostingTypes,
-  updateDeploymentQuota,
+} = require("../../../utilities/builder/hosting")
-} = require("./aws")
-const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
-const newid = require("../../../db/newid")
-const env = require("../../../environment")
-
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
-
 const DeploymentStatus = {
   SUCCESS: "SUCCESS",
   PENDING: "PENDING",
   FAILURE: "FAILURE",
 }
 
+// default to AWS deployment, this will be updated before use (if required)
+let deploymentService = require("./awsDeploy")
+
 // checks that deployments are in a good state, any pending will be updated
 async function checkAllDeployments(deployments) {
   let updated = false
@@ -27,7 +24,7 @@ async function checkAllDeployments(deployments) {
       deployment.status === DeploymentStatus.PENDING &&
       Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS
     ) {
-      deployment.status = status
+      deployment.status = DeploymentStatus.FAILURE
       deployment.err = "Timed out"
       updated = true
     }
@@ -35,54 +32,10 @@ async function checkAllDeployments(deployments) {
   return { updated, deployments }
 }
 
-function replicate(local, remote) {
-  return new Promise((resolve, reject) => {
-    const replication = local.sync(remote)
-
-    replication.on("complete", () => resolve())
-    replication.on("error", err => reject(err))
-  })
-}
-
-async function replicateCouch({ appId, session }) {
-  const localDb = new PouchDB(appId)
-  const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, {
-    fetch: function(url, opts) {
-      opts.headers.set("Cookie", `${session};`)
-      return CouchDB.fetch(url, opts)
-    },
-  })
-
-  return replicate(localDb, remoteDb)
-}
-
-async function getCurrentInstanceQuota(appId) {
-  const db = new PouchDB(appId)
-
-  const rows = await db.allDocs({
-    startkey: DocumentTypes.ROW + SEPARATOR,
-    endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
-  })
-
-  const users = await db.allDocs({
-    startkey: DocumentTypes.USER + SEPARATOR,
-    endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
-  })
-
-  const existingRows = rows.rows.length
-  const existingUsers = users.rows.length
-
-  const designDoc = await db.get("_design/database")
-
-  return {
-    rows: existingRows,
-    users: existingUsers,
-    views: Object.keys(designDoc.views).length,
-  }
-}
-
 async function storeLocalDeploymentHistory(deployment) {
-  const db = new PouchDB(deployment.appId)
+  const appId = deployment.getAppId()
+  const deploymentJSON = deployment.getJSON()
+  const db = new PouchDB(appId)
 
   let deploymentDoc
   try {
@@ -91,7 +44,7 @@ async function storeLocalDeploymentHistory(deployment) {
     deploymentDoc = { _id: "_local/deployments", history: {} }
   }
 
-  const deploymentId = deployment._id || newid()
+  const deploymentId = deploymentJSON._id
 
   // first time deployment
   if (!deploymentDoc.history[deploymentId])
@@ -99,56 +52,42 @@ async function storeLocalDeploymentHistory(deployment) {
 
   deploymentDoc.history[deploymentId] = {
     ...deploymentDoc.history[deploymentId],
-    ...deployment,
+    ...deploymentJSON,
     updatedAt: Date.now(),
   }
 
   await db.put(deploymentDoc)
-  return {
+  deployment.fromJSON(deploymentDoc.history[deploymentId])
-    _id: deploymentId,
+  return deployment
-    ...deploymentDoc.history[deploymentId],
-  }
 }
 
-async function deployApp({ appId, deploymentId }) {
+async function deployApp(deployment) {
+  const appId = deployment.getAppId()
   try {
-    const instanceQuota = await getCurrentInstanceQuota(appId)
+    await deployment.init()
-    const verification = await verifyDeployment({
+    deployment.setVerification(
-      appId,
+      await deploymentService.preDeployment(deployment)
-      quota: instanceQuota,
+    )
-    })
 
-    console.log(`Uploading assets for appID ${appId} assets to s3..`)
+    console.log(`Uploading assets for appID ${appId}..`)
 
-    await uploadAppAssets({
+    await deploymentService.deploy(deployment)
-      appId,
-      ...verification,
-    })
 
-    // replicate the DB to the couchDB cluster in prod
+    // replicate the DB to the main couchDB cluster
-    console.log("Replicating local PouchDB to remote..")
+    console.log("Replicating local PouchDB to CouchDB..")
-    await replicateCouch({
+    await deploymentService.replicateDb(deployment)
-      appId,
-      session: verification.couchDbSession,
-    })
 
-    await updateDeploymentQuota(verification.quota)
+    await deploymentService.postDeployment(deployment)
 
-    await storeLocalDeploymentHistory({
+    deployment.setStatus(DeploymentStatus.SUCCESS)
-      _id: deploymentId,
+    await storeLocalDeploymentHistory(deployment)
-      appId,
-      cfDistribution: verification.cfDistribution,
-      quota: verification.quota,
-      status: DeploymentStatus.SUCCESS,
-    })
   } catch (err) {
-    await storeLocalDeploymentHistory({
+    deployment.setStatus(DeploymentStatus.FAILURE, err.message)
-      _id: deploymentId,
+    await storeLocalDeploymentHistory(deployment)
-      appId,
+    throw {
-      status: DeploymentStatus.FAILURE,
+      ...err,
-      err: err.message,
+      message: `Deployment Failed: ${err.message}`,
-    })
+    }
-    throw new Error(`Deployment Failed: ${err.message}`)
   }
 }
 
@@ -183,15 +122,17 @@ exports.deploymentProgress = async function(ctx) {
 }
 
 exports.deployApp = async function(ctx) {
-  const deployment = await storeLocalDeploymentHistory({
+  // start by checking whether to deploy local or to cloud
-    appId: ctx.user.appId,
+  const hostingInfo = await getHostingInfo()
-    status: DeploymentStatus.PENDING,
+  deploymentService =
-  })
+    hostingInfo.type === HostingTypes.CLOUD
+      ? require("./awsDeploy")
+      : require("./selfDeploy")
+  let deployment = new Deployment(ctx.user.appId)
+  deployment.setStatus(DeploymentStatus.PENDING)
+  deployment = await storeLocalDeploymentHistory(deployment)
 
-  deployApp({
+  await deployApp(deployment)
-    ...ctx.user,
-    deploymentId: deployment._id,
-  })
 
   ctx.body = deployment
 }

packages/server/src/api/controllers/deploy/quota.js

@@ -0,0 +1,27 @@
+const PouchDB = require("../../../db")
+const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
+
+exports.getAppQuota = async function(appId) {
+  const db = new PouchDB(appId)
+
+  const rows = await db.allDocs({
+    startkey: DocumentTypes.ROW + SEPARATOR,
+    endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
+  })
+
+  const users = await db.allDocs({
+    startkey: DocumentTypes.USER + SEPARATOR,
+    endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
+  })
+
+  const existingRows = rows.rows.length
+  const existingUsers = users.rows.length
+
+  const designDoc = await db.get("_design/database")
+
+  return {
+    rows: existingRows,
+    users: existingUsers,
+    views: Object.keys(designDoc.views).length,
+  }
+}

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -0,0 +1,69 @@
+const AWS = require("aws-sdk")
+const {
+  deployToObjectStore,
+  performReplication,
+  fetchCredentials,
+} = require("./utils")
+const {
+  getWorkerUrl,
+  getCouchUrl,
+  getMinioUrl,
+  getSelfHostKey,
+} = require("../../../utilities/builder/hosting")
+
+exports.preDeployment = async function() {
+  const url = `${await getWorkerUrl()}/api/deploy`
+  try {
+    const json = await fetchCredentials(url, {
+      selfHostKey: await getSelfHostKey(),
+    })
+
+    // response contains:
+    // couchDbSession, bucket, objectStoreSession
+
+    // set credentials here, means any time we're verified we're ready to go
+    if (json.objectStoreSession) {
+      AWS.config.update({
+        accessKeyId: json.objectStoreSession.accessKeyId,
+        secretAccessKey: json.objectStoreSession.secretAccessKey,
+      })
+    }
+
+    return json
+  } catch (err) {
+    throw {
+      message: "Unauthorised to deploy, check self hosting key",
+      status: 401,
+    }
+  }
+}
+
+exports.postDeployment = async function() {
+  // we don't actively need to do anything after deployment in self hosting
+}
+
+exports.deploy = async function(deployment) {
+  const appId = deployment.getAppId()
+  const verification = deployment.getVerification()
+  const objClient = new AWS.S3({
+    endpoint: await getMinioUrl(),
+    s3ForcePathStyle: true, // needed with minio?
+    signatureVersion: "v4",
+    params: {
+      Bucket: verification.bucket,
+    },
+  })
+  // no metadata, aws has account ID in metadata
+  const metadata = {}
+  await deployToObjectStore(appId, objClient, metadata)
+}
+
+exports.replicateDb = async function(deployment) {
+  const appId = deployment.getAppId()
+  const verification = deployment.getVerification()
+  return performReplication(
+    appId,
+    verification.couchDbSession,
+    await getCouchUrl()
+  )
+}

packages/server/src/api/controllers/deploy/utils.js

@@ -0,0 +1,131 @@
+const fs = require("fs")
+const sanitize = require("sanitize-s3-objectkey")
+const { walkDir } = require("../../../utilities")
+const { join } = require("../../../utilities/centralPath")
+const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
+const fetch = require("node-fetch")
+const PouchDB = require("../../../db")
+const CouchDB = require("pouchdb")
+
+const CONTENT_TYPE_MAP = {
+  html: "text/html",
+  css: "text/css",
+  js: "application/javascript",
+}
+
+exports.fetchCredentials = async function(url, body) {
+  const response = await fetch(url, {
+    method: "POST",
+    body: JSON.stringify(body),
+    headers: { "Content-Type": "application/json" },
+  })
+
+  const json = await response.json()
+  if (json.errors) {
+    throw new Error(json.errors)
+  }
+
+  if (response.status !== 200) {
+    throw new Error(
+      `Error fetching temporary credentials: ${JSON.stringify(json)}`
+    )
+  }
+
+  return json
+}
+
+exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
+  const extension = [...file.name.split(".")].pop()
+  const fileBytes = fs.readFileSync(file.path)
+
+  const upload = await client
+    .upload({
+      // windows file paths need to be converted to forward slashes for s3
+      Key: sanitize(s3Key).replace(/\\/g, "/"),
+      Body: fileBytes,
+      ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
+      Metadata: metadata,
+    })
+    .promise()
+
+  return {
+    size: file.size,
+    name: file.name,
+    extension,
+    url: upload.Location,
+    key: upload.Key,
+  }
+}
+
+exports.deployToObjectStore = async function(appId, objectClient, metadata) {
+  const appAssetsPath = join(budibaseAppsDir(), appId, "public")
+
+  let uploads = []
+
+  // Upload HTML, CSS and JS for each page of the web app
+  walkDir(appAssetsPath, function(filePath) {
+    const filePathParts = filePath.split("/")
+    const appAssetUpload = exports.prepareUpload({
+      file: {
+        path: filePath,
+        name: filePathParts.pop(),
+      },
+      s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
+      client: objectClient,
+      metadata,
+    })
+    uploads.push(appAssetUpload)
+  })
+
+  // Upload file attachments
+  const db = new PouchDB(appId)
+  let fileUploads
+  try {
+    fileUploads = await db.get("_local/fileuploads")
+  } catch (err) {
+    fileUploads = { _id: "_local/fileuploads", uploads: [] }
+  }
+
+  for (let file of fileUploads.uploads) {
+    if (file.uploaded) continue
+
+    const attachmentUpload = exports.prepareUpload({
+      file,
+      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
+      client: objectClient,
+      metadata,
+    })
+
+    uploads.push(attachmentUpload)
+
+    // mark file as uploaded
+    file.uploaded = true
+  }
+
+  db.put(fileUploads)
+
+  try {
+    return await Promise.all(uploads)
+  } catch (err) {
+    console.error("Error uploading budibase app assets to s3", err)
+    throw err
+  }
+}
+
+exports.performReplication = (appId, session, dbUrl) => {
+  return new Promise((resolve, reject) => {
+    const local = new PouchDB(appId)
+
+    const remote = new CouchDB(`${dbUrl}/${appId}`, {
+      fetch: function(url, opts) {
+        opts.headers.set("Cookie", `${session};`)
+        return CouchDB.fetch(url, opts)
+      },
+    })
+
+    const replication = local.sync(remote)
+
+    replication.on("complete", () => resolve())
+    replication.on("error", err => reject(err))
+  })
+}

packages/server/src/api/controllers/hosting.js

@@ -0,0 +1,39 @@
+const CouchDB = require("../../db")
+const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
+const {
+  getHostingInfo,
+  HostingTypes,
+  getAppUrl,
+} = require("../../utilities/builder/hosting")
+
+exports.fetchInfo = async ctx => {
+  ctx.body = {
+    types: Object.values(HostingTypes),
+  }
+}
+
+exports.save = async ctx => {
+  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const { type } = ctx.request.body
+  if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
+    ctx.body = await db.remove({
+      ...ctx.request.body,
+      _id: HOSTING_DOC,
+    })
+  } else {
+    ctx.body = await db.put({
+      ...ctx.request.body,
+      _id: HOSTING_DOC,
+    })
+  }
+}
+
+exports.fetch = async ctx => {
+  ctx.body = await getHostingInfo()
+}
+
+exports.fetchUrls = async ctx => {
+  ctx.body = {
+    app: await getAppUrl(ctx.appId),
+  }
+}

packages/server/src/api/controllers/static/index.js

@@ -6,7 +6,7 @@ const fetch = require("node-fetch")
 const fs = require("fs-extra")
 const uuid = require("uuid")
 const AWS = require("aws-sdk")
-const { prepareUploadForS3 } = require("../deploy/aws")
+const { prepareUpload } = require("../deploy/utils")
 const handlebars = require("handlebars")
 const {
   budibaseAppsDir,
@@ -17,6 +17,15 @@ const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
 const fileProcessor = require("../../../utilities/fileProcessor")
 const env = require("../../../environment")
 
+function objectStoreUrl() {
+  if (env.SELF_HOSTED) {
+    // can use a relative url for this as all goes through the proxy (this is hosted in minio)
+    return `/app-assets/assets`
+  } else {
+    return "https://cdn.app.budi.live/assets"
+  }
+}
+
 // this was the version before we started versioning the component library
 const COMP_LIB_BASE_APP_VERSION = "0.2.5"
 
@@ -53,7 +62,7 @@ exports.uploadFile = async function(ctx) {
       const fileExtension = [...file.name.split(".")].pop()
       const processedFileName = `${uuid.v4()}.${fileExtension}`
 
-      return prepareUploadForS3({
+      return prepareUpload({
         file,
         s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
         s3,
@@ -148,6 +157,7 @@ exports.serveApp = async function(ctx) {
     title: appInfo.name,
     production: env.CLOUD,
     appId: ctx.params.appId,
+    objectStoreUrl: objectStoreUrl(),
   })
 
   const template = handlebars.compile(
@@ -158,6 +168,7 @@ exports.serveApp = async function(ctx) {
     head,
     body: html,
     style: css.code,
+    appId: ctx.params.appId,
   })
 }
 
@@ -166,8 +177,9 @@ exports.serveAttachment = async function(ctx) {
   const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
 
   // Serve from CloudFront
+  // TODO: need to replace this with link to self hosted object store
   if (env.CLOUD) {
-    const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}`
+    const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file)
     const response = await fetch(S3_URL)
     const body = await response.text()
     ctx.set("Content-Type", response.headers.get("Content-Type"))
@@ -213,7 +225,13 @@ exports.serveComponentLibrary = async function(ctx) {
       componentLib += `-${COMP_LIB_BASE_APP_VERSION}`
     }
     const S3_URL = encodeURI(
-      `https://${appId}.app.budi.live/assets/${componentLib}/${ctx.query.library}/dist/index.js`
+      join(
+        objectStoreUrl(appId),
+        componentLib,
+        ctx.query.library,
+        "dist",
+        "index.js"
+      )
     )
     const response = await fetch(S3_URL)
     const body = await response.text()
@@ -222,5 +240,5 @@ exports.serveComponentLibrary = async function(ctx) {
     return
   }
 
-  await send(ctx, "/index.js", { root: componentLibraryPath })
+  await send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
 }

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -4,12 +4,11 @@
 
   export let appId
   export let production
-
+  export let objectStoreUrl
-  export const PRODUCTION_ASSETS_URL = `https://${appId}.app.budi.live`
 
   function publicPath(path) {
     if (production) {
-      return `${PRODUCTION_ASSETS_URL}/assets/${appId}/${path}`
+      return `${objectStoreUrl}/${appId}/${path}`
     }
 
     return `/assets/${path}`

packages/server/src/api/controllers/static/templates/app.hbs

@@ -5,6 +5,9 @@
     {{{head}}}   
   </head>
 
-  {{{body}}}
+  <script>
+    window["##BUDIBASE_APP_ID##"] = "{{appId}}"
+  </script>
 
+  {{{body}}}
 </html>

packages/server/src/api/routes/hosting.js

@@ -0,0 +1,14 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/hosting")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("../../utilities/security/permissions")
+
+const router = Router()
+
+router
+  .get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo)
+  .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
+  .get("/api/hosting", authorized(BUILDER), controller.fetch)
+  .post("/api/hosting", authorized(BUILDER), controller.save)
+
+module.exports = router

packages/server/src/api/routes/index.js

@@ -17,6 +17,7 @@ const templatesRoutes = require("./templates")
 const analyticsRoutes = require("./analytics")
 const routingRoutes = require("./routing")
 const permissionRoutes = require("./permission")
+const hostingRoutes = require("./hosting")
 
 exports.mainRoutes = [
   deployRoutes,
@@ -34,6 +35,7 @@ exports.mainRoutes = [
   webhookRoutes,
   routingRoutes,
   permissionRoutes,
+  hostingRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/app.js

@@ -9,6 +9,7 @@ const env = require("./environment")
 const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
+const selfhost = require("./selfhost")
 
 const app = new Koa()
 
@@ -49,9 +50,12 @@ destroyable(server)
 
 server.on("close", () => console.log("Server Closed"))
 
-module.exports = server.listen(env.PORT || 4001, () => {
+module.exports = server.listen(env.PORT || 4001, async () => {
   console.log(`Budibase running on ${JSON.stringify(server.address())}`)
   automations.init()
+  if (env.SELF_HOSTED) {
+    await selfhost.init()
+  }
 })
 
 process.on("uncaughtException", err => {

packages/server/src/constants/index.js

@@ -41,3 +41,5 @@ const USERS_TABLE_SCHEMA = {
 
 exports.AuthTypes = AuthTypes
 exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA
+exports.BUILDER_CONFIG_DB = "builder-config-db"
+exports.HOSTING_DOC = "hosting-doc"

packages/server/src/constants/layouts.js

@@ -1,3 +1,5 @@
+const { getLogoUrl } = require("../utilities")
+
 const BASE_LAYOUT_PROP_IDS = {
   PRIVATE: "layout_private_master",
   PUBLIC: "layout_public_master",
@@ -107,8 +109,7 @@ const BASE_LAYOUTS = [
                 active: {},
                 selected: {},
               },
-              logoUrl:
+              logoUrl: getLogoUrl(),
-                "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg",
               title: "",
               backgroundColor: "",
               color: "",

packages/server/src/constants/screens.js

@@ -1,5 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
+const { getLogoUrl } = require("../utilities")
 
 exports.createHomeScreen = () => ({
   description: "",
@@ -138,8 +139,7 @@ exports.createLoginScreen = app => ({
           active: {},
           selected: {},
         },
-        logo:
+        logo: getLogoUrl(),
-          "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg",
         title: `Log in to ${app.name}`,
         buttonText: "Log In",
         _children: [],

packages/server/src/environment.js

@@ -27,6 +27,7 @@ module.exports = {
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   CLOUD: process.env.CLOUD,
+  SELF_HOSTED: process.env.SELF_HOSTED,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   AWS_REGION: process.env.AWS_REGION,
   DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,

packages/server/src/middleware/authorized.js

@@ -7,7 +7,7 @@ const {
   doesHavePermission,
 } = require("../utilities/security/permissions")
 const env = require("../environment")
-const { apiKeyTable } = require("../db/dynamoClient")
+const { isAPIKeyValid } = require("../utilities/security/apikey")
 const { AuthTypes } = require("../constants")
 
 const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER]
@@ -21,11 +21,7 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
   }
   if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
     // api key header passed by external webhook
-    const apiKeyInfo = await apiKeyTable.get({
+    if (await isAPIKeyValid(ctx.headers["x-api-key"])) {
-      primary: ctx.headers["x-api-key"],
-    })
-
-    if (apiKeyInfo) {
       ctx.auth = {
         authenticated: AuthTypes.EXTERNAL,
         apiKey: ctx.headers["x-api-key"],

packages/server/src/middleware/usageQuota.js

@@ -43,6 +43,10 @@ module.exports = async (ctx, next) => {
       return
     }
   }
+  // if running in builder or a self hosted cloud usage quotas should not be executed
+  if (!env.CLOUD || env.SELF_HOSTED) {
+    return next()
+  }
   // update usage for uploads to be the total size
   if (property === usageQuota.Properties.UPLOAD) {
     const files =
@@ -51,9 +55,6 @@ module.exports = async (ctx, next) => {
         : [ctx.request.files.file]
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
-  if (!env.CLOUD) {
-    return next()
-  }
   try {
     await usageQuota.update(ctx.auth.apiKey, property, usage)
     return next()

packages/server/src/selfhost/README.md

@@ -0,0 +1,7 @@
+### Self hosting
+This directory contains utilities that are needed for self hosted platforms to operate.
+These will mostly be utilities, necessary to the operation of the server e.g. storing self
+hosting specific options and attributes to CouchDB.
+
+All the internal operations should be exposed through the `index.js` so importing
+the self host directory should give you everything you need.

packages/server/src/selfhost/index.js

@@ -0,0 +1,44 @@
+const CouchDB = require("../db")
+const env = require("../environment")
+const newid = require("../db/newid")
+
+const SELF_HOST_DB = "self-host-db"
+const SELF_HOST_DOC = "self-host-info"
+
+async function createSelfHostDB(db) {
+  await db.put({
+    _id: "_design/database",
+    views: {},
+  })
+  const selfHostInfo = {
+    _id: SELF_HOST_DOC,
+    apiKeyId: newid(),
+  }
+  await db.put(selfHostInfo)
+  return selfHostInfo
+}
+
+exports.init = async () => {
+  if (!env.SELF_HOSTED) {
+    return
+  }
+  const db = new CouchDB(SELF_HOST_DB)
+  try {
+    await db.get(SELF_HOST_DOC)
+  } catch (err) {
+    // failed to retrieve
+    if (err.status === 404) {
+      await createSelfHostDB(db)
+    }
+  }
+}
+
+exports.getSelfHostInfo = async () => {
+  const db = new CouchDB(SELF_HOST_DB)
+  return db.get(SELF_HOST_DOC)
+}
+
+exports.getSelfHostAPIKey = async () => {
+  const info = await exports.getSelfHostInfo()
+  return info ? info.apiKeyId : null
+}

packages/server/src/utilities/builder/hosting.js

@@ -0,0 +1,83 @@
+const CouchDB = require("../../db")
+const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
+
+const PROD_HOSTING_URL = "app.budi.live"
+
+function getProtocol(hostingInfo) {
+  return hostingInfo.useHttps ? "https://" : "http://"
+}
+
+async function getURLWithPath(pathIfSelfHosted) {
+  const hostingInfo = await exports.getHostingInfo()
+  const protocol = getProtocol(hostingInfo)
+  const path =
+    hostingInfo.type === exports.HostingTypes.SELF ? pathIfSelfHosted : ""
+  return `${protocol}${hostingInfo.hostingUrl}${path}`
+}
+
+exports.HostingTypes = {
+  CLOUD: "cloud",
+  SELF: "self",
+}
+
+exports.getHostingInfo = async () => {
+  const db = new CouchDB(BUILDER_CONFIG_DB)
+  let doc
+  try {
+    doc = await db.get(HOSTING_DOC)
+  } catch (err) {
+    // don't write this doc, want to be able to update these default props
+    // for our servers with a new release without needing to worry about state of
+    // PouchDB in peoples installations
+    doc = {
+      _id: HOSTING_DOC,
+      type: exports.HostingTypes.CLOUD,
+      hostingUrl: PROD_HOSTING_URL,
+      selfHostKey: "",
+      templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
+      useHttps: true,
+    }
+  }
+  return doc
+}
+
+exports.getAppUrl = async appId => {
+  const hostingInfo = await exports.getHostingInfo()
+  const protocol = getProtocol(hostingInfo)
+  let url
+  if (hostingInfo.type === exports.HostingTypes.CLOUD) {
+    url = `${protocol}${appId}.${hostingInfo.hostingUrl}`
+  } else {
+    url = `${protocol}${hostingInfo.hostingUrl}/app`
+  }
+  return url
+}
+
+exports.getWorkerUrl = async () => {
+  return getURLWithPath("/worker")
+}
+
+exports.getMinioUrl = async () => {
+  return getURLWithPath("/")
+}
+
+exports.getCouchUrl = async () => {
+  return getURLWithPath("/db")
+}
+
+exports.getSelfHostKey = async () => {
+  const hostingInfo = await exports.getHostingInfo()
+  return hostingInfo.selfHostKey
+}
+
+exports.getTemplatesUrl = async (appId, type, name) => {
+  const hostingInfo = await exports.getHostingInfo()
+  const protocol = getProtocol(hostingInfo)
+  let path
+  if (type && name) {
+    path = `templates/type/${name}.tar.gz`
+  } else {
+    path = "manifest.json"
+  }
+  return `${protocol}${hostingInfo.templatesUrl}/${path}`
+}

packages/server/src/utilities/index.js

@@ -168,3 +168,7 @@ exports.coerceRowValues = (row, table) => {
   }
   return clonedRow
 }
+
+exports.getLogoUrl = () => {
+  return "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg"
+}

packages/server/src/utilities/security/apikey.js

@@ -0,0 +1,23 @@
+const { apiKeyTable } = require("../../db/dynamoClient")
+const env = require("../../environment")
+const { getSelfHostAPIKey } = require("../../selfhost")
+
+/**
+ * This file purely exists so that we can centralise all logic pertaining to API keys, as their usage differs
+ * in our Cloud environment versus self hosted.
+ */
+
+exports.isAPIKeyValid = async apiKeyId => {
+  if (env.CLOUD && !env.SELF_HOSTED) {
+    let apiKeyInfo = await apiKeyTable.get({
+      primary: apiKeyId,
+    })
+    return apiKeyInfo != null
+  }
+  if (env.SELF_HOSTED) {
+    const selfHostKey = await getSelfHostAPIKey()
+    // if the api key supplied is correct then return structure similar
+    return apiKeyId === selfHostKey ? { pk: apiKeyId } : null
+  }
+  return false
+}

packages/worker/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+.env

packages/worker/Dockerfile

@@ -0,0 +1,15 @@
+FROM node:12-alpine
+
+WORKDIR /app
+
+# copy files and install dependencies
+COPY . ./
+RUN yarn
+
+EXPOSE 4001
+
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
+CMD ["yarn", "run:docker"]

packages/worker/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@budibase/deployment",
+  "email": "hi@budibase.com",
+  "version": "0.3.8",
+  "description": "Budibase Deployment Server",
+  "main": "src/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Budibase/budibase.git"
+  },
+  "keywords": [
+    "budibase"
+  ],
+  "scripts": {
+    "run:docker": "node src/index.js"
+  },
+  "author": "Budibase",
+  "license": "AGPL-3.0-or-later",
+  "dependencies": {
+    "@koa/router": "^8.0.0",
+    "aws-sdk": "^2.811.0",
+    "got": "^11.8.1",
+    "joi": "^17.2.1",
+    "koa": "^2.7.0",
+    "koa-body": "^4.2.0",
+    "koa-compress": "^4.0.1",
+    "koa-pino-logger": "^3.0.0",
+    "koa-send": "^5.0.0",
+    "koa-session": "^5.12.0",
+    "koa-static": "^5.0.0",
+    "pino-pretty": "^4.0.0",
+    "server-destroy": "^1.0.1"
+  }
+}

packages/worker/src/api/controllers/deploy.js

@@ -0,0 +1,92 @@
+const env = require("../../environment")
+const got = require("got")
+const AWS = require("aws-sdk")
+
+const APP_BUCKET = "app-assets"
+// this doesn't matter in self host
+const REGION = "eu-west-1"
+const PUBLIC_READ_POLICY = {
+  Version: "2012-10-17",
+  Statement: [
+    {
+      Effect: "Allow",
+      Principal: {
+        AWS: ["*"],
+      },
+      Action: "s3:GetObject",
+      Resource: [`arn:aws:s3:::${APP_BUCKET}/*`],
+    },
+  ],
+}
+
+async function getCouchSession() {
+  // fetch session token for the api user
+  const session = await got.post(`${env.RAW_COUCH_DB_URL}/_session`, {
+    responseType: "json",
+    credentials: "include",
+    json: {
+      username: env.COUCH_DB_USERNAME,
+      password: env.COUCH_DB_PASSWORD,
+    },
+  })
+
+  const cookie = session.headers["set-cookie"][0]
+  // Get the session cookie value only
+  return cookie.split(";")[0]
+}
+
+async function getMinioSession() {
+  AWS.config.update({
+    accessKeyId: env.MINIO_ACCESS_KEY,
+    secretAccessKey: env.MINIO_SECRET_KEY,
+  })
+
+  // make sure the bucket exists
+  const objClient = new AWS.S3({
+    endpoint: env.RAW_MINIO_URL,
+    region: REGION,
+    s3ForcePathStyle: true, // needed with minio?
+    params: {
+      Bucket: APP_BUCKET,
+    },
+  })
+  // make sure the bucket exists
+  try {
+    await objClient
+      .headBucket({
+        Bucket: APP_BUCKET,
+      })
+      .promise()
+  } catch (err) {
+    // bucket doesn't exist create it
+    if (err.statusCode === 404) {
+      await objClient
+        .createBucket({
+          Bucket: APP_BUCKET,
+        })
+        .promise()
+    } else {
+      throw err
+    }
+  }
+  // always make sure policy is correct
+  await objClient
+    .putBucketPolicy({
+      Bucket: APP_BUCKET,
+      Policy: JSON.stringify(PUBLIC_READ_POLICY),
+    })
+    .promise()
+  // Ideally want to send back some pre-signed URLs for files that are to be uploaded
+  return {
+    accessKeyId: env.MINIO_ACCESS_KEY,
+    secretAccessKey: env.MINIO_SECRET_KEY,
+  }
+}
+
+exports.deploy = async ctx => {
+  ctx.body = {
+    couchDbSession: await getCouchSession(),
+    bucket: APP_BUCKET,
+    objectStoreSession: await getMinioSession(),
+  }
+}

packages/worker/src/api/index.js

@@ -0,0 +1,45 @@
+const Router = require("@koa/router")
+const compress = require("koa-compress")
+const zlib = require("zlib")
+const { routes } = require("./routes")
+
+const router = new Router()
+
+router
+  .use(
+    compress({
+      threshold: 2048,
+      gzip: {
+        flush: zlib.Z_SYNC_FLUSH,
+      },
+      deflate: {
+        flush: zlib.Z_SYNC_FLUSH,
+      },
+      br: false,
+    })
+  )
+  .use("/health", ctx => (ctx.status = 200))
+
+// error handling middleware
+router.use(async (ctx, next) => {
+  try {
+    await next()
+  } catch (err) {
+    ctx.log.error(err)
+    ctx.status = err.status || err.statusCode || 500
+    ctx.body = {
+      message: err.message,
+      status: ctx.status,
+    }
+  }
+})
+
+router.get("/health", ctx => (ctx.status = 200))
+
+// authenticated routes
+for (let route of routes) {
+  router.use(route.routes())
+  router.use(route.allowedMethods())
+}
+
+module.exports = router

packages/worker/src/api/routes/deploy.js

@@ -0,0 +1,9 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/deploy")
+const checkKey = require("../../middleware/check-key")
+
+const router = Router()
+
+router.post("/api/deploy", checkKey, controller.deploy)
+
+module.exports = router

packages/worker/src/api/routes/index.js

@@ -0,0 +1,3 @@
+const deployRoutes = require("./deploy")
+
+exports.routes = [deployRoutes]

packages/worker/src/environment.js

@@ -0,0 +1,18 @@
+module.exports = {
+  SELF_HOSTED: process.env.SELF_HOSTED,
+  WORKER_API_KEY: process.env.WORKER_API_KEY,
+  PORT: process.env.PORT,
+  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
+  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME,
+  COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
+  RAW_COUCH_DB_URL: process.env.RAW_COUCH_DB_URL,
+  RAW_MINIO_URL: process.env.RAW_MINIO_URL,
+  COUCH_DB_PORT: process.env.COUCH_DB_PORT,
+  MINIO_PORT: process.env.MINIO_PORT,
+  SELF_HOST_KEY: process.env.SELF_HOST_KEY,
+  _set(key, value) {
+    process.env[key] = value
+    module.exports[key] = value
+  },
+}

packages/worker/src/index.js

@@ -0,0 +1,48 @@
+const Koa = require("koa")
+const destroyable = require("server-destroy")
+const koaBody = require("koa-body")
+const logger = require("koa-pino-logger")
+const http = require("http")
+const api = require("./api")
+const env = require("./environment")
+
+const app = new Koa()
+
+if (!env.SELF_HOSTED) {
+  throw "Currently this service only supports use in self hosting"
+}
+
+// set up top level koa middleware
+app.use(koaBody({ multipart: true }))
+
+app.use(
+  logger({
+    prettyPrint: {
+      levelFirst: true,
+    },
+    level: env.LOG_LEVEL || "error",
+  })
+)
+
+// api routes
+app.use(api.routes())
+
+const server = http.createServer(app.callback())
+destroyable(server)
+
+server.on("close", () => console.log("Server Closed"))
+
+module.exports = server.listen(env.PORT || 4002, async () => {
+  console.log(`Worker running on ${JSON.stringify(server.address())}`)
+})
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  server.close()
+  server.destroy()
+})
+
+process.on("SIGTERM", () => {
+  server.close()
+  server.destroy()
+})

packages/worker/src/middleware/check-key.js

@@ -0,0 +1,12 @@
+const env = require("../environment")
+
+module.exports = async (ctx, next) => {
+  if (
+    !ctx.request.body.selfHostKey ||
+    env.SELF_HOST_KEY !== ctx.request.body.selfHostKey
+  ) {
+    ctx.throw(401, "Deployment unauthorised")
+  } else {
+    await next()
+  }
+}