MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #974 from Budibase/feature/self-hosting 

Self hosting with docker compose
This commit is contained in:
Michael Drury 2021-01-11 11:50:19 +00:00 committed by GitHub
parent e8f18e2853 bf50f9af89
commit 1af889eddd
79 changed files with 3335 additions and 902 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/release.yml vendored
View File

@ -61,3 +61,12 @@ jobs:
# macOS notarization API key
API_KEY_ID: ${{ secrets.api_key_id }}
API_KEY_ISSUER_ID: ${{ secrets.api_key_issuer_id }}
- name: Build/release Docker images
# only run the docker image build on linux, easiest way
if: startsWith(matrix.os, 'ubuntu')
env:
DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
run: yarn build:docker

1
.gitignore vendored
View File

@ -62,6 +62,7 @@ typings/
# dotenv environment variables file
.env
!hosting/.env
# parcel-bundler cache (https://parceljs.org/)
.cache

1
hosting/.env Symbolic link
View File

@ -0,0 +1 @@
hosting.properties

16
hosting/build/docker-compose.yaml Normal file
View File

@ -0,0 +1,16 @@
version: "3"
services:
app-service:
build: ./server
volumes:
- ./server:/app
environment:
SELF_HOSTED: 1
PORT: 4002
worker-service:
build: ./worker
environment:
SELF_HOSTED: 1,
PORT: 4003

1
hosting/build/server Symbolic link
View File

@ -0,0 +1 @@
../../packages/server/

1
hosting/build/worker Symbolic link
View File

@ -0,0 +1 @@
../../packages/worker/

90
hosting/docker-compose.yaml Normal file
View File

@ -0,0 +1,90 @@
version: "3"
services:
app-service:
image: budibase/budibase-apps
ports:
- "${APP_PORT}:4002"
environment:
SELF_HOSTED: 1
COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
LOGO_URL: ${LOGO_URL}
PORT: 4002
JWT_SECRET: ${JWT_SECRET}
depends_on:
- worker-service
worker-service:
image: budibase/budibase-worker
ports:
- "${WORKER_PORT}:4003"
environment:
SELF_HOSTED: 1,
DEPLOYMENT_API_KEY: ${WORKER_API_KEY}
PORT: 4003
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
RAW_MINIO_URL: http://minio-service:9000
COUCH_DB_USERNAME: ${COUCH_DB_USER}
COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
RAW_COUCH_DB_URL: http://couchdb-service:5984
SELF_HOST_KEY: ${HOSTING_KEY}
depends_on:
- minio-service
- couch-init
minio-service:
image: minio/minio
volumes:
- minio_data:/data
ports:
- "${MINIO_PORT}:9000"
environment:
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
command: server /data
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
proxy-service:
image: envoyproxy/envoy:v1.16-latest
volumes:
- ./envoy.yaml:/etc/envoy/envoy.yaml
ports:
- "${MAIN_PORT}:10000"
- "9901:9901"
depends_on:
- minio-service
- worker-service
- app-service
- couchdb-service
couchdb-service:
image: apache/couchdb:3.0
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}
ports:
- "${COUCH_DB_PORT}:5984"
- "4369:4369"
- "9100:9100"
volumes:
- couchdb_data:/couchdb
couch-init:
image: curlimages/curl
environment:
PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
depends_on:
- couchdb-service
command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
volumes:
couchdb_data:
driver: local
minio_data:
driver: local

104
hosting/envoy.yaml Normal file
View File

@ -0,0 +1,104 @@
static_resources:
listeners:
- name: main_listener
address:
socket_address: { address: 0.0.0.0, port_value: 10000 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: ingress
codec_type: auto
route_config:
name: local_route
virtual_hosts:
- name: local_services
domains: ["*"]
routes:
- match: { prefix: "/app/" }
route:
cluster: app-service
prefix_rewrite: "/"
# special case for when API requests are made, can just forward, not to minio
- match: { prefix: "/api/" }
route:
cluster: app-service
- match: { prefix: "/worker/" }
route:
cluster: worker-service
prefix_rewrite: "/"
- match: { prefix: "/db/" }
route:
cluster: couchdb-service
prefix_rewrite: "/"
# minio is on the default route because this works
# best, minio + AWS SDK doesn't handle path proxy
- match: { prefix: "/" }
route:
cluster: minio-service
http_filters:
- name: envoy.filters.http.router
clusters:
- name: app-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: app-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: app-service
port_value: 4002
- name: minio-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: minio-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: minio-service
port_value: 9000
- name: worker-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: worker-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: worker-service
port_value: 4003
- name: couchdb-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: couchdb-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: couchdb-service
port_value: 5984

25
hosting/hosting.properties Normal file
View File

@ -0,0 +1,25 @@
# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
MAIN_PORT=10000
# Use this password when configuring your self hosting settings
# This should be updated
HOSTING_KEY=budibase
# This section contains customisation options
LOGO_URL=https://logoipsum.com/logo/logo-15.svg
# This section contains all secrets pertaining to the system
# These should be updated
JWT_SECRET=testsecret
MINIO_ACCESS_KEY=budibase
MINIO_SECRET_KEY=budibase
COUCH_DB_PASSWORD=budibase
COUCH_DB_USER=budibase
WORKER_API_KEY=budibase
# This section contains variables that do not need to be altered under normal circumstances
APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
BUDIBASE_ENVIRONMENT=PRODUCTION

4
hosting/scripts/linux/install-docker-compose.sh Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

6
hosting/scripts/linux/install-docker.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
echo "**** WARNING - not for production environments ****"
# warning this is a convience script, for production installations install docker
# properly for your environment!
curl -fsSL https://get.docker.com -o get-docker.sh
sudo sh get-docker.sh

9
hosting/scripts/linux/release-to-docker-hub.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
pushd ../../build
docker-compose build --force app-service
docker-compose build --force worker-service
docker tag build_app-service budibase/budibase-apps:latest
docker push budibase/budibase-apps
docker tag build_worker-service budibase/budibase-worker:latest
docker push budibase/budibase-worker
popd

2
hosting/start.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/bash
docker-compose --env-file hosting.properties up

20
hosting/utils/testing.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
function dockerInstalled {
echo "Checking docker installation..."
if [ ! -x "$(command -v docker)" ]; then
echo "Please install docker to continue"
exit -1
fi
}
dockerInstalled
source "${BASH_SOURCE%/*}/hosting.properties"
opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key"
if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then
opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old"
fi
docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data

3
package.json
View File

@ -32,7 +32,8 @@
"lint:fix": "eslint --fix packages",
"format": "prettier --write \"{,!(node_modules)/**/}*.{js,jsx,svelte}\"",
"test:e2e": "lerna run cy:test",
"test:e2e:ci": "lerna run cy:ci"
"test:e2e:ci": "lerna run cy:ci",
"build:docker": "cd hosting/scripts/linux/ && ./release-to-docker-hub.sh && cd -"
},
"dependencies": {
"@fortawesome/fontawesome": "^1.1.8"

5
packages/builder/src/builderStore/index.js
View File

@ -1,6 +1,8 @@
import { getFrontendStore } from "./store/frontend"
import { getBackendUiStore } from "./store/backend"
import { getAutomationStore } from "./store/automation/"
import { getAutomationStore } from "./store/automation"
import { getHostingStore } from "./store/hosting"
import { getThemeStore } from "./store/theme"
import { derived, writable } from "svelte/store"
import analytics from "analytics"
@ -11,6 +13,7 @@ export const store = getFrontendStore()
export const backendUiStore = getBackendUiStore()
export const automationStore = getAutomationStore()
export const themeStore = getThemeStore()
export const hostingStore = getHostingStore()
export const currentAsset = derived(store, $store => {
const type = $store.currentFrontEndType

2
packages/builder/src/builderStore/store/frontend.js
View File

@ -7,6 +7,7 @@ import {
import {
allScreens,
backendUiStore,
hostingStore,
currentAsset,
mainLayout,
selectedComponent,
@ -70,6 +71,7 @@ export const getFrontendStore = () => {
appInstance: pkg.application.instance,
}))
await hostingStore.actions.fetch()
await backendUiStore.actions.database.select(pkg.application.instance)
},
routing: {

38
packages/builder/src/builderStore/store/hosting.js Normal file
View File

@ -0,0 +1,38 @@
import { writable } from "svelte/store"
import api from "../api"
const INITIAL_BACKEND_UI_STATE = {
hostingInfo: {},
appUrl: "",
}
export const getHostingStore = () => {
const store = writable({ ...INITIAL_BACKEND_UI_STATE })
store.actions = {
fetch: async () => {
const responses = await Promise.all([
api.get("/api/hosting/"),
api.get("/api/hosting/urls"),
])
const [info, urls] = await Promise.all(responses.map(resp => resp.json()))
store.update(state => {
state.hostingInfo = info
state.appUrl = urls.app
return state
})
return info
},
save: async hostingInfo => {
const response = await api.post("/api/hosting", hostingInfo)
const revision = (await response.json()).rev
store.update(state => {
state.hostingInfo = {
...hostingInfo,
_rev: revision,
}
return state
})
},
}
return store
}

5
packages/builder/src/components/automation/Shared/WebhookDisplay.svelte
View File

@ -1,16 +1,17 @@
<script>
import { notifier } from "builderStore/store/notifications"
import { Input } from "@budibase/bbui"
import { store } from "builderStore"
import { store, hostingStore } from "builderStore"
export let value
export let production = false
$: appId = $store.appId
$: appUrl = $hostingStore.appUrl
function fullWebhookURL(uri) {
if (production) {
return `https://${appId}.app.budi.live/${uri}`
return `${appUrl}/${uri}`
} else {
return `http://localhost:4001/${uri}`
}

7
packages/builder/src/components/deploy/DeploymentHistory.svelte
View File

@ -6,6 +6,7 @@
import api from "builderStore/api"
import { notifier } from "builderStore/store/notifications"
import CreateWebhookDeploymentModal from "./CreateWebhookDeploymentModal.svelte"
import { hostingStore } from "builderStore"
const DeploymentStatus = {
SUCCESS: "SUCCESS",
@ -35,7 +36,7 @@
let errorReason
let poll
let deployments = []
let deploymentUrl = `https://${appId}.app.budi.live/${appId}`
let deploymentUrl = `${$hostingStore.appUrl}/${appId}`
const formatDate = (date, format) =>
Intl.DateTimeFormat("en-GB", DATE_OPTIONS[format]).format(date)
@ -95,9 +96,7 @@
<h4>Deployment History</h4>
<div class="deploy-div">
{#if deployments.some(deployment => deployment.status === DeploymentStatus.SUCCESS)}
<a target="_blank" href={`https://${appId}.app.budi.live/${appId}`}>
View Your Deployed App →
</a>
<a target="_blank" href={deploymentUrl}> View Your Deployed App → </a>
<Button primary on:click={() => modal.show()}>View webhooks</Button>
{/if}
</div>

105
packages/builder/src/components/settings/ThemeEditor.svelte
View File

@ -1,82 +1,46 @@
<script>
import { themeStore } from "builderStore"
import { Label, DropdownMenu, Toggle, Button, Slider } from "@budibase/bbui"
import { Label, Toggle, Button, Slider } from "@budibase/bbui"
let anchor
let popover
let showAdvanced = false
</script>
<div class="topnavitemright" on:click={popover.show} bind:this={anchor}>
<i class="ri-paint-fill" />
</div>
<div class="dropdown">
<DropdownMenu bind:this={popover} {anchor} align="right">
<div class="content">
<div>
<Label extraSmall grey>Theme</Label>
<Toggle thin text="Dark theme" bind:checked={$themeStore.darkMode} />
</div>
{#if $themeStore.darkMode && !showAdvanced}
<div class="button">
<Button text on:click={() => (showAdvanced = true)}>Customise</Button>
</div>
{/if}
{#if $themeStore.darkMode && showAdvanced}
<Slider
label="Hue"
bind:value={$themeStore.hue}
min="0"
max="360"
showValue />
<Slider
label="Saturation"
bind:value={$themeStore.saturation}
min="0"
max="100"
showValue />
<Slider
label="Lightness"
bind:value={$themeStore.lightness}
min="0"
max="32"
showValue />
<div class="button">
<Button text on:click={themeStore.reset}>Reset</Button>
</div>
{/if}
<div class="content">
<div>
<Toggle thin text="Dark theme" bind:checked={$themeStore.darkMode} />
</div>
{#if $themeStore.darkMode && !showAdvanced}
<div class="button">
<Button text on:click={() => (showAdvanced = true)}>Customise</Button>
</div>
</DropdownMenu>
{/if}
{#if $themeStore.darkMode && showAdvanced}
<Slider
label="Hue"
bind:value={$themeStore.hue}
min="0"
max="360"
showValue />
<Slider
label="Saturation"
bind:value={$themeStore.saturation}
min="0"
max="100"
showValue />
<Slider
label="Lightness"
bind:value={$themeStore.lightness}
min="0"
max="32"
showValue />
<div class="button">
<Button text on:click={themeStore.reset}>Reset</Button>
</div>
{/if}
</div>
<style>
.dropdown {
z-index: 2;
}
i {
font-size: 18px;
color: var(--grey-7);
}
.topnavitemright {
cursor: pointer;
color: var(--grey-7);
margin: 0 12px 0 0;
font-weight: 500;
font-size: 1rem;
display: flex;
flex-direction: row;
justify-content: center;
align-items: center;
height: 24px;
width: 24px;
}
.topnavitemright:hover i {
color: var(--ink);
}
.content {
padding: var(--spacing-xl);
display: flex;
flex-direction: column;
justify-content: flex-start;
@ -84,11 +48,6 @@
gap: var(--spacing-l);
}
h5 {
margin: 0;
font-weight: 500;
}
.button {
align-self: flex-start;
}

55
packages/builder/src/components/settings/ThemeEditorDropdown.svelte Normal file
View File

@ -0,0 +1,55 @@
<script>
import { Label, DropdownMenu } from "@budibase/bbui"
import ThemeEditor from "./ThemeEditor.svelte"
let anchor
let popover
</script>
<div class="topnavitemright" on:click={popover.show} bind:this={anchor}>
<i class="ri-paint-fill" />
</div>
<div class="dropdown">
<DropdownMenu bind:this={popover} {anchor} align="right">
<div class="content">
<Label extraSmall grey>Theme</Label>
<ThemeEditor />
</div>
</DropdownMenu>
</div>
<style>
.dropdown {
z-index: 2;
}
i {
font-size: 18px;
color: var(--grey-7);
}
.topnavitemright {
cursor: pointer;
color: var(--grey-7);
margin: 0 12px 0 0;
font-weight: 500;
font-size: 1rem;
display: flex;
flex-direction: row;
justify-content: center;
align-items: center;
height: 24px;
width: 24px;
}
.topnavitemright:hover i {
color: var(--ink);
}
h5 {
margin: 0;
font-weight: 500;
}
.content {
padding: var(--spacing-xl);
}
</style>

32
packages/builder/src/components/start/BuilderSettingsButton.svelte Normal file
View File

@ -0,0 +1,32 @@
<script>
import { TextButton as Button, Modal } from "@budibase/bbui"
import BuilderSettingsModal from "./BuilderSettingsModal.svelte"
let modal
</script>
<div>
<Button text on:click={modal.show}>
<i class="ri-settings-3-fill" />
<p>Settings</p>
</Button>
</div>
<Modal bind:this={modal} width="30%">
<BuilderSettingsModal />
</Modal>
<style>
div i {
font-size: 26px;
color: var(--grey-7);
margin-left: 12px;
}
div p {
font-family: var(--font-sans);
font-size: var(--font-size-s);
color: var(--ink);
font-weight: 400;
margin: 0 0 0 12px;
}
</style>

72
packages/builder/src/components/start/BuilderSettingsModal.svelte Normal file
View File

@ -0,0 +1,72 @@
<script>
import { notifier } from "builderStore/store/notifications"
import { hostingStore } from "builderStore"
import { Input, ModalContent, Toggle } from "@budibase/bbui"
import ThemeEditor from "components/settings/ThemeEditor.svelte"
import analytics from "analytics"
import { onMount } from "svelte"
let hostingInfo
let selfhosted = false
async function save() {
hostingInfo.type = selfhosted ? "self" : "cloud"
if (!selfhosted && hostingInfo._rev) {
hostingInfo = {
type: hostingInfo.type,
_id: hostingInfo._id,
_rev: hostingInfo._rev,
}
}
try {
await hostingStore.actions.save(hostingInfo)
notifier.success(`Settings saved.`)
} catch (err) {
notifier.danger(`Failed to update builder settings.`)
}
}
function updateSelfHosting(event) {
if (hostingInfo.type === "cloud" && event.target.checked) {
hostingInfo.hostingUrl = "localhost:10000"
hostingInfo.useHttps = false
hostingInfo.selfHostKey = "budibase"
}
}
onMount(async () => {
hostingInfo = await hostingStore.actions.fetch()
selfhosted = hostingInfo.type === "self"
})
</script>
<ModalContent title="Builder settings" confirmText="Save" onConfirm={save}>
<h5>Theme</h5>
<ThemeEditor />
<h5>Hosting</h5>
<p>
This section contains settings that relate to the deployment and hosting of
apps made in this builder.
</p>
<Toggle
thin
text="Self hosted"
on:change={updateSelfHosting}
bind:checked={selfhosted} />
{#if selfhosted}
<Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" />
<Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" />
<Toggle thin text="HTTPS" bind:checked={hostingInfo.useHttps} />
{/if}
</ModalContent>
<style>
h5 {
margin: 0;
font-size: 14px;
}
p {
margin: 0;
font-size: 12px;
}
</style>

76
packages/builder/src/components/start/CreateAppModal.svelte
View File

@ -1,6 +1,11 @@
<script>
import { writable } from "svelte/store"
import { store, automationStore, backendUiStore } from "builderStore"
import {
store,
automationStore,
backendUiStore,
hostingStore,
} from "builderStore"
import { string, object } from "yup"
import api, { get } from "builderStore/api"
import Form from "@svelteschool/svelte-forms"
@ -12,6 +17,7 @@
import { fade } from "svelte/transition"
import { post } from "builderStore/api"
import analytics from "analytics"
import { onMount } from "svelte"
//Move this to context="module" once svelte-forms is updated so that it can bind to stores correctly
const createAppStore = writable({ currentStep: 0, values: {} })
@ -23,6 +29,7 @@
let lastApiKey
let fetchApiKeyPromise
const validateApiKey = async apiKey => {
if (isApiKeyValid) return true
if (!apiKey) return false
// make sure we only fetch once, unless API Key is changed
@ -39,43 +46,46 @@
return isApiKeyValid
}
const apiValidation = {
apiKey: string()
.required("Please enter your API key.")
.test("valid-apikey", "This API key is invalid", validateApiKey),
}
const infoValidation = {
applicationName: string().required("Your application must have a name."),
}
const userValidation = {
email: string()
.email()
.required("Your application needs a first user."),
password: string().required("Please enter a password for your first user."),
roleId: string().required("You need to select a role for your user."),
}
let submitting = false
let errors = {}
let validationErrors = {}
let validationSchemas = [
{
apiKey: string()
.required("Please enter your API key.")
.test("valid-apikey", "This API key is invalid", validateApiKey),
},
{
applicationName: string().required("Your application must have a name."),
},
{
email: string()
.email()
.required("Your application needs a first user."),
password: string().required(
"Please enter a password for your first user."
),
roleId: string().required("You need to select a role for your user."),
},
]
let validationSchemas = [apiValidation, infoValidation, userValidation]
let steps = [
{
component: API,
function buildStep(component) {
return {
component,
errors,
},
{
component: Info,
errors,
},
{
component: User,
errors,
},
]
}
}
// steps need to be initialized for cypress from the get go
let steps = [buildStep(API), buildStep(Info), buildStep(User)]
onMount(async () => {
let hostingInfo = await hostingStore.actions.fetch()
// re-init the steps based on whether self hosting or cloud hosted
if (hostingInfo.type === "self") {
isApiKeyValid = true
steps = [buildStep(Info), buildStep(User)]
validationSchemas = [infoValidation, userValidation]
}
})
if (hasKey) {
validationSchemas.shift()

1
packages/builder/src/components/userInterface/AppPreview/CurrentItemPreview.svelte
View File

@ -30,6 +30,7 @@
}
$: selectedComponentId = $store.selectedComponentId ?? ""
$: previewData = {
appId: $store.appId,
layout,
screen,
selectedComponentId,

3
packages/builder/src/components/userInterface/AppPreview/iframeTemplate.html
View File

@ -22,10 +22,11 @@
}
// Extract data from message
const { selectedComponentId, layout, screen, previewType } = JSON.parse(event.data)
const { selectedComponentId, layout, screen, previewType, appId } = JSON.parse(event.data)
// Set some flags so the app knows we're in the builder
window["##BUDIBASE_IN_BUILDER##"] = true
window["##BUDIBASE_APP_ID##"] = appId
window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId

4
packages/builder/src/pages/[application]/_reset.svelte
View File

@ -2,7 +2,7 @@
import { store, automationStore, backendUiStore } from "builderStore"
import { Button } from "@budibase/bbui"
import SettingsLink from "components/settings/Link.svelte"
import ThemeEditor from "components/settings/ThemeEditor.svelte"
import ThemeEditorDropdown from "components/settings/ThemeEditorDropdown.svelte"
import FeedbackNavLink from "components/userInterface/Feedback/FeedbackNavLink.svelte"
import { get } from "builderStore/api"
import { isActive, goto, layout } from "@sveltech/routify"
@ -67,7 +67,7 @@
{/each}
</div>
<div class="toprightnav">
<ThemeEditor />
<ThemeEditorDropdown />
<FeedbackNavLink />
<div class="topnavitemright">
<a

48
packages/builder/src/pages/_layout.svelte
View File

@ -7,6 +7,9 @@
CommunityIcon,
BugIcon,
} from "components/common/Icons"
import BuilderSettingsButton from "components/start/BuilderSettingsButton.svelte"
let modal
</script>
<div class="root">
@ -16,27 +19,30 @@
</div>
<div class="nav-section">
<Link icon={AppsIcon} title="Apps" href="/" active />
<Link
icon={HostingIcon}
title="Hosting"
href="https://portal.budi.live/" />
<Link
icon={DocumentationIcon}
title="Documentation"
href="https://docs.budibase.com/" />
<Link
icon={CommunityIcon}
title="Community"
href="https://github.com/Budibase/budibase/discussions" />
<Link
icon={BugIcon}
title="Raise an issue"
href="https://github.com/Budibase/budibase/issues/new/choose" />
<div class="nav-top">
<Link icon={AppsIcon} title="Apps" href="/" active />
<Link
icon={HostingIcon}
title="Hosting"
href="https://portal.budi.live/" />
<Link
icon={DocumentationIcon}
title="Documentation"
href="https://docs.budibase.com/" />
<Link
icon={CommunityIcon}
title="Community"
href="https://github.com/Budibase/budibase/discussions" />
<Link
icon={BugIcon}
title="Raise an issue"
href="https://github.com/Budibase/budibase/issues/new/choose" />
</div>
<div class="nav-bottom">
<BuilderSettingsButton />
</div>
</div>
</div>
<div class="main">
<slot />
</div>
@ -76,8 +82,10 @@
}
.nav-section {
margin: 20px 0px;
margin: 20px 0 0 0;
display: flex;
flex-direction: column;
justify-content: space-between;
height: 100%;
}
</style>

2
packages/client/rollup.config.js
View File

@ -10,7 +10,7 @@ export default {
output: [
{
sourcemap: true,
format: "esm",
format: "iife",
file: `./dist/budibase-client.js`,
},
],

15
packages/client/src/api/api.js
View File

@ -1,17 +1,8 @@
import { getAppId } from "../utils/getAppId"
/**
* API cache for cached request responses.
*/
let cache = {}
/**
* Makes a fully formatted URL based on the SDK configuration.
*/
const makeFullURL = path => {
return `/${path}`.replace("//", "/")
}
/**
* Handler for API errors.
*/
@ -29,7 +20,7 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
let headers = {
Accept: "application/json",
"Content-Type": "application/json",
"x-budibase-app-id": getAppId(),
"x-budibase-app-id": window["##BUDIBASE_APP_ID##"],
}
if (!window["##BUDIBASE_IN_BUILDER##"]) {
headers["x-budibase-type"] = "client"
@ -82,8 +73,8 @@ const makeCachedApiCall = async params => {
*/
const requestApiCall = method => async params => {
const { url, cache = false } = params
const fullURL = makeFullURL(url)
const enrichedParams = { ...params, method, url: fullURL }
const fixedUrl = `/${url}`.replace("//", "/")
const enrichedParams = { ...params, method, fixedUrl }
return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams)
}

1
packages/client/src/index.js
View File

@ -7,6 +7,7 @@ const loadBudibase = () => {
// Update builder store with any builder flags
builderStore.set({
inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
appId: window["##BUDIBASE_APP_ID##"],
layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],

2
packages/client/src/sdk.js
View File

@ -2,7 +2,6 @@ import * as API from "./api"
import { authStore, routeStore, screenStore, bindingStore } from "./store"
import { styleable } from "./utils/styleable"
import { linkable } from "./utils/linkable"
import { getAppId } from "./utils/getAppId"
import DataProvider from "./components/DataProvider.svelte"
export default {
@ -12,7 +11,6 @@ export default {
screenStore,
styleable,
linkable,
getAppId,
DataProvider,
setBindableValue: bindingStore.actions.setBindableValue,
}

6
packages/client/src/store/auth.js
View File

@ -1,8 +1,8 @@
import * as API from "../api"
import { getAppId } from "../utils/getAppId"
import { writable } from "svelte/store"
import { writable, get } from "svelte/store"
import { initialise } from "./initialise"
import { routeStore } from "./routes"
import { builderStore } from "./builder"
const createAuthStore = () => {
const store = writable("")
@ -25,7 +25,7 @@ const createAuthStore = () => {
}
const logOut = async () => {
store.set("")
const appId = getAppId()
const appId = get(builderStore).appId
if (appId) {
for (let environment of ["local", "cloud"]) {
window.document.cookie = `budibase:${appId}:${environment}=; Path=/; Expires=Thu, 01 Jan 1970 00:00:01 GMT;`

1
packages/client/src/store/builder.js
View File

@ -3,6 +3,7 @@ import { writable } from "svelte/store"
const createBuilderStore = () => {
const initialState = {
inBuilder: false,
appId: null,
layout: null,
screen: null,
selectedComponentId: null,

5
packages/client/src/store/screens.js
View File

@ -1,8 +1,7 @@
import { writable, derived } from "svelte/store"
import { writable, derived, get } from "svelte/store"
import { routeStore } from "./routes"
import { builderStore } from "./builder"
import * as API from "../api"
import { getAppId } from "../utils/getAppId"
const createScreenStore = () => {
const config = writable({
@ -40,7 +39,7 @@ const createScreenStore = () => {
)
const fetchScreens = async () => {
const appDefinition = await API.fetchAppDefinition(getAppId())
const appDefinition = await API.fetchAppDefinition(get(builderStore).appId)
config.set({
screens: appDefinition.screens,
layouts: appDefinition.layouts,

47
packages/client/src/utils/getAppId.js
View File

@ -1,47 +0,0 @@
const COOKIE_SEPARATOR = ";"
const APP_PREFIX = "app_"
const KEY_VALUE_SPLIT = "="
function confirmAppId(possibleAppId) {
return possibleAppId && possibleAppId.startsWith(APP_PREFIX)
? possibleAppId
: undefined
}
function tryGetFromCookie({ cookies }) {
if (!cookies) {
return undefined
}
const cookie = cookies
.split(COOKIE_SEPARATOR)
.find(cookie => cookie.trim().startsWith("budibase:currentapp"))
let appId
if (cookie && cookie.split(KEY_VALUE_SPLIT).length === 2) {
appId = cookie.split("=")[1]
}
return confirmAppId(appId)
}
function tryGetFromPath() {
const appId = location.pathname.split("/")[1]
return confirmAppId(appId)
}
function tryGetFromSubdomain() {
const parts = window.location.host.split(".")
const appId = parts[1] ? parts[0] : undefined
return confirmAppId(appId)
}
export const getAppId = (cookies = window.document.cookie) => {
const functions = [tryGetFromSubdomain, tryGetFromPath, tryGetFromCookie]
// try getting the app Id in order
let appId
for (let func of functions) {
appId = func({ cookies })
if (appId) {
break
}
}
return appId
}

8
packages/server/Dockerfile
View File

@ -4,12 +4,16 @@ WORKDIR /app
ENV CLOUD=1
ENV COUCH_DB_URL=https://couchdb.budi.live:5984
env BUDIBASE_ENVIRONMENT=PRODUCTION
ENV BUDIBASE_ENVIRONMENT=PRODUCTION
# copy files and install dependencies
COPY . ./
RUN yarn
RUN yarn
EXPOSE 4001
# have to add node environment production after install
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
CMD ["yarn", "run:docker"]

56
packages/server/scripts/createApiKeyAndAppId.js
View File

@ -1,56 +0,0 @@
// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
const dynamoClient = require("../src/db/dynamoClient")
const env = require("../src/environment")
if (process.argv[2] == null || process.argv[3] == null) {
console.error(
"Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
)
process.exit(-1)
}
const FAKE_STRING = "fakestring"
// set fake credentials for local dynamo to actually work
env._set("AWS_ACCESS_KEY_ID", "KEY_ID")
env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY")
dynamoClient.init("http://localhost:8333")
async function run() {
await dynamoClient.apiKeyTable.put({
item: {
pk: process.argv[2],
accountId: FAKE_STRING,
trackingId: FAKE_STRING,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
rows: 0,
storage: 0,
users: 0,
views: 0,
},
usageLimits: {
automationRuns: 10,
rows: 10,
storage: 1000,
users: 10,
views: 10,
},
},
})
await dynamoClient.apiKeyTable.put({
item: {
pk: process.argv[3],
apiKey: process.argv[2],
},
})
}
run()
.then(() => {
console.log("Rows should have been created.")
})
.catch(err => {
console.error("Cannot create rows - " + err)
})

13
packages/server/src/api/controllers/apikeys.js
View File

@ -3,13 +3,20 @@ const { join } = require("../../utilities/centralPath")
const readline = require("readline")
const { budibaseAppsDir } = require("../../utilities/budibaseDir")
const env = require("../../environment")
const selfhost = require("../../selfhost")
const ENV_FILE_PATH = "/.env"
exports.fetch = async function(ctx) {
ctx.status = 200
ctx.body = {
budibase: env.BUDIBASE_API_KEY,
userId: env.USERID_API_KEY,
if (env.SELF_HOSTED) {
ctx.body = {
selfhost: await selfhost.getSelfHostAPIKey(),
}
} else {
ctx.body = {
budibase: env.BUDIBASE_API_KEY,
userId: env.USERID_API_KEY,
}
}
}

3
packages/server/src/api/controllers/application.js
View File

@ -150,6 +150,9 @@ exports.create = async function(ctx) {
name: ctx.request.body.name,
template: ctx.request.body.template,
instance: instance,
deployment: {
type: "cloud",
},
}
const instanceDb = new CouchDB(appId)
await instanceDb.put(newApplication)

4
packages/server/src/api/controllers/auth.js
View File

@ -35,8 +35,8 @@ exports.authenticate = async ctx => {
roleId: dbUser.roleId,
version: app.version,
}
// if in cloud add the user api key
if (env.CLOUD) {
// if in cloud add the user api key, unless self hosted
if (env.CLOUD && !env.SELF_HOSTED) {
const { apiKey } = await getAPIKey(ctx.user.appId)
payload.apiKey = apiKey
}

88
packages/server/src/api/controllers/deploy/Deployment.js Normal file
View File

@ -0,0 +1,88 @@
const { getAppQuota } = require("./quota")
const env = require("../../../environment")
const newid = require("../../../db/newid")
/**
* This is used to pass around information about the deployment that is occurring
*/
class Deployment {
constructor(appId, id = null) {
this.appId = appId
this._id = id || newid()
}
// purely so that we can do quota stuff outside the main deployment context
async init() {
if (!env.SELF_HOSTED) {
this.setQuota(await getAppQuota(this.appId))
}
}
setQuota(quota) {
if (!quota) {
return
}
this.quota = quota
}
getQuota() {
return this.quota
}
getAppId() {
return this.appId
}
setVerification(verification) {
if (!verification) {
return
}
this.verification = verification
if (this.verification.quota) {
this.quota = this.verification.quota
}
}
getVerification() {
return this.verification
}
setStatus(status, err = null) {
this.status = status
if (err) {
this.err = err
}
}
fromJSON(json) {
if (json.verification) {
this.setVerification(json.verification)
}
if (json.quota) {
this.setQuota(json.quota)
}
if (json.status) {
this.setStatus(json.status, json.err)
}
}
getJSON() {
const obj = {
_id: this._id,
appId: this.appId,
status: this.status,
}
if (this.err) {
obj.err = this.err
}
if (this.verification && this.verification.cfDistribution) {
obj.cfDistribution = this.verification.cfDistribution
}
if (this.quota) {
obj.quota = this.quota
}
return obj
}
}
module.exports = Deployment

189
packages/server/src/api/controllers/deploy/aws.js
View File

@ -1,189 +0,0 @@
const fs = require("fs")
const { join } = require("../../../utilities/centralPath")
const AWS = require("aws-sdk")
const fetch = require("node-fetch")
const sanitize = require("sanitize-s3-objectkey")
const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
const PouchDB = require("../../../db")
const env = require("../../../environment")
/**
* Finalises the deployment, updating the quota for the user API key
* The verification process returns the levels to update to.
* Calls the "deployment-success" lambda.
* @param {object} quota The usage quota levels returned from the verifyDeploy
* @returns {Promise<object>} The usage has been updated against the user API key.
*/
exports.updateDeploymentQuota = async function(quota) {
const DEPLOYMENT_SUCCESS_URL =
env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
method: "POST",
body: JSON.stringify({
apiKey: env.BUDIBASE_API_KEY,
quota,
}),
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
})
if (response.status !== 200) {
throw new Error(`Error updating deployment quota for API Key`)
}
return await response.json()
}
/**
* Verifies the users API key and
* Verifies that the deployment fits within the quota of the user
* Links to the "check-api-key" lambda.
* @param {String} appId - appId being deployed
* @param {String} appId - appId being deployed
* @param {quota} quota - current quota being changed with this application
*/
exports.verifyDeployment = async function({ appId, quota }) {
const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, {
method: "POST",
body: JSON.stringify({
apiKey: env.BUDIBASE_API_KEY,
appId,
quota,
}),
})
const json = await response.json()
if (json.errors) {
throw new Error(json.errors)
}
if (response.status !== 200) {
throw new Error(
`Error fetching temporary credentials for api key: ${env.BUDIBASE_API_KEY}`
)
}
// set credentials here, means any time we're verified we're ready to go
if (json.credentials) {
AWS.config.update({
accessKeyId: json.credentials.AccessKeyId,
secretAccessKey: json.credentials.SecretAccessKey,
sessionToken: json.credentials.SessionToken,
})
}
return json
}
const CONTENT_TYPE_MAP = {
html: "text/html",
css: "text/css",
js: "application/javascript",
}
/**
* Recursively walk a directory tree and execute a callback on all files.
* @param {String} dirPath - Directory to traverse
* @param {Function} callback - callback to execute on files
*/
function walkDir(dirPath, callback) {
for (let filename of fs.readdirSync(dirPath)) {
const filePath = `${dirPath}/${filename}`
const stat = fs.lstatSync(filePath)
if (stat.isFile()) {
callback(filePath)
} else {
walkDir(filePath, callback)
}
}
}
async function prepareUploadForS3({ s3Key, metadata, s3, file }) {
const extension = [...file.name.split(".")].pop()
const fileBytes = fs.readFileSync(file.path)
const upload = await s3
.upload({
// windows filepaths need to be converted to forward slashes for s3
Key: sanitize(s3Key).replace(/\\/g, "/"),
Body: fileBytes,
ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
Metadata: metadata,
})
.promise()
return {
size: file.size,
name: file.name,
extension,
url: upload.Location,
key: upload.Key,
}
}
exports.prepareUploadForS3 = prepareUploadForS3
exports.uploadAppAssets = async function({ appId, bucket, accountId }) {
const s3 = new AWS.S3({
params: {
Bucket: bucket,
},
})
const appAssetsPath = join(budibaseAppsDir(), appId, "public")
let uploads = []
// Upload HTML and JS of the web app
walkDir(appAssetsPath, function(filePath) {
const filePathParts = filePath.split("/")
const appAssetUpload = prepareUploadForS3({
file: {
path: filePath,
name: filePathParts.pop(),
},
s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
s3,
metadata: { accountId },
})
uploads.push(appAssetUpload)
})
// Upload file attachments
const db = new PouchDB(appId)
let fileUploads
try {
fileUploads = await db.get("_local/fileuploads")
} catch (err) {
fileUploads = { _id: "_local/fileuploads", uploads: [] }
}
for (let file of fileUploads.uploads) {
if (file.uploaded) continue
const attachmentUpload = prepareUploadForS3({
file,
s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
s3,
metadata: { accountId },
})
uploads.push(attachmentUpload)
// mark file as uploaded
file.uploaded = true
}
db.put(fileUploads)
try {
return await Promise.all(uploads)
} catch (err) {
console.error("Error uploading budibase app assets to s3", err)
throw err
}
}

85
packages/server/src/api/controllers/deploy/awsDeploy.js Normal file
View File

@ -0,0 +1,85 @@
const AWS = require("aws-sdk")
const fetch = require("node-fetch")
const env = require("../../../environment")
const {
deployToObjectStore,
performReplication,
fetchCredentials,
} = require("./utils")
/**
* Verifies the users API key and
* Verifies that the deployment fits within the quota of the user
* Links to the "check-api-key" lambda.
* @param {object} deployment - information about the active deployment, including the appId and quota.
*/
exports.preDeployment = async function(deployment) {
const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, {
apiKey: env.BUDIBASE_API_KEY,
appId: deployment.getAppId(),
quota: deployment.getQuota(),
})
// set credentials here, means any time we're verified we're ready to go
if (json.credentials) {
AWS.config.update({
accessKeyId: json.credentials.AccessKeyId,
secretAccessKey: json.credentials.SecretAccessKey,
sessionToken: json.credentials.SessionToken,
})
}
return json
}
/**
* Finalises the deployment, updating the quota for the user API key
* The verification process returns the levels to update to.
* Calls the "deployment-success" lambda.
* @param {object} deployment information about the active deployment, including the quota info.
* @returns {Promise<object>} The usage has been updated against the user API key.
*/
exports.postDeployment = async function(deployment) {
const DEPLOYMENT_SUCCESS_URL =
env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
method: "POST",
body: JSON.stringify({
apiKey: env.BUDIBASE_API_KEY,
quota: deployment.getQuota(),
}),
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
})
if (response.status !== 200) {
throw new Error(`Error updating deployment quota for API Key`)
}
return await response.json()
}
exports.deploy = async function(deployment) {
const appId = deployment.getAppId()
const { bucket, accountId } = deployment.getVerification()
const metadata = { accountId }
const s3Client = new AWS.S3({
params: {
Bucket: bucket,
},
})
await deployToObjectStore(appId, s3Client, metadata)
}
exports.replicateDb = async function(deployment) {
const appId = deployment.getAppId()
const verification = deployment.getVerification()
return performReplication(
appId,
verification.couchDbSession,
env.DEPLOYMENT_DB_URL
)
}

149
packages/server/src/api/controllers/deploy/index.js
View File

@ -1,23 +1,20 @@
const CouchDB = require("pouchdb")
const PouchDB = require("../../../db")
const Deployment = require("./Deployment")
const {
uploadAppAssets,
verifyDeployment,
updateDeploymentQuota,
} = require("./aws")
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
const newid = require("../../../db/newid")
const env = require("../../../environment")
getHostingInfo,
HostingTypes,
} = require("../../../utilities/builder/hosting")
// the max time we can wait for an invalidation to complete before considering it failed
const MAX_PENDING_TIME_MS = 30 * 60000
const DeploymentStatus = {
SUCCESS: "SUCCESS",
PENDING: "PENDING",
FAILURE: "FAILURE",
}
// default to AWS deployment, this will be updated before use (if required)
let deploymentService = require("./awsDeploy")
// checks that deployments are in a good state, any pending will be updated
async function checkAllDeployments(deployments) {
let updated = false
@ -27,7 +24,7 @@ async function checkAllDeployments(deployments) {
deployment.status === DeploymentStatus.PENDING &&
Date.now() - deployment.updatedAt > MAX_PENDING_TIME_MS
) {
deployment.status = status
deployment.status = DeploymentStatus.FAILURE
deployment.err = "Timed out"
updated = true
}
@ -35,54 +32,10 @@ async function checkAllDeployments(deployments) {
return { updated, deployments }
}
function replicate(local, remote) {
return new Promise((resolve, reject) => {
const replication = local.sync(remote)
replication.on("complete", () => resolve())
replication.on("error", err => reject(err))
})
}
async function replicateCouch({ appId, session }) {
const localDb = new PouchDB(appId)
const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, {
fetch: function(url, opts) {
opts.headers.set("Cookie", `${session};`)
return CouchDB.fetch(url, opts)
},
})
return replicate(localDb, remoteDb)
}
async function getCurrentInstanceQuota(appId) {
const db = new PouchDB(appId)
const rows = await db.allDocs({
startkey: DocumentTypes.ROW + SEPARATOR,
endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
})
const users = await db.allDocs({
startkey: DocumentTypes.USER + SEPARATOR,
endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
})
const existingRows = rows.rows.length
const existingUsers = users.rows.length
const designDoc = await db.get("_design/database")
return {
rows: existingRows,
users: existingUsers,
views: Object.keys(designDoc.views).length,
}
}
async function storeLocalDeploymentHistory(deployment) {
const db = new PouchDB(deployment.appId)
const appId = deployment.getAppId()
const deploymentJSON = deployment.getJSON()
const db = new PouchDB(appId)
let deploymentDoc
try {
@ -91,7 +44,7 @@ async function storeLocalDeploymentHistory(deployment) {
deploymentDoc = { _id: "_local/deployments", history: {} }
}
const deploymentId = deployment._id || newid()
const deploymentId = deploymentJSON._id
// first time deployment
if (!deploymentDoc.history[deploymentId])
@ -99,56 +52,42 @@ async function storeLocalDeploymentHistory(deployment) {
deploymentDoc.history[deploymentId] = {
...deploymentDoc.history[deploymentId],
...deployment,
...deploymentJSON,
updatedAt: Date.now(),
}
await db.put(deploymentDoc)
return {
_id: deploymentId,
...deploymentDoc.history[deploymentId],
}
deployment.fromJSON(deploymentDoc.history[deploymentId])
return deployment
}
async function deployApp({ appId, deploymentId }) {
async function deployApp(deployment) {
const appId = deployment.getAppId()
try {
const instanceQuota = await getCurrentInstanceQuota(appId)
const verification = await verifyDeployment({
appId,
quota: instanceQuota,
})
await deployment.init()
deployment.setVerification(
await deploymentService.preDeployment(deployment)
)
console.log(`Uploading assets for appID ${appId} assets to s3..`)
console.log(`Uploading assets for appID ${appId}..`)
await uploadAppAssets({
appId,
...verification,
})
await deploymentService.deploy(deployment)
// replicate the DB to the couchDB cluster in prod
console.log("Replicating local PouchDB to remote..")
await replicateCouch({
appId,
session: verification.couchDbSession,
})
// replicate the DB to the main couchDB cluster
console.log("Replicating local PouchDB to CouchDB..")
await deploymentService.replicateDb(deployment)
await updateDeploymentQuota(verification.quota)
await deploymentService.postDeployment(deployment)
await storeLocalDeploymentHistory({
_id: deploymentId,
appId,
cfDistribution: verification.cfDistribution,
quota: verification.quota,
status: DeploymentStatus.SUCCESS,
})
deployment.setStatus(DeploymentStatus.SUCCESS)
await storeLocalDeploymentHistory(deployment)
} catch (err) {
await storeLocalDeploymentHistory({
_id: deploymentId,
appId,
status: DeploymentStatus.FAILURE,
err: err.message,
})
throw new Error(`Deployment Failed: ${err.message}`)
deployment.setStatus(DeploymentStatus.FAILURE, err.message)
await storeLocalDeploymentHistory(deployment)
throw {
...err,
message: `Deployment Failed: ${err.message}`,
}
}
}
@ -183,15 +122,17 @@ exports.deploymentProgress = async function(ctx) {
}
exports.deployApp = async function(ctx) {
const deployment = await storeLocalDeploymentHistory({
appId: ctx.user.appId,
status: DeploymentStatus.PENDING,
})
// start by checking whether to deploy local or to cloud
const hostingInfo = await getHostingInfo()
deploymentService =
hostingInfo.type === HostingTypes.CLOUD
? require("./awsDeploy")
: require("./selfDeploy")
let deployment = new Deployment(ctx.user.appId)
deployment.setStatus(DeploymentStatus.PENDING)
deployment = await storeLocalDeploymentHistory(deployment)
deployApp({
...ctx.user,
deploymentId: deployment._id,
})
await deployApp(deployment)
ctx.body = deployment
}

27
packages/server/src/api/controllers/deploy/quota.js Normal file
View File

@ -0,0 +1,27 @@
const PouchDB = require("../../../db")
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils")
exports.getAppQuota = async function(appId) {
const db = new PouchDB(appId)
const rows = await db.allDocs({
startkey: DocumentTypes.ROW + SEPARATOR,
endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
})
const users = await db.allDocs({
startkey: DocumentTypes.USER + SEPARATOR,
endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
})
const existingRows = rows.rows.length
const existingUsers = users.rows.length
const designDoc = await db.get("_design/database")
return {
rows: existingRows,
users: existingUsers,
views: Object.keys(designDoc.views).length,
}
}

69
packages/server/src/api/controllers/deploy/selfDeploy.js Normal file
View File

@ -0,0 +1,69 @@
const AWS = require("aws-sdk")
const {
deployToObjectStore,
performReplication,
fetchCredentials,
} = require("./utils")
const {
getWorkerUrl,
getCouchUrl,
getMinioUrl,
getSelfHostKey,
} = require("../../../utilities/builder/hosting")
exports.preDeployment = async function() {
const url = `${await getWorkerUrl()}/api/deploy`
try {
const json = await fetchCredentials(url, {
selfHostKey: await getSelfHostKey(),
})
// response contains:
// couchDbSession, bucket, objectStoreSession
// set credentials here, means any time we're verified we're ready to go
if (json.objectStoreSession) {
AWS.config.update({
accessKeyId: json.objectStoreSession.accessKeyId,
secretAccessKey: json.objectStoreSession.secretAccessKey,
})
}
return json
} catch (err) {
throw {
message: "Unauthorised to deploy, check self hosting key",
status: 401,
}
}
}
exports.postDeployment = async function() {
// we don't actively need to do anything after deployment in self hosting
}
exports.deploy = async function(deployment) {
const appId = deployment.getAppId()
const verification = deployment.getVerification()
const objClient = new AWS.S3({
endpoint: await getMinioUrl(),
s3ForcePathStyle: true, // needed with minio?
signatureVersion: "v4",
params: {
Bucket: verification.bucket,
},
})
// no metadata, aws has account ID in metadata
const metadata = {}
await deployToObjectStore(appId, objClient, metadata)
}
exports.replicateDb = async function(deployment) {
const appId = deployment.getAppId()
const verification = deployment.getVerification()
return performReplication(
appId,
verification.couchDbSession,
await getCouchUrl()
)
}

131
packages/server/src/api/controllers/deploy/utils.js Normal file
View File

@ -0,0 +1,131 @@
const fs = require("fs")
const sanitize = require("sanitize-s3-objectkey")
const { walkDir } = require("../../../utilities")
const { join } = require("../../../utilities/centralPath")
const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
const fetch = require("node-fetch")
const PouchDB = require("../../../db")
const CouchDB = require("pouchdb")
const CONTENT_TYPE_MAP = {
html: "text/html",
css: "text/css",
js: "application/javascript",
}
exports.fetchCredentials = async function(url, body) {
const response = await fetch(url, {
method: "POST",
body: JSON.stringify(body),
headers: { "Content-Type": "application/json" },
})
const json = await response.json()
if (json.errors) {
throw new Error(json.errors)
}
if (response.status !== 200) {
throw new Error(
`Error fetching temporary credentials: ${JSON.stringify(json)}`
)
}
return json
}
exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
const extension = [...file.name.split(".")].pop()
const fileBytes = fs.readFileSync(file.path)
const upload = await client
.upload({
// windows file paths need to be converted to forward slashes for s3
Key: sanitize(s3Key).replace(/\\/g, "/"),
Body: fileBytes,
ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
Metadata: metadata,
})
.promise()
return {
size: file.size,
name: file.name,
extension,
url: upload.Location,
key: upload.Key,
}
}
exports.deployToObjectStore = async function(appId, objectClient, metadata) {
const appAssetsPath = join(budibaseAppsDir(), appId, "public")
let uploads = []
// Upload HTML, CSS and JS for each page of the web app
walkDir(appAssetsPath, function(filePath) {
const filePathParts = filePath.split("/")
const appAssetUpload = exports.prepareUpload({
file: {
path: filePath,
name: filePathParts.pop(),
},
s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
client: objectClient,
metadata,
})
uploads.push(appAssetUpload)
})
// Upload file attachments
const db = new PouchDB(appId)
let fileUploads
try {
fileUploads = await db.get("_local/fileuploads")
} catch (err) {
fileUploads = { _id: "_local/fileuploads", uploads: [] }
}
for (let file of fileUploads.uploads) {
if (file.uploaded) continue
const attachmentUpload = exports.prepareUpload({
file,
s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
client: objectClient,
metadata,
})
uploads.push(attachmentUpload)
// mark file as uploaded
file.uploaded = true
}
db.put(fileUploads)
try {
return await Promise.all(uploads)
} catch (err) {
console.error("Error uploading budibase app assets to s3", err)
throw err
}
}
exports.performReplication = (appId, session, dbUrl) => {
return new Promise((resolve, reject) => {
const local = new PouchDB(appId)
const remote = new CouchDB(`${dbUrl}/${appId}`, {
fetch: function(url, opts) {
opts.headers.set("Cookie", `${session};`)
return CouchDB.fetch(url, opts)
},
})
const replication = local.sync(remote)
replication.on("complete", () => resolve())
replication.on("error", err => reject(err))
})
}

39
packages/server/src/api/controllers/hosting.js Normal file
View File

@ -0,0 +1,39 @@
const CouchDB = require("../../db")
const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
const {
getHostingInfo,
HostingTypes,
getAppUrl,
} = require("../../utilities/builder/hosting")
exports.fetchInfo = async ctx => {
ctx.body = {
types: Object.values(HostingTypes),
}
}
exports.save = async ctx => {
const db = new CouchDB(BUILDER_CONFIG_DB)
const { type } = ctx.request.body
if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
ctx.body = await db.remove({
...ctx.request.body,
_id: HOSTING_DOC,
})
} else {
ctx.body = await db.put({
...ctx.request.body,
_id: HOSTING_DOC,
})
}
}
exports.fetch = async ctx => {
ctx.body = await getHostingInfo()
}
exports.fetchUrls = async ctx => {
ctx.body = {
app: await getAppUrl(ctx.appId),
}
}

28
packages/server/src/api/controllers/static/index.js
View File

@ -6,7 +6,7 @@ const fetch = require("node-fetch")
const fs = require("fs-extra")
const uuid = require("uuid")
const AWS = require("aws-sdk")
const { prepareUploadForS3 } = require("../deploy/aws")
const { prepareUpload } = require("../deploy/utils")
const handlebars = require("handlebars")
const {
budibaseAppsDir,
@ -17,6 +17,15 @@ const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
const fileProcessor = require("../../../utilities/fileProcessor")
const env = require("../../../environment")
function objectStoreUrl() {
if (env.SELF_HOSTED) {
// can use a relative url for this as all goes through the proxy (this is hosted in minio)
return `/app-assets/assets`
} else {
return "https://cdn.app.budi.live/assets"
}
}
// this was the version before we started versioning the component library
const COMP_LIB_BASE_APP_VERSION = "0.2.5"
@ -53,7 +62,7 @@ exports.uploadFile = async function(ctx) {
const fileExtension = [...file.name.split(".")].pop()
const processedFileName = `${uuid.v4()}.${fileExtension}`
return prepareUploadForS3({
return prepareUpload({
file,
s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
s3,
@ -148,6 +157,7 @@ exports.serveApp = async function(ctx) {
title: appInfo.name,
production: env.CLOUD,
appId: ctx.params.appId,
objectStoreUrl: objectStoreUrl(),
})
const template = handlebars.compile(
@ -158,6 +168,7 @@ exports.serveApp = async function(ctx) {
head,
body: html,
style: css.code,
appId: ctx.params.appId,
})
}
@ -166,8 +177,9 @@ exports.serveAttachment = async function(ctx) {
const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
// Serve from CloudFront
// TODO: need to replace this with link to self hosted object store
if (env.CLOUD) {
const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}`
const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file)
const response = await fetch(S3_URL)
const body = await response.text()
ctx.set("Content-Type", response.headers.get("Content-Type"))
@ -213,7 +225,13 @@ exports.serveComponentLibrary = async function(ctx) {
componentLib += `-${COMP_LIB_BASE_APP_VERSION}`
}
const S3_URL = encodeURI(
`https://${appId}.app.budi.live/assets/${componentLib}/${ctx.query.library}/dist/index.js`
join(
objectStoreUrl(appId),
componentLib,
ctx.query.library,
"dist",
"index.js"
)
)
const response = await fetch(S3_URL)
const body = await response.text()
@ -222,5 +240,5 @@ exports.serveComponentLibrary = async function(ctx) {
return
}
await send(ctx, "/index.js", { root: componentLibraryPath })
await send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
}

5
packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
View File

@ -4,12 +4,11 @@
export let appId
export let production
export const PRODUCTION_ASSETS_URL = `https://${appId}.app.budi.live`
export let objectStoreUrl
function publicPath(path) {
if (production) {
return `${PRODUCTION_ASSETS_URL}/assets/${appId}/${path}`
return `${objectStoreUrl}/${appId}/${path}`
}
return `/assets/${path}`

5
packages/server/src/api/controllers/static/templates/app.hbs
View File

@ -5,6 +5,9 @@
{{{head}}}
</head>
{{{body}}}
<script>
window["##BUDIBASE_APP_ID##"] = "{{appId}}"
</script>
{{{body}}}
</html>

14
packages/server/src/api/routes/hosting.js Normal file
View File

@ -0,0 +1,14 @@
const Router = require("@koa/router")
const controller = require("../controllers/hosting")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("../../utilities/security/permissions")
const router = Router()
router
.get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo)
.get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
.get("/api/hosting", authorized(BUILDER), controller.fetch)
.post("/api/hosting", authorized(BUILDER), controller.save)
module.exports = router

2
packages/server/src/api/routes/index.js
View File

@ -17,6 +17,7 @@ const templatesRoutes = require("./templates")
const analyticsRoutes = require("./analytics")
const routingRoutes = require("./routing")
const permissionRoutes = require("./permission")
const hostingRoutes = require("./hosting")
exports.mainRoutes = [
deployRoutes,
@ -34,6 +35,7 @@ exports.mainRoutes = [
webhookRoutes,
routingRoutes,
permissionRoutes,
hostingRoutes,
// these need to be handled last as they still use /api/:tableId
// this could be breaking as koa may recognise other routes as this
tableRoutes,

6
packages/server/src/app.js
View File

@ -9,6 +9,7 @@ const env = require("./environment")
const eventEmitter = require("./events")
const automations = require("./automations/index")
const Sentry = require("@sentry/node")
const selfhost = require("./selfhost")
const app = new Koa()
@ -49,9 +50,12 @@ destroyable(server)
server.on("close", () => console.log("Server Closed"))
module.exports = server.listen(env.PORT || 4001, () => {
module.exports = server.listen(env.PORT || 4001, async () => {
console.log(`Budibase running on ${JSON.stringify(server.address())}`)
automations.init()
if (env.SELF_HOSTED) {
await selfhost.init()
}
})
process.on("uncaughtException", err => {

2
packages/server/src/constants/index.js
View File

@ -41,3 +41,5 @@ const USERS_TABLE_SCHEMA = {
exports.AuthTypes = AuthTypes
exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA
exports.BUILDER_CONFIG_DB = "builder-config-db"
exports.HOSTING_DOC = "hosting-doc"

5
packages/server/src/constants/layouts.js
View File

@ -1,3 +1,5 @@
const { getLogoUrl } = require("../utilities")
const BASE_LAYOUT_PROP_IDS = {
PRIVATE: "layout_private_master",
PUBLIC: "layout_public_master",
@ -107,8 +109,7 @@ const BASE_LAYOUTS = [
active: {},
selected: {},
},
logoUrl:
"https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg",
logoUrl: getLogoUrl(),
title: "",
backgroundColor: "",
color: "",

4
packages/server/src/constants/screens.js
View File

@ -1,5 +1,6 @@
const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
const { getLogoUrl } = require("../utilities")
exports.createHomeScreen = () => ({
description: "",
@ -138,8 +139,7 @@ exports.createLoginScreen = app => ({
active: {},
selected: {},
},
logo:
"https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg",
logo: getLogoUrl(),
title: `Log in to ${app.name}`,
buttonText: "Log In",
_children: [],

3
packages/server/src/environment.js
View File

@ -27,6 +27,7 @@ module.exports = {
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
CLOUD: process.env.CLOUD,
SELF_HOSTED: process.env.SELF_HOSTED,
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
AWS_REGION: process.env.AWS_REGION,
DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
@ -35,6 +36,8 @@ module.exports = {
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
// self hosting features
LOGO_URL: process.env.LOGO_URL,
_set(key, value) {
process.env[key] = value
module.exports[key] = value

8
packages/server/src/middleware/authorized.js
View File

@ -7,7 +7,7 @@ const {
doesHavePermission,
} = require("../utilities/security/permissions")
const env = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient")
const { isAPIKeyValid } = require("../utilities/security/apikey")
const { AuthTypes } = require("../constants")
const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER]
@ -21,11 +21,7 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
}
if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
// api key header passed by external webhook
const apiKeyInfo = await apiKeyTable.get({
primary: ctx.headers["x-api-key"],
})
if (apiKeyInfo) {
if (await isAPIKeyValid(ctx.headers["x-api-key"])) {
ctx.auth = {
authenticated: AuthTypes.EXTERNAL,
apiKey: ctx.headers["x-api-key"],

7
packages/server/src/middleware/usageQuota.js
View File

@ -43,6 +43,10 @@ module.exports = async (ctx, next) => {
return
}
}
// if running in builder or a self hosted cloud usage quotas should not be executed
if (!env.CLOUD || env.SELF_HOSTED) {
return next()
}
// update usage for uploads to be the total size
if (property === usageQuota.Properties.UPLOAD) {
const files =
@ -51,9 +55,6 @@ module.exports = async (ctx, next) => {
: [ctx.request.files.file]
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
if (!env.CLOUD) {
return next()
}
try {
await usageQuota.update(ctx.auth.apiKey, property, usage)
return next()

7
packages/server/src/selfhost/README.md Normal file
View File

@ -0,0 +1,7 @@
### Self hosting
This directory contains utilities that are needed for self hosted platforms to operate.
These will mostly be utilities, necessary to the operation of the server e.g. storing self
hosting specific options and attributes to CouchDB.
All the internal operations should be exposed through the `index.js` so importing
the self host directory should give you everything you need.

44
packages/server/src/selfhost/index.js Normal file
View File

@ -0,0 +1,44 @@
const CouchDB = require("../db")
const env = require("../environment")
const newid = require("../db/newid")
const SELF_HOST_DB = "self-host-db"
const SELF_HOST_DOC = "self-host-info"
async function createSelfHostDB(db) {
await db.put({
_id: "_design/database",
views: {},
})
const selfHostInfo = {
_id: SELF_HOST_DOC,
apiKeyId: newid(),
}
await db.put(selfHostInfo)
return selfHostInfo
}
exports.init = async () => {
if (!env.SELF_HOSTED) {
return
}
const db = new CouchDB(SELF_HOST_DB)
try {
await db.get(SELF_HOST_DOC)
} catch (err) {
// failed to retrieve
if (err.status === 404) {
await createSelfHostDB(db)
}
}
}
exports.getSelfHostInfo = async () => {
const db = new CouchDB(SELF_HOST_DB)
return db.get(SELF_HOST_DOC)
}
exports.getSelfHostAPIKey = async () => {
const info = await exports.getSelfHostInfo()
return info ? info.apiKeyId : null
}

83
packages/server/src/utilities/builder/hosting.js Normal file
View File

@ -0,0 +1,83 @@
const CouchDB = require("../../db")
const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
const PROD_HOSTING_URL = "app.budi.live"
function getProtocol(hostingInfo) {
return hostingInfo.useHttps ? "https://" : "http://"
}
async function getURLWithPath(pathIfSelfHosted) {
const hostingInfo = await exports.getHostingInfo()
const protocol = getProtocol(hostingInfo)
const path =
hostingInfo.type === exports.HostingTypes.SELF ? pathIfSelfHosted : ""
return `${protocol}${hostingInfo.hostingUrl}${path}`
}
exports.HostingTypes = {
CLOUD: "cloud",
SELF: "self",
}
exports.getHostingInfo = async () => {
const db = new CouchDB(BUILDER_CONFIG_DB)
let doc
try {
doc = await db.get(HOSTING_DOC)
} catch (err) {
// don't write this doc, want to be able to update these default props
// for our servers with a new release without needing to worry about state of
// PouchDB in peoples installations
doc = {
_id: HOSTING_DOC,
type: exports.HostingTypes.CLOUD,
hostingUrl: PROD_HOSTING_URL,
selfHostKey: "",
templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
useHttps: true,
}
}
return doc
}
exports.getAppUrl = async appId => {
const hostingInfo = await exports.getHostingInfo()
const protocol = getProtocol(hostingInfo)
let url
if (hostingInfo.type === exports.HostingTypes.CLOUD) {
url = `${protocol}${appId}.${hostingInfo.hostingUrl}`
} else {
url = `${protocol}${hostingInfo.hostingUrl}/app`
}
return url
}
exports.getWorkerUrl = async () => {
return getURLWithPath("/worker")
}
exports.getMinioUrl = async () => {
return getURLWithPath("/")
}
exports.getCouchUrl = async () => {
return getURLWithPath("/db")
}
exports.getSelfHostKey = async () => {
const hostingInfo = await exports.getHostingInfo()
return hostingInfo.selfHostKey
}
exports.getTemplatesUrl = async (appId, type, name) => {
const hostingInfo = await exports.getHostingInfo()
const protocol = getProtocol(hostingInfo)
let path
if (type && name) {
path = `templates/type/${name}.tar.gz`
} else {
path = "manifest.json"
}
return `${protocol}${hostingInfo.templatesUrl}/${path}`
}

13
packages/server/src/utilities/index.js
View File

@ -168,3 +168,16 @@ exports.coerceRowValues = (row, table) => {
}
return clonedRow
}
/**
* Gets the correct link to the logo URL depending on if running in Cloud or if running in self hosting.
* @returns {string} A URL which links to the correct default logo for new apps.
*/
exports.getLogoUrl = () => {
const BB_LOGO_URL =
"https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg"
if (env.SELF_HOSTED) {
return env.LOGO_URL || BB_LOGO_URL
}
return BB_LOGO_URL
}

23
packages/server/src/utilities/security/apikey.js Normal file
View File

@ -0,0 +1,23 @@
const { apiKeyTable } = require("../../db/dynamoClient")
const env = require("../../environment")
const { getSelfHostAPIKey } = require("../../selfhost")
/**
* This file purely exists so that we can centralise all logic pertaining to API keys, as their usage differs
* in our Cloud environment versus self hosted.
*/
exports.isAPIKeyValid = async apiKeyId => {
if (env.CLOUD && !env.SELF_HOSTED) {
let apiKeyInfo = await apiKeyTable.get({
primary: apiKeyId,
})
return apiKeyInfo != null
}
if (env.SELF_HOSTED) {
const selfHostKey = await getSelfHostAPIKey()
// if the api key supplied is correct then return structure similar
return apiKeyId === selfHostKey ? { pk: apiKeyId } : null
}
return false
}

849
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff

2
packages/worker/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
node_modules/
.env

15
packages/worker/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM node:12-alpine
WORKDIR /app
# copy files and install dependencies
COPY . ./
RUN yarn
EXPOSE 4001
# have to add node environment production after install
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
CMD ["yarn", "run:docker"]

34
packages/worker/package.json Normal file
View File

@ -0,0 +1,34 @@
{
"name": "@budibase/deployment",
"email": "hi@budibase.com",
"version": "0.3.8",
"description": "Budibase Deployment Server",
"main": "src/index.js",
"repository": {
"type": "git",
"url": "https://github.com/Budibase/budibase.git"
},
"keywords": [
"budibase"
],
"scripts": {
"run:docker": "node src/index.js"
},
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@koa/router": "^8.0.0",
"aws-sdk": "^2.811.0",
"got": "^11.8.1",
"joi": "^17.2.1",
"koa": "^2.7.0",
"koa-body": "^4.2.0",
"koa-compress": "^4.0.1",
"koa-pino-logger": "^3.0.0",
"koa-send": "^5.0.0",
"koa-session": "^5.12.0",
"koa-static": "^5.0.0",
"pino-pretty": "^4.0.0",
"server-destroy": "^1.0.1"
}
}

92
packages/worker/src/api/controllers/deploy.js Normal file
View File

@ -0,0 +1,92 @@
const env = require("../../environment")
const got = require("got")
const AWS = require("aws-sdk")
const APP_BUCKET = "app-assets"
// this doesn't matter in self host
const REGION = "eu-west-1"
const PUBLIC_READ_POLICY = {
Version: "2012-10-17",
Statement: [
{
Effect: "Allow",
Principal: {
AWS: ["*"],
},
Action: "s3:GetObject",
Resource: [`arn:aws:s3:::${APP_BUCKET}/*`],
},
],
}
async function getCouchSession() {
// fetch session token for the api user
const session = await got.post(`${env.RAW_COUCH_DB_URL}/_session`, {
responseType: "json",
credentials: "include",
json: {
username: env.COUCH_DB_USERNAME,
password: env.COUCH_DB_PASSWORD,
},
})
const cookie = session.headers["set-cookie"][0]
// Get the session cookie value only
return cookie.split(";")[0]
}
async function getMinioSession() {
AWS.config.update({
accessKeyId: env.MINIO_ACCESS_KEY,
secretAccessKey: env.MINIO_SECRET_KEY,
})
// make sure the bucket exists
const objClient = new AWS.S3({
endpoint: env.RAW_MINIO_URL,
region: REGION,
s3ForcePathStyle: true, // needed with minio?
params: {
Bucket: APP_BUCKET,
},
})
// make sure the bucket exists
try {
await objClient
.headBucket({
Bucket: APP_BUCKET,
})
.promise()
} catch (err) {
// bucket doesn't exist create it
if (err.statusCode === 404) {
await objClient
.createBucket({
Bucket: APP_BUCKET,
})
.promise()
} else {
throw err
}
}
// always make sure policy is correct
await objClient
.putBucketPolicy({
Bucket: APP_BUCKET,
Policy: JSON.stringify(PUBLIC_READ_POLICY),
})
.promise()
// Ideally want to send back some pre-signed URLs for files that are to be uploaded
return {
accessKeyId: env.MINIO_ACCESS_KEY,
secretAccessKey: env.MINIO_SECRET_KEY,
}
}
exports.deploy = async ctx => {
ctx.body = {
couchDbSession: await getCouchSession(),
bucket: APP_BUCKET,
objectStoreSession: await getMinioSession(),
}
}

45
packages/worker/src/api/index.js Normal file
View File

@ -0,0 +1,45 @@
const Router = require("@koa/router")
const compress = require("koa-compress")
const zlib = require("zlib")
const { routes } = require("./routes")
const router = new Router()
router
.use(
compress({
threshold: 2048,
gzip: {
flush: zlib.Z_SYNC_FLUSH,
},
deflate: {
flush: zlib.Z_SYNC_FLUSH,
},
br: false,
})
)
.use("/health", ctx => (ctx.status = 200))
// error handling middleware
router.use(async (ctx, next) => {
try {
await next()
} catch (err) {
ctx.log.error(err)
ctx.status = err.status || err.statusCode || 500
ctx.body = {
message: err.message,
status: ctx.status,
}
}
})
router.get("/health", ctx => (ctx.status = 200))
// authenticated routes
for (let route of routes) {
router.use(route.routes())
router.use(route.allowedMethods())
}
module.exports = router

9
packages/worker/src/api/routes/deploy.js Normal file
View File

@ -0,0 +1,9 @@
const Router = require("@koa/router")
const controller = require("../controllers/deploy")
const checkKey = require("../../middleware/check-key")
const router = Router()
router.post("/api/deploy", checkKey, controller.deploy)
module.exports = router

3
packages/worker/src/api/routes/index.js Normal file
View File

@ -0,0 +1,3 @@
const deployRoutes = require("./deploy")
exports.routes = [deployRoutes]

18
packages/worker/src/environment.js Normal file
View File

@ -0,0 +1,18 @@
module.exports = {
SELF_HOSTED: process.env.SELF_HOSTED,
WORKER_API_KEY: process.env.WORKER_API_KEY,
PORT: process.env.PORT,
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
RAW_COUCH_DB_URL: process.env.RAW_COUCH_DB_URL,
RAW_MINIO_URL: process.env.RAW_MINIO_URL,
COUCH_DB_PORT: process.env.COUCH_DB_PORT,
MINIO_PORT: process.env.MINIO_PORT,
SELF_HOST_KEY: process.env.SELF_HOST_KEY,
_set(key, value) {
process.env[key] = value
module.exports[key] = value
},
}

48
packages/worker/src/index.js Normal file
View File

@ -0,0 +1,48 @@
const Koa = require("koa")
const destroyable = require("server-destroy")
const koaBody = require("koa-body")
const logger = require("koa-pino-logger")
const http = require("http")
const api = require("./api")
const env = require("./environment")
const app = new Koa()
if (!env.SELF_HOSTED) {
throw "Currently this service only supports use in self hosting"
}
// set up top level koa middleware
app.use(koaBody({ multipart: true }))
app.use(
logger({
prettyPrint: {
levelFirst: true,
},
level: env.LOG_LEVEL || "error",
})
)
// api routes
app.use(api.routes())
const server = http.createServer(app.callback())
destroyable(server)
server.on("close", () => console.log("Server Closed"))
module.exports = server.listen(env.PORT || 4002, async () => {
console.log(`Worker running on ${JSON.stringify(server.address())}`)
})
process.on("uncaughtException", err => {
console.error(err)
server.close()
server.destroy()
})
process.on("SIGTERM", () => {
server.close()
server.destroy()
})

12
packages/worker/src/middleware/check-key.js Normal file
View File

@ -0,0 +1,12 @@
const env = require("../environment")
module.exports = async (ctx, next) => {
if (
!ctx.request.body.selfHostKey ||
env.SELF_HOST_KEY !== ctx.request.body.selfHostKey
) {
ctx.throw(401, "Deployment unauthorised")
} else {
await next()
}
}

1166
packages/worker/yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff