diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 817cd17652..68e71f4cf7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,3 +61,12 @@ jobs: # macOS notarization API key API_KEY_ID: ${{ secrets.api_key_id }} API_KEY_ISSUER_ID: ${{ secrets.api_key_issuer_id }} + + - name: Build/release Docker images + # only run the docker image build on linux, easiest way + if: startsWith(matrix.os, 'ubuntu') + env: + DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} + run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + run: yarn build:docker diff --git a/.gitignore b/.gitignore index 54401c6b36..edad41cdec 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ typings/ # dotenv environment variables file .env +!hosting/.env # parcel-bundler cache (https://parceljs.org/) .cache diff --git a/hosting/.env b/hosting/.env new file mode 120000 index 0000000000..bb1b54ad77 --- /dev/null +++ b/hosting/.env @@ -0,0 +1 @@ +hosting.properties \ No newline at end of file diff --git a/hosting/build/docker-compose.yaml b/hosting/build/docker-compose.yaml new file mode 100644 index 0000000000..6988e3841b --- /dev/null +++ b/hosting/build/docker-compose.yaml @@ -0,0 +1,16 @@ +version: "3" + +services: + app-service: + build: ./server + volumes: + - ./server:/app + environment: + SELF_HOSTED: 1 + PORT: 4002 + + worker-service: + build: ./worker + environment: + SELF_HOSTED: 1, + PORT: 4003 diff --git a/hosting/build/server b/hosting/build/server new file mode 120000 index 0000000000..c40730cce5 --- /dev/null +++ b/hosting/build/server @@ -0,0 +1 @@ +../../packages/server/ \ No newline at end of file diff --git a/hosting/build/worker b/hosting/build/worker new file mode 120000 index 0000000000..8582fefbee --- /dev/null +++ b/hosting/build/worker @@ -0,0 +1 @@ +../../packages/worker/ \ No newline at end of file diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml new file mode 100644 index 0000000000..32659ec53c --- /dev/null +++ b/hosting/docker-compose.yaml @@ -0,0 +1,90 @@ +version: "3" + +services: + app-service: + image: budibase/budibase-apps + ports: + - "${APP_PORT}:4002" + environment: + SELF_HOSTED: 1 + COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 + BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} + LOGO_URL: ${LOGO_URL} + PORT: 4002 + JWT_SECRET: ${JWT_SECRET} + depends_on: + - worker-service + + worker-service: + image: budibase/budibase-worker + ports: + - "${WORKER_PORT}:4003" + environment: + SELF_HOSTED: 1, + DEPLOYMENT_API_KEY: ${WORKER_API_KEY} + PORT: 4003 + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + RAW_MINIO_URL: http://minio-service:9000 + COUCH_DB_USERNAME: ${COUCH_DB_USER} + COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} + RAW_COUCH_DB_URL: http://couchdb-service:5984 + SELF_HOST_KEY: ${HOSTING_KEY} + depends_on: + - minio-service + - couch-init + + minio-service: + image: minio/minio + volumes: + - minio_data:/data + ports: + - "${MINIO_PORT}:9000" + environment: + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + command: server /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + proxy-service: + image: envoyproxy/envoy:v1.16-latest + volumes: + - ./envoy.yaml:/etc/envoy/envoy.yaml + ports: + - "${MAIN_PORT}:10000" + - "9901:9901" + depends_on: + - minio-service + - worker-service + - app-service + - couchdb-service + + couchdb-service: + image: apache/couchdb:3.0 + environment: + - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} + - COUCHDB_USER=${COUCH_DB_USER} + ports: + - "${COUCH_DB_PORT}:5984" + - "4369:4369" + - "9100:9100" + volumes: + - couchdb_data:/couchdb + + couch-init: + image: curlimages/curl + environment: + PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" + depends_on: + - couchdb-service + command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] + +volumes: + couchdb_data: + driver: local + minio_data: + driver: local diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml new file mode 100644 index 0000000000..3c816cb1ca --- /dev/null +++ b/hosting/envoy.yaml @@ -0,0 +1,104 @@ +static_resources: + listeners: + - name: main_listener + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress + codec_type: auto + route_config: + name: local_route + virtual_hosts: + - name: local_services + domains: ["*"] + routes: + - match: { prefix: "/app/" } + route: + cluster: app-service + prefix_rewrite: "/" + + # special case for when API requests are made, can just forward, not to minio + - match: { prefix: "/api/" } + route: + cluster: app-service + + - match: { prefix: "/worker/" } + route: + cluster: worker-service + prefix_rewrite: "/" + + - match: { prefix: "/db/" } + route: + cluster: couchdb-service + prefix_rewrite: "/" + + # minio is on the default route because this works + # best, minio + AWS SDK doesn't handle path proxy + - match: { prefix: "/" } + route: + cluster: minio-service + + http_filters: + - name: envoy.filters.http.router + + clusters: + - name: app-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: app-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: app-service + port_value: 4002 + + - name: minio-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: minio-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: minio-service + port_value: 9000 + + - name: worker-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: worker-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: worker-service + port_value: 4003 + + - name: couchdb-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: couchdb-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: couchdb-service + port_value: 5984 + diff --git a/hosting/hosting.properties b/hosting/hosting.properties new file mode 100644 index 0000000000..5776f108e3 --- /dev/null +++ b/hosting/hosting.properties @@ -0,0 +1,25 @@ +# Use the main port in the builder for your self hosting URL, e.g. localhost:10000 +MAIN_PORT=10000 + +# Use this password when configuring your self hosting settings +# This should be updated +HOSTING_KEY=budibase + +# This section contains customisation options +LOGO_URL=https://logoipsum.com/logo/logo-15.svg + +# This section contains all secrets pertaining to the system +# These should be updated +JWT_SECRET=testsecret +MINIO_ACCESS_KEY=budibase +MINIO_SECRET_KEY=budibase +COUCH_DB_PASSWORD=budibase +COUCH_DB_USER=budibase +WORKER_API_KEY=budibase + +# This section contains variables that do not need to be altered under normal circumstances +APP_PORT=4002 +WORKER_PORT=4003 +MINIO_PORT=4004 +COUCH_DB_PORT=4005 +BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/hosting/scripts/linux/install-docker-compose.sh b/hosting/scripts/linux/install-docker-compose.sh new file mode 100755 index 0000000000..6d466a7655 --- /dev/null +++ b/hosting/scripts/linux/install-docker-compose.sh @@ -0,0 +1,4 @@ +#!/bin/bash +sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose diff --git a/hosting/scripts/linux/install-docker.sh b/hosting/scripts/linux/install-docker.sh new file mode 100755 index 0000000000..a71809c31f --- /dev/null +++ b/hosting/scripts/linux/install-docker.sh @@ -0,0 +1,6 @@ +#!/bin/bash +echo "**** WARNING - not for production environments ****" +# warning this is a convience script, for production installations install docker +# properly for your environment! +curl -fsSL https://get.docker.com -o get-docker.sh +sudo sh get-docker.sh diff --git a/hosting/scripts/linux/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh new file mode 100755 index 0000000000..b1921916e4 --- /dev/null +++ b/hosting/scripts/linux/release-to-docker-hub.sh @@ -0,0 +1,9 @@ +#!/bin/bash +pushd ../../build +docker-compose build --force app-service +docker-compose build --force worker-service +docker tag build_app-service budibase/budibase-apps:latest +docker push budibase/budibase-apps +docker tag build_worker-service budibase/budibase-worker:latest +docker push budibase/budibase-worker +popd diff --git a/hosting/start.sh b/hosting/start.sh new file mode 100755 index 0000000000..b32098a3b7 --- /dev/null +++ b/hosting/start.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker-compose --env-file hosting.properties up diff --git a/hosting/utils/testing.sh b/hosting/utils/testing.sh new file mode 100755 index 0000000000..94f2fe8896 --- /dev/null +++ b/hosting/utils/testing.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function dockerInstalled { + echo "Checking docker installation..." + if [ ! -x "$(command -v docker)" ]; then + echo "Please install docker to continue" + exit -1 + fi +} + +dockerInstalled + +source "${BASH_SOURCE%/*}/hosting.properties" + +opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key" +if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then + opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old" +fi + +docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data diff --git a/package.json b/package.json index d58e36517d..fd601dd099 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ "lint:fix": "eslint --fix packages", "format": "prettier --write \"{,!(node_modules)/**/}*.{js,jsx,svelte}\"", "test:e2e": "lerna run cy:test", - "test:e2e:ci": "lerna run cy:ci" + "test:e2e:ci": "lerna run cy:ci", + "build:docker": "cd hosting/scripts/linux/ && ./release-to-docker-hub.sh && cd -" }, "dependencies": { "@fortawesome/fontawesome": "^1.1.8" diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index cf668670bb..ca2864aeaf 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -1,6 +1,8 @@ import { getFrontendStore } from "./store/frontend" import { getBackendUiStore } from "./store/backend" -import { getAutomationStore } from "./store/automation/" +import { getAutomationStore } from "./store/automation" +import { getHostingStore } from "./store/hosting" + import { getThemeStore } from "./store/theme" import { derived, writable } from "svelte/store" import analytics from "analytics" @@ -11,6 +13,7 @@ export const store = getFrontendStore() export const backendUiStore = getBackendUiStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() +export const hostingStore = getHostingStore() export const currentAsset = derived(store, $store => { const type = $store.currentFrontEndType diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 1e451ff63e..cd4d22515c 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -7,6 +7,7 @@ import { import { allScreens, backendUiStore, + hostingStore, currentAsset, mainLayout, selectedComponent, @@ -71,6 +72,7 @@ export const getFrontendStore = () => { appInstance: pkg.application.instance, })) + await hostingStore.actions.fetch() await backendUiStore.actions.database.select(pkg.application.instance) }, routing: { diff --git a/packages/builder/src/builderStore/store/hosting.js b/packages/builder/src/builderStore/store/hosting.js new file mode 100644 index 0000000000..36067773b5 --- /dev/null +++ b/packages/builder/src/builderStore/store/hosting.js @@ -0,0 +1,38 @@ +import { writable } from "svelte/store" +import api from "../api" + +const INITIAL_BACKEND_UI_STATE = { + hostingInfo: {}, + appUrl: "", +} + +export const getHostingStore = () => { + const store = writable({ ...INITIAL_BACKEND_UI_STATE }) + store.actions = { + fetch: async () => { + const responses = await Promise.all([ + api.get("/api/hosting/"), + api.get("/api/hosting/urls"), + ]) + const [info, urls] = await Promise.all(responses.map(resp => resp.json())) + store.update(state => { + state.hostingInfo = info + state.appUrl = urls.app + return state + }) + return info + }, + save: async hostingInfo => { + const response = await api.post("/api/hosting", hostingInfo) + const revision = (await response.json()).rev + store.update(state => { + state.hostingInfo = { + ...hostingInfo, + _rev: revision, + } + return state + }) + }, + } + return store +} diff --git a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte index 3299206e78..2edcbec3f1 100644 --- a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte +++ b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte @@ -1,16 +1,17 @@ - -
- -
- - - - - diff --git a/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte b/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte deleted file mode 100644 index dfd7c11141..0000000000 --- a/packages/builder/src/components/backend/DataTable/modals/EditIntegrationConfig.svelte +++ /dev/null @@ -1,129 +0,0 @@ - - -
-
-
Schema
- {#if smartSchemaRow} - 
{JSON.stringify(smartSchemaRow, undefined, 2)}
- {/if} - {#each fields as field, idx} -
- - - deleteField(idx)} /> -
- {/each} - - -
- -
-
Datasource
- {#each Object.keys(table.integration) as configKey} - {#if configKey === 'query'} -