From ac6b2aa6c8fb9a288ace0e5f8ba67952a4340234 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Nov 2020 18:04:47 +0000 Subject: [PATCH 01/48] Initial work, layout out some work which will eventually become docker compose, just using bash scripts for now. --- hosting/hosting.properties | 6 + hosting/start.sh | 20 ++ .../server/scripts/createApiKeyAndAppId.js | 56 ----- packages/server/src/environment.js | 1 + packages/server/yarn.lock | 200 +----------------- 5 files changed, 30 insertions(+), 253 deletions(-) create mode 100644 hosting/hosting.properties create mode 100755 hosting/start.sh delete mode 100644 packages/server/scripts/createApiKeyAndAppId.js diff --git a/hosting/hosting.properties b/hosting/hosting.properties new file mode 100644 index 0000000000..ce3ab326c9 --- /dev/null +++ b/hosting/hosting.properties @@ -0,0 +1,6 @@ +minio_access_key=budibase +minio_secret_key=budibase +minio_port=9000 +# specify the following below settings if switching credentials +#minio_secret_key_old=minioadmin +#minio_access_key_old=minioadmin diff --git a/hosting/start.sh b/hosting/start.sh new file mode 100755 index 0000000000..94f2fe8896 --- /dev/null +++ b/hosting/start.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function dockerInstalled { + echo "Checking docker installation..." + if [ ! -x "$(command -v docker)" ]; then + echo "Please install docker to continue" + exit -1 + fi +} + +dockerInstalled + +source "${BASH_SOURCE%/*}/hosting.properties" + +opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key" +if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then + opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old" +fi + +docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data diff --git a/packages/server/scripts/createApiKeyAndAppId.js b/packages/server/scripts/createApiKeyAndAppId.js deleted file mode 100644 index 76b0893d30..0000000000 --- a/packages/server/scripts/createApiKeyAndAppId.js +++ /dev/null @@ -1,56 +0,0 @@ -// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running -const dynamoClient = require("../src/db/dynamoClient") -const env = require("../src/environment") - -if (process.argv[2] == null || process.argv[3] == null) { - console.error( - "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js " - ) - process.exit(-1) -} - -const FAKE_STRING = "fakestring" - -// set fake credentials for local dynamo to actually work -env._set("AWS_ACCESS_KEY_ID", "KEY_ID") -env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY") -dynamoClient.init("http://localhost:8333") - -async function run() { - await dynamoClient.apiKeyTable.put({ - item: { - pk: process.argv[2], - accountId: FAKE_STRING, - trackingId: FAKE_STRING, - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - users: 0, - views: 0, - }, - usageLimits: { - automationRuns: 10, - rows: 10, - storage: 1000, - users: 10, - views: 10, - }, - }, - }) - await dynamoClient.apiKeyTable.put({ - item: { - pk: process.argv[3], - apiKey: process.argv[2], - }, - }) -} - -run() - .then(() => { - console.log("Rows should have been created.") - }) - .catch(err => { - console.error("Cannot create rows - " + err) - }) diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 003390c502..5fbf57448a 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -27,6 +27,7 @@ module.exports = { BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, CLOUD: process.env.CLOUD, + SELF_HOSTED: process.env.SELF_HOSTED, DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, AWS_REGION: process.env.AWS_REGION, DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index ef8f515468..d4dadec135 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -200,15 +200,6 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" -"@budibase/client@^0.3.8": - version "0.3.8" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.3.8.tgz#75df7e97e8f0d9b58c00e2bb0d3b4a55f8d04735" - integrity sha512-tnFdmCdXKS+uZGoipr69Wa0oVoFHmyoV0ydihI6q0gKQH0KutypVHAaul2qPB8t5a/mTZopC//2WdmCeX1GKVg== - dependencies: - deep-equal "^2.0.1" - mustache "^4.0.1" - regexparam "^1.3.0" - "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -1236,11 +1227,6 @@ array-equal@^1.0.0: resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93" integrity sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM= -array-filter@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83" - integrity sha1-uveeYubvTCpMC4MSMtr/7CUfnYM= - array-unique@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" @@ -1313,13 +1299,6 @@ atomic-sleep@^1.0.0: resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b" integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ== -available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz#6b098ca9d8039079ee3f77f7b783c4480ba513f5" - integrity sha512-XWX3OX8Onv97LMk/ftVyBibpGwY5a8SmuxZPzeOxqmuEqUCOM9ZE+uIaD1VNJ5QnvU2UQusvmKbuM1FR8QWGfQ== - dependencies: - array-filter "^1.0.0" - aws-sdk@^2.767.0: version "2.771.0" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.771.0.tgz#ff4beb0a04d6ab1ae962c85dfb42e3e9bfe2b93b" @@ -1687,14 +1666,6 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" -call-bind@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.0.tgz#24127054bb3f9bdcb4b1fb82418186072f77b8ce" - integrity sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w== - dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.0" - callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" @@ -2200,26 +2171,6 @@ decompress@^4.2.1: pify "^2.3.0" strip-dirs "^2.0.0" -deep-equal@^2.0.1: - version "2.0.4" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.0.4.tgz#6b0b407a074666033169df3acaf128e1c6f3eab6" - integrity sha512-BUfaXrVoCfgkOQY/b09QdO9L3XNoF2XH0A3aY9IQwQL/ZjLOe8FQgCNVl1wiolhsFo8kFdO9zdPViCPbmaJA5w== - dependencies: - es-abstract "^1.18.0-next.1" - es-get-iterator "^1.1.0" - is-arguments "^1.0.4" - is-date-object "^1.0.2" - is-regex "^1.1.1" - isarray "^2.0.5" - object-is "^1.1.3" - object-keys "^1.1.1" - object.assign "^4.1.1" - regexp.prototype.flags "^1.3.0" - side-channel "^1.0.3" - which-boxed-primitive "^1.0.1" - which-collection "^1.0.1" - which-typed-array "^1.1.2" - deep-equal@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" @@ -2601,7 +2552,7 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.17.0-next.1, es-abstract@^1.17.2, es-abstract@^1.17.4, es-abstract@^1.17.5: +es-abstract@^1.17.0-next.1, es-abstract@^1.17.2, es-abstract@^1.17.5: version "1.17.7" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.17.7.tgz#a4de61b2f66989fc7421676c1cb9787573ace54c" integrity sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g== @@ -2618,7 +2569,7 @@ es-abstract@^1.17.0-next.1, es-abstract@^1.17.2, es-abstract@^1.17.4, es-abstrac string.prototype.trimend "^1.0.1" string.prototype.trimstart "^1.0.1" -es-abstract@^1.18.0-next.0, es-abstract@^1.18.0-next.1: +es-abstract@^1.18.0-next.0: version "1.18.0-next.1" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.1.tgz#6e3a0a4bda717e5023ab3b8e90bec36108d22c68" integrity sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA== @@ -2636,20 +2587,6 @@ es-abstract@^1.18.0-next.0, es-abstract@^1.18.0-next.1: string.prototype.trimend "^1.0.1" string.prototype.trimstart "^1.0.1" -es-get-iterator@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.1.tgz#b93ddd867af16d5118e00881396533c1c6647ad9" - integrity sha512-qorBw8Y7B15DVLaJWy6WdEV/ZkieBcu6QCq/xzWzGOKJqgG1j754vXRfZ3NY7HSShneqU43mPB4OkQBTkvHhFw== - dependencies: - call-bind "^1.0.0" - get-intrinsic "^1.0.1" - has-symbols "^1.0.1" - is-arguments "^1.0.4" - is-map "^2.0.1" - is-set "^2.0.1" - is-string "^1.0.5" - isarray "^2.0.5" - es-to-primitive@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" @@ -3327,15 +3264,6 @@ get-caller-file@^2.0.1, get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.0, get-intrinsic@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.0.1.tgz#94a9768fcbdd0595a1c9273aacf4c89d075631be" - integrity sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg== - dependencies: - function-bind "^1.1.1" - has "^1.0.3" - has-symbols "^1.0.1" - get-stream@3.0.0, get-stream@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" @@ -3860,21 +3788,11 @@ is-accessor-descriptor@^1.0.0: dependencies: kind-of "^6.0.0" -is-arguments@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.0.4.tgz#3faf966c7cba0ff437fb31f6250082fcf0448cf3" - integrity sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA== - is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= -is-bigint@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.0.tgz#73da8c33208d00f130e9b5e15d23eac9215601c4" - integrity sha512-t5mGUXC/xRheCK431ylNiSkGGpBp8bHENBcENTkDT6ppwPzEVxNGZRvgvmOEfbWkFhA7D2GEuE2mmQTr78sl2g== - is-binary-path@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" @@ -3882,11 +3800,6 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-boolean-object@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.0.1.tgz#10edc0900dd127697a92f6f9807c7617d68ac48e" - integrity sha512-TqZuVwa/sppcrhUCAYkGBk7w0yxfQQnxq28fjkO53tnK9FQXmdwz2JS5+GjsWQ6RByES1K40nI+yDic5c9/aAQ== - is-buffer@^1.1.5: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" @@ -3923,7 +3836,7 @@ is-data-descriptor@^1.0.0: dependencies: kind-of "^6.0.0" -is-date-object@^1.0.1, is-date-object@^1.0.2: +is-date-object@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e" integrity sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g== @@ -4008,11 +3921,6 @@ is-installed-globally@^0.3.1: global-dirs "^2.0.1" is-path-inside "^3.0.1" -is-map@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.1.tgz#520dafc4307bb8ebc33b813de5ce7c9400d644a1" - integrity sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw== - is-natural-number@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/is-natural-number/-/is-natural-number-4.0.1.tgz#ab9d76e1db4ced51e35de0c72ebecf09f734cde8" @@ -4028,11 +3936,6 @@ is-npm@^4.0.0: resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d" integrity sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig== -is-number-object@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.4.tgz#36ac95e741cf18b283fc1ddf5e83da798e3ec197" - integrity sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw== - is-number@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" @@ -4084,21 +3987,11 @@ is-retry-allowed@^1.1.0: resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== -is-set@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.1.tgz#d1604afdab1724986d30091575f54945da7e5f43" - integrity sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA== - is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= -is-string@^1.0.4, is-string@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6" - integrity sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ== - is-symbol@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.3.tgz#38e1014b9e6329be0de9d24a414fd7441ec61937" @@ -4115,16 +4008,6 @@ is-type-of@^1.0.0: is-class-hotfix "~0.0.6" isstream "~0.1.2" -is-typed-array@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.3.tgz#a4ff5a5e672e1a55f99c7f54e59597af5c1df04d" - integrity sha512-BSYUBOK/HJibQ30wWkWold5txYwMUXQct9YHAQJr8fSwvZoiglcqB0pd7vEN23+Tsi9IUEjztdOSzl4qLVYGTQ== - dependencies: - available-typed-arrays "^1.0.0" - es-abstract "^1.17.4" - foreach "^2.0.5" - has-symbols "^1.0.1" - is-typedarray@^1.0.0, is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -4135,16 +4018,6 @@ is-utf8@^0.2.0: resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= -is-weakmap@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2" - integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA== - -is-weakset@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.1.tgz#e9a0af88dbd751589f5e50d80f4c98b780884f83" - integrity sha512-pi4vhbhVHGLxohUw7PhGsueT4vRGFoXhP7+RGN0jKIv9+8PWYCQTqtADngrxOm2g46hoH0+g8uZZBzMrvVGDmw== - is-windows@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" @@ -4177,11 +4050,6 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= -isarray@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" - integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== - isbinaryfile@^4.0.6: version "4.0.6" resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.6.tgz#edcb62b224e2b4710830b67498c8e4e5a4d2610b" @@ -5720,14 +5588,6 @@ object-inspect@^1.8.0: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0" integrity sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA== -object-is@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.3.tgz#2e3b9e65560137455ee3bd62aec4d90a2ea1cc81" - integrity sha512-teyqLvFWzLkq5B9ki8FVWA902UER2qkxmdA4nLf+wjOLAWgxzCWZNCxpDq9MvE8MmhWNr+I8w3BN49Vx36Y6Xg== - dependencies: - define-properties "^1.1.3" - es-abstract "^1.18.0-next.1" - object-keys@^1.0.12, object-keys@^1.0.6, object-keys@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" @@ -6605,19 +6465,6 @@ regex-not@^1.0.0, regex-not@^1.0.2: extend-shallow "^3.0.2" safe-regex "^1.1.0" -regexp.prototype.flags@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz#7aba89b3c13a64509dabcf3ca8d9fbb9bdf5cb75" - integrity sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ== - dependencies: - define-properties "^1.1.3" - es-abstract "^1.17.0-next.1" - -regexparam@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f" - integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g== - regexpp@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f" @@ -6977,14 +6824,6 @@ shellwords@^0.1.1: resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b" integrity sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww== -side-channel@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.3.tgz#cdc46b057550bbab63706210838df5d4c19519c3" - integrity sha512-A6+ByhlLkksFoUepsGxfj5x1gTSrs+OydsRptUxeNCabQpCFUvcwIczgOigI8vhY/OJCnPnyE9rGiwgvr9cS1g== - dependencies: - es-abstract "^1.18.0-next.0" - object-inspect "^1.8.0" - signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -8012,44 +7851,11 @@ whatwg-url@^7.0.0: tr46 "^1.0.1" webidl-conversions "^4.0.2" -which-boxed-primitive@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.1.tgz#cbe8f838ebe91ba2471bb69e9edbda67ab5a5ec1" - integrity sha512-7BT4TwISdDGBgaemWU0N0OU7FeAEJ9Oo2P1PHRm/FCWoEi2VLWC9b6xvxAA3C/NMpxg3HXVgi0sMmGbNUbNepQ== - dependencies: - is-bigint "^1.0.0" - is-boolean-object "^1.0.0" - is-number-object "^1.0.3" - is-string "^1.0.4" - is-symbol "^1.0.2" - -which-collection@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.1.tgz#70eab71ebbbd2aefaf32f917082fc62cdcb70906" - integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A== - dependencies: - is-map "^2.0.1" - is-set "^2.0.1" - is-weakmap "^2.0.1" - is-weakset "^2.0.1" - which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= -which-typed-array@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.2.tgz#e5f98e56bda93e3dac196b01d47c1156679c00b2" - integrity sha512-KT6okrd1tE6JdZAy3o2VhMoYPh3+J6EMZLyrxBQsZflI1QCZIxMrIYLkosd8Twf+YfknVIHmYQPgJt238p8dnQ== - dependencies: - available-typed-arrays "^1.0.2" - es-abstract "^1.17.5" - foreach "^2.0.5" - function-bind "^1.1.1" - has-symbols "^1.0.1" - is-typed-array "^1.1.3" - which@^1.2.9, which@^1.3.0: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" From 103161c7a83eb3cb525d9856f5537c5c73c35e37 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Dec 2020 13:39:34 +0000 Subject: [PATCH 02/48] Expanding on deployment, attempting to hide all the quota stuff behind layers that we can more easily abstract, as well as hiding away any AWS specific features in the index of deployment. --- .../src/api/controllers/deploy/Deployment.js | 66 ++++++++++ .../deploy/{aws.js => awsDeploy.js} | 124 ++++++------------ .../src/api/controllers/deploy/index.js | 89 ++++--------- .../src/api/controllers/deploy/quota.js | 27 ++++ .../src/api/controllers/deploy/selfDeploy.js | 5 + .../src/api/controllers/deploy/utils.js | 31 +++++ .../src/api/controllers/static/index.js | 6 +- packages/server/src/utilities/index.js | 19 +++ 8 files changed, 214 insertions(+), 153 deletions(-) create mode 100644 packages/server/src/api/controllers/deploy/Deployment.js rename packages/server/src/api/controllers/deploy/{aws.js => awsDeploy.js} (61%) create mode 100644 packages/server/src/api/controllers/deploy/quota.js create mode 100644 packages/server/src/api/controllers/deploy/selfDeploy.js create mode 100644 packages/server/src/api/controllers/deploy/utils.js diff --git a/packages/server/src/api/controllers/deploy/Deployment.js b/packages/server/src/api/controllers/deploy/Deployment.js new file mode 100644 index 0000000000..609ce0e8fb --- /dev/null +++ b/packages/server/src/api/controllers/deploy/Deployment.js @@ -0,0 +1,66 @@ +const { getAppQuota } = require("./quota") +const env = require("../../../environment") + +/** + * This is used to pass around information about the deployment that is occurring + */ +class Deployment { + constructor(id, appId) { + this._id = id + this.appId = appId + } + + // purely so that we can do quota stuff outside the main deployment context + async init() { + if (!env.SELF_HOSTED) { + this.setQuota(await getAppQuota(this.appId)) + } + } + + setQuota(quota) { + this.quota = quota + } + + getQuota() { + return this.quota + } + + getAppId() { + return this.appId + } + + setVerification(verification) { + this.verification = verification + } + + getVerification() { + return this.verification + } + + setStatus(status, err = null) { + this.status = status + if (err) { + this.err = err + } + } + + getJSON() { + const obj = { + _id: this._id, + appId: this.appId, + status: this.status, + } + if (this.err) { + obj.err = this.err + } + if (this.verification && this.verification.cfDistribution) { + obj.cfDistribution = this.verification.cfDistribution + } + if (this.verification && this.verification.quota) { + obj.quota = this.verification.quota + } + return obj + } +} + +module.exports = Deployment diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/awsDeploy.js similarity index 61% rename from packages/server/src/api/controllers/deploy/aws.js rename to packages/server/src/api/controllers/deploy/awsDeploy.js index d478c4efde..5a36871ae8 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -1,57 +1,26 @@ const fs = require("fs") const { join } = require("../../../utilities/centralPath") -const AWS = require("aws-sdk") +const AwsDeploy = require("aws-sdk") const fetch = require("node-fetch") -const sanitize = require("sanitize-s3-objectkey") const { budibaseAppsDir } = require("../../../utilities/budibaseDir") const PouchDB = require("../../../db") const env = require("../../../environment") - -/** - * Finalises the deployment, updating the quota for the user API key - * The verification process returns the levels to update to. - * Calls the "deployment-success" lambda. - * @param {object} quota The usage quota levels returned from the verifyDeploy - * @returns {Promise} The usage has been updated against the user API key. - */ -exports.updateDeploymentQuota = async function(quota) { - const DEPLOYMENT_SUCCESS_URL = - env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" - - const response = await fetch(DEPLOYMENT_SUCCESS_URL, { - method: "POST", - body: JSON.stringify({ - apiKey: env.BUDIBASE_API_KEY, - quota, - }), - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - }) - - if (response.status !== 200) { - throw new Error(`Error updating deployment quota for API Key`) - } - - return await response.json() -} +const { prepareUpload } = require("./utils") +const { walkDir } = require("../../../utilities") /** * Verifies the users API key and * Verifies that the deployment fits within the quota of the user * Links to the "check-api-key" lambda. - * @param {String} appId - appId being deployed - * @param {String} appId - appId being deployed - * @param {quota} quota - current quota being changed with this application + * @param {object} deployment - information about the active deployment, including the appId and quota. */ -exports.verifyDeployment = async function({ appId, quota }) { +exports.preDeployment = async function(deployment) { const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, { method: "POST", body: JSON.stringify({ apiKey: env.BUDIBASE_API_KEY, - appId, - quota, + appId: deployment.getAppId(), + quota: deployment.getQuota(), }), }) @@ -68,7 +37,7 @@ exports.verifyDeployment = async function({ appId, quota }) { // set credentials here, means any time we're verified we're ready to go if (json.credentials) { - AWS.config.update({ + AwsDeploy.config.update({ accessKeyId: json.credentials.AccessKeyId, secretAccessKey: json.credentials.SecretAccessKey, sessionToken: json.credentials.SessionToken, @@ -78,57 +47,40 @@ exports.verifyDeployment = async function({ appId, quota }) { return json } -const CONTENT_TYPE_MAP = { - html: "text/html", - css: "text/css", - js: "application/javascript", -} - /** - * Recursively walk a directory tree and execute a callback on all files. - * @param {String} dirPath - Directory to traverse - * @param {Function} callback - callback to execute on files + * Finalises the deployment, updating the quota for the user API key + * The verification process returns the levels to update to. + * Calls the "deployment-success" lambda. + * @param {object} deployment information about the active deployment, including the quota info. + * @returns {Promise} The usage has been updated against the user API key. */ -function walkDir(dirPath, callback) { - for (let filename of fs.readdirSync(dirPath)) { - const filePath = `${dirPath}/${filename}` - const stat = fs.lstatSync(filePath) +exports.postDeployment = async function(deployment) { + const DEPLOYMENT_SUCCESS_URL = + env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" - if (stat.isFile()) { - callback(filePath) - } else { - walkDir(filePath, callback) - } + const response = await fetch(DEPLOYMENT_SUCCESS_URL, { + method: "POST", + body: JSON.stringify({ + apiKey: env.BUDIBASE_API_KEY, + quota: deployment.getQuota(), + }), + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + }) + + if (response.status !== 200) { + throw new Error(`Error updating deployment quota for API Key`) } + + return await response.json() } -async function prepareUploadForS3({ s3Key, metadata, s3, file }) { - const extension = [...file.name.split(".")].pop() - const fileBytes = fs.readFileSync(file.path) - - const upload = await s3 - .upload({ - // windows filepaths need to be converted to forward slashes for s3 - Key: sanitize(s3Key).replace(/\\/g, "/"), - Body: fileBytes, - ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], - Metadata: metadata, - }) - .promise() - - return { - size: file.size, - name: file.name, - extension, - url: upload.Location, - key: upload.Key, - } -} - -exports.prepareUploadForS3 = prepareUploadForS3 - -exports.uploadAppAssets = async function({ appId, bucket, accountId }) { - const s3 = new AWS.S3({ +exports.deploy = async function(deployment) { + const appId = deployment.getAppId() + const { bucket, accountId } = deployment.getVerification() + const s3 = new AwsDeploy.S3({ params: { Bucket: bucket, }, @@ -143,7 +95,7 @@ exports.uploadAppAssets = async function({ appId, bucket, accountId }) { for (let page of appPages) { // Upload HTML, CSS and JS for each page of the web app walkDir(join(appAssetsPath, page), function(filePath) { - const appAssetUpload = prepareUploadForS3({ + const appAssetUpload = prepareUpload({ file: { path: filePath, name: [...filePath.split("/")].pop(), @@ -168,7 +120,7 @@ exports.uploadAppAssets = async function({ appId, bucket, accountId }) { for (let file of fileUploads.uploads) { if (file.uploaded) continue - const attachmentUpload = prepareUploadForS3({ + const attachmentUpload = prepareUpload({ file, s3Key: `assets/${appId}/attachments/${file.processedFileName}`, s3, diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 03f91430db..00cd7254bc 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -1,13 +1,12 @@ const CouchDB = require("pouchdb") const PouchDB = require("../../../db") -const { - uploadAppAssets, - verifyDeployment, - updateDeploymentQuota, -} = require("./aws") -const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils") const newid = require("../../../db/newid") const env = require("../../../environment") +const deployment = env.SELF_HOSTED + ? require("./selfDeploy") + : require("./awsDeploy") +const { deploy, preDeployment, postDeployment } = deployment +const Deployment = require("./Deployment") // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 @@ -44,7 +43,9 @@ function replicate(local, remote) { }) } -async function replicateCouch({ appId, session }) { +async function replicateCouch(deployment) { + const appId = deployment.getAppId() + const { session } = deployment.getVerification() const localDb = new PouchDB(appId) const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, { fetch: function(url, opts) { @@ -56,33 +57,10 @@ async function replicateCouch({ appId, session }) { return replicate(localDb, remoteDb) } -async function getCurrentInstanceQuota(appId) { - const db = new PouchDB(appId) - - const rows = await db.allDocs({ - startkey: DocumentTypes.ROW + SEPARATOR, - endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX, - }) - - const users = await db.allDocs({ - startkey: DocumentTypes.USER + SEPARATOR, - endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX, - }) - - const existingRows = rows.rows.length - const existingUsers = users.rows.length - - const designDoc = await db.get("_design/database") - - return { - rows: existingRows, - users: existingUsers, - views: Object.keys(designDoc.views).length, - } -} - async function storeLocalDeploymentHistory(deployment) { - const db = new PouchDB(deployment.appId) + const appId = deployment.getAppId() + const deploymentJSON = deployment.getJSON() + const db = new PouchDB(appId) let deploymentDoc try { @@ -91,7 +69,7 @@ async function storeLocalDeploymentHistory(deployment) { deploymentDoc = { _id: "_local/deployments", history: {} } } - const deploymentId = deployment._id || newid() + const deploymentId = deploymentJSON._id || newid() // first time deployment if (!deploymentDoc.history[deploymentId]) @@ -99,7 +77,7 @@ async function storeLocalDeploymentHistory(deployment) { deploymentDoc.history[deploymentId] = { ...deploymentDoc.history[deploymentId], - ...deployment, + ...deploymentJSON, updatedAt: Date.now(), } @@ -111,43 +89,26 @@ async function storeLocalDeploymentHistory(deployment) { } async function deployApp({ appId, deploymentId }) { + const deployment = new Deployment(deploymentId, appId) try { - const instanceQuota = await getCurrentInstanceQuota(appId) - const verification = await verifyDeployment({ - appId, - quota: instanceQuota, - }) + await deployment.init() + deployment.setVerification(await preDeployment(deployment)) - console.log(`Uploading assets for appID ${appId} assets to s3..`) + console.log(`Uploading assets for appID ${appId}..`) - await uploadAppAssets({ - appId, - ...verification, - }) + await deploy(deployment) // replicate the DB to the couchDB cluster in prod console.log("Replicating local PouchDB to remote..") - await replicateCouch({ - appId, - session: verification.couchDbSession, - }) + await replicateCouch(deployment) - await updateDeploymentQuota(verification.quota) + await postDeployment(deployment) - await storeLocalDeploymentHistory({ - _id: deploymentId, - appId, - cfDistribution: verification.cfDistribution, - quota: verification.quota, - status: DeploymentStatus.SUCCESS, - }) + deployment.setStatus(DeploymentStatus.SUCCESS) + await storeLocalDeploymentHistory(deployment) } catch (err) { - await storeLocalDeploymentHistory({ - _id: deploymentId, - appId, - status: DeploymentStatus.FAILURE, - err: err.message, - }) + deployment.setStatus(DeploymentStatus.FAILURE, err.message) + await storeLocalDeploymentHistory(deployment) throw new Error(`Deployment Failed: ${err.message}`) } } @@ -188,7 +149,7 @@ exports.deployApp = async function(ctx) { status: DeploymentStatus.PENDING, }) - deployApp({ + await deployApp({ ...ctx.user, deploymentId: deployment._id, }) diff --git a/packages/server/src/api/controllers/deploy/quota.js b/packages/server/src/api/controllers/deploy/quota.js new file mode 100644 index 0000000000..4e4c869670 --- /dev/null +++ b/packages/server/src/api/controllers/deploy/quota.js @@ -0,0 +1,27 @@ +const PouchDB = require("../../../db") +const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils") + +exports.getAppQuota = async function(appId) { + const db = new PouchDB(appId) + + const rows = await db.allDocs({ + startkey: DocumentTypes.ROW + SEPARATOR, + endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX, + }) + + const users = await db.allDocs({ + startkey: DocumentTypes.USER + SEPARATOR, + endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX, + }) + + const existingRows = rows.rows.length + const existingUsers = users.rows.length + + const designDoc = await db.get("_design/database") + + return { + rows: existingRows, + users: existingUsers, + views: Object.keys(designDoc.views).length, + } +} diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js new file mode 100644 index 0000000000..ad81b6943a --- /dev/null +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -0,0 +1,5 @@ +exports.preDeployment = async function(deployment) {} + +exports.postDeployment = async function(deployment) {} + +exports.deploy = async function(deployment) {} diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js new file mode 100644 index 0000000000..75fe3133ba --- /dev/null +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -0,0 +1,31 @@ +const fs = require("fs") +const sanitize = require("sanitize-s3-objectkey") + +const CONTENT_TYPE_MAP = { + html: "text/html", + css: "text/css", + js: "application/javascript", +} + +exports.prepareUpload = async function({ s3Key, metadata, s3, file }) { + const extension = [...file.name.split(".")].pop() + const fileBytes = fs.readFileSync(file.path) + + const upload = await s3 + .upload({ + // windows file paths need to be converted to forward slashes for s3 + Key: sanitize(s3Key).replace(/\\/g, "/"), + Body: fileBytes, + ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], + Metadata: metadata, + }) + .promise() + + return { + size: file.size, + name: file.name, + extension, + url: upload.Location, + key: upload.Key, + } +} diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index b048bbd9fe..a510291b41 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -6,7 +6,7 @@ const fetch = require("node-fetch") const fs = require("fs-extra") const uuid = require("uuid") const AWS = require("aws-sdk") -const { prepareUploadForS3 } = require("../deploy/aws") +const { prepareUpload } = require("../deploy/utils") const handlebars = require("handlebars") const { budibaseAppsDir, @@ -54,7 +54,7 @@ exports.uploadFile = async function(ctx) { const fileExtension = [...file.name.split(".")].pop() const processedFileName = `${uuid.v4()}.${fileExtension}` - return prepareUploadForS3({ + return prepareUpload({ file, s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`, s3, @@ -235,5 +235,5 @@ exports.serveComponentLibrary = async function(ctx) { return } - await send(ctx, "/index.js", { root: componentLibraryPath }) + await send(ctx, "/awsDeploy.js", { root: componentLibraryPath }) } diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index cde10b6b62..9fc3863118 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -1,5 +1,6 @@ const env = require("../environment") const { DocumentTypes, SEPARATOR } = require("../db/utils") +const fs = require("fs") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -74,3 +75,21 @@ exports.setCookie = (ctx, name, value) => { exports.isClient = ctx => { return ctx.headers["x-budibase-type"] === "client" } + +/** + * Recursively walk a directory tree and execute a callback on all files. + * @param {String} dirPath - Directory to traverse + * @param {Function} callback - callback to execute on files + */ +exports.walkDir = (dirPath, callback) => { + for (let filename of fs.readdirSync(dirPath)) { + const filePath = `${dirPath}/${filename}` + const stat = fs.lstatSync(filePath) + + if (stat.isFile()) { + callback(filePath) + } else { + exports.walkDir(filePath, callback) + } + } +} From 3dde3e05811a5d809a15deb3f974d99cff296938 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Dec 2020 16:51:17 +0000 Subject: [PATCH 03/48] Making sure deployment object is used everywhere to hide all underlying deployment properties. --- .../src/api/controllers/deploy/Deployment.js | 24 ++++++++++++++--- .../src/api/controllers/deploy/index.js | 26 +++++++------------ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/Deployment.js b/packages/server/src/api/controllers/deploy/Deployment.js index 609ce0e8fb..d65d2714dc 100644 --- a/packages/server/src/api/controllers/deploy/Deployment.js +++ b/packages/server/src/api/controllers/deploy/Deployment.js @@ -1,13 +1,14 @@ const { getAppQuota } = require("./quota") const env = require("../../../environment") +const newid = require("../../../db/newid") /** * This is used to pass around information about the deployment that is occurring */ class Deployment { - constructor(id, appId) { - this._id = id + constructor(appId, id = null) { this.appId = appId + this._id = id || newid() } // purely so that we can do quota stuff outside the main deployment context @@ -30,6 +31,9 @@ class Deployment { } setVerification(verification) { + if (this.verification.quota) { + this.quota = this.verification.quota + } this.verification = verification } @@ -44,6 +48,18 @@ class Deployment { } } + fromJSON(json) { + if (json.verification) { + this.setVerification(json.verification) + } + if (json.quota) { + this.setQuota(json.quota) + } + if (json.status) { + this.setStatus(json.status, json.err) + } + } + getJSON() { const obj = { _id: this._id, @@ -56,8 +72,8 @@ class Deployment { if (this.verification && this.verification.cfDistribution) { obj.cfDistribution = this.verification.cfDistribution } - if (this.verification && this.verification.quota) { - obj.quota = this.verification.quota + if (this.quota) { + obj.quota = this.quota } return obj } diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 00cd7254bc..a37fad62a0 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -1,6 +1,6 @@ const CouchDB = require("pouchdb") const PouchDB = require("../../../db") -const newid = require("../../../db/newid") + const env = require("../../../environment") const deployment = env.SELF_HOSTED ? require("./selfDeploy") @@ -69,7 +69,7 @@ async function storeLocalDeploymentHistory(deployment) { deploymentDoc = { _id: "_local/deployments", history: {} } } - const deploymentId = deploymentJSON._id || newid() + const deploymentId = deploymentJSON._id // first time deployment if (!deploymentDoc.history[deploymentId]) @@ -82,14 +82,12 @@ async function storeLocalDeploymentHistory(deployment) { } await db.put(deploymentDoc) - return { - _id: deploymentId, - ...deploymentDoc.history[deploymentId], - } + deployment.fromJSON(deploymentDoc.history[deploymentId]) + return deployment } -async function deployApp({ appId, deploymentId }) { - const deployment = new Deployment(deploymentId, appId) +async function deployApp(deployment) { + const appId = deployment.getAppId() try { await deployment.init() deployment.setVerification(await preDeployment(deployment)) @@ -144,15 +142,11 @@ exports.deploymentProgress = async function(ctx) { } exports.deployApp = async function(ctx) { - const deployment = await storeLocalDeploymentHistory({ - appId: ctx.user.appId, - status: DeploymentStatus.PENDING, - }) + let deployment = new Deployment(ctx.user.appId) + deployment.setStatus(DeploymentStatus.PENDING) + deployment = await storeLocalDeploymentHistory(deployment) - await deployApp({ - ...ctx.user, - deploymentId: deployment._id, - }) + await deployApp(deployment) ctx.body = deployment } From a5d2e9a3b9719782f179fd0c27d281c1ee2fc4ef Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Dec 2020 17:45:20 +0000 Subject: [PATCH 04/48] Some work to not contact lambda or use quotas when deploying locally. --- .../src/api/controllers/deploy/awsDeploy.js | 71 ++----------------- .../src/api/controllers/deploy/selfDeploy.js | 32 ++++++++- .../src/api/controllers/deploy/utils.js | 66 ++++++++++++++++- packages/server/src/environment.js | 2 + 4 files changed, 101 insertions(+), 70 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/awsDeploy.js b/packages/server/src/api/controllers/deploy/awsDeploy.js index 5a36871ae8..433913644a 100644 --- a/packages/server/src/api/controllers/deploy/awsDeploy.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -1,12 +1,7 @@ -const fs = require("fs") -const { join } = require("../../../utilities/centralPath") -const AwsDeploy = require("aws-sdk") +const AWS = require("aws-sdk") const fetch = require("node-fetch") -const { budibaseAppsDir } = require("../../../utilities/budibaseDir") -const PouchDB = require("../../../db") const env = require("../../../environment") -const { prepareUpload } = require("./utils") -const { walkDir } = require("../../../utilities") +const { deployToObjectStore } = require("./utils") /** * Verifies the users API key and @@ -37,7 +32,7 @@ exports.preDeployment = async function(deployment) { // set credentials here, means any time we're verified we're ready to go if (json.credentials) { - AwsDeploy.config.update({ + AWS.config.update({ accessKeyId: json.credentials.AccessKeyId, secretAccessKey: json.credentials.SecretAccessKey, sessionToken: json.credentials.SessionToken, @@ -80,65 +75,11 @@ exports.postDeployment = async function(deployment) { exports.deploy = async function(deployment) { const appId = deployment.getAppId() const { bucket, accountId } = deployment.getVerification() - const s3 = new AwsDeploy.S3({ + const metadata = { accountId } + const s3Client = new AWS.S3({ params: { Bucket: bucket, }, }) - - const appAssetsPath = join(budibaseAppsDir(), appId, "public") - - const appPages = fs.readdirSync(appAssetsPath) - - let uploads = [] - - for (let page of appPages) { - // Upload HTML, CSS and JS for each page of the web app - walkDir(join(appAssetsPath, page), function(filePath) { - const appAssetUpload = prepareUpload({ - file: { - path: filePath, - name: [...filePath.split("/")].pop(), - }, - s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), - s3, - metadata: { accountId }, - }) - uploads.push(appAssetUpload) - }) - } - - // Upload file attachments - const db = new PouchDB(appId) - let fileUploads - try { - fileUploads = await db.get("_local/fileuploads") - } catch (err) { - fileUploads = { _id: "_local/fileuploads", uploads: [] } - } - - for (let file of fileUploads.uploads) { - if (file.uploaded) continue - - const attachmentUpload = prepareUpload({ - file, - s3Key: `assets/${appId}/attachments/${file.processedFileName}`, - s3, - metadata: { accountId }, - }) - - uploads.push(attachmentUpload) - - // mark file as uploaded - file.uploaded = true - } - - db.put(fileUploads) - - try { - return await Promise.all(uploads) - } catch (err) { - console.error("Error uploading budibase app assets to s3", err) - throw err - } + await deployToObjectStore(appId, s3Client, metadata) } diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index ad81b6943a..a18736a5f4 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -1,5 +1,31 @@ -exports.preDeployment = async function(deployment) {} +const env = require("../../../environment") +const AWS = require("aws-sdk") +const { deployToObjectStore } = require("./utils") -exports.postDeployment = async function(deployment) {} +const APP_BUCKET = "app-assets" -exports.deploy = async function(deployment) {} +exports.preDeployment = async function() { + AWS.config.update({ + accessKeyId: env.MINIO_ACCESS_KEY, + secretAccessKey: env.MINIO_SECRET_KEY, + }) +} + +exports.postDeployment = async function() { + // we don't actively need to do anything after deployment in self hosting +} + +exports.deploy = async function(deployment) { + const appId = deployment.getAppId() + var objClient = new AWS.S3({ + endpoint: "http://localhost:9000", + s3ForcePathStyle: true, // needed with minio? + signatureVersion: "v4", + params: { + Bucket: APP_BUCKET, + }, + }) + // no metadata, aws has account ID in metadata + const metadata = {} + await deployToObjectStore(appId, objClient, metadata) +} diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index 75fe3133ba..f0a4b44e1d 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -1,5 +1,9 @@ const fs = require("fs") const sanitize = require("sanitize-s3-objectkey") +const { walkDir } = require("../../../utilities") +const { join } = require("../../../utilities/centralPath") +const { budibaseAppsDir } = require("../../../utilities/budibaseDir") +const PouchDB = require("../../../db") const CONTENT_TYPE_MAP = { html: "text/html", @@ -7,11 +11,11 @@ const CONTENT_TYPE_MAP = { js: "application/javascript", } -exports.prepareUpload = async function({ s3Key, metadata, s3, file }) { +exports.prepareUpload = async function({ s3Key, metadata, client, file }) { const extension = [...file.name.split(".")].pop() const fileBytes = fs.readFileSync(file.path) - const upload = await s3 + const upload = await client .upload({ // windows file paths need to be converted to forward slashes for s3 Key: sanitize(s3Key).replace(/\\/g, "/"), @@ -29,3 +33,61 @@ exports.prepareUpload = async function({ s3Key, metadata, s3, file }) { key: upload.Key, } } + +exports.deployToObjectStore = async function(appId, objectClient, metadata) { + const appAssetsPath = join(budibaseAppsDir(), appId, "public") + + const appPages = fs.readdirSync(appAssetsPath) + + let uploads = [] + + for (let page of appPages) { + // Upload HTML, CSS and JS for each page of the web app + walkDir(join(appAssetsPath, page), function(filePath) { + const appAssetUpload = exports.prepareUpload({ + file: { + path: filePath, + name: [...filePath.split("/")].pop(), + }, + s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), + client: objectClient, + metadata, + }) + uploads.push(appAssetUpload) + }) + } + + // Upload file attachments + const db = new PouchDB(appId) + let fileUploads + try { + fileUploads = await db.get("_local/fileuploads") + } catch (err) { + fileUploads = { _id: "_local/fileuploads", uploads: [] } + } + + for (let file of fileUploads.uploads) { + if (file.uploaded) continue + + const attachmentUpload = exports.prepareUpload({ + file, + s3Key: `assets/${appId}/attachments/${file.processedFileName}`, + client: objectClient, + metadata, + }) + + uploads.push(attachmentUpload) + + // mark file as uploaded + file.uploaded = true + } + + db.put(fileUploads) + + try { + return await Promise.all(uploads) + } catch (err) { + console.error("Error uploading budibase app assets to s3", err) + throw err + } +} diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 5fbf57448a..578645ec66 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -36,6 +36,8 @@ module.exports = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, + MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, + MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, _set(key, value) { process.env[key] = value module.exports[key] = value From ee37bca225c421777d24ea704498233e57bb9a50 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 4 Dec 2020 12:07:51 +0000 Subject: [PATCH 05/48] Quick fixes for self deployment. --- packages/server/src/api/controllers/deploy/Deployment.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/api/controllers/deploy/Deployment.js b/packages/server/src/api/controllers/deploy/Deployment.js index d65d2714dc..3b95652c44 100644 --- a/packages/server/src/api/controllers/deploy/Deployment.js +++ b/packages/server/src/api/controllers/deploy/Deployment.js @@ -19,6 +19,9 @@ class Deployment { } setQuota(quota) { + if (!quota) { + return + } this.quota = quota } @@ -31,6 +34,9 @@ class Deployment { } setVerification(verification) { + if (!verification) { + return + } if (this.verification.quota) { this.quota = this.verification.quota } From bf82348f9fd2f123fc7e2622600ea0c85267eeb2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 4 Dec 2020 12:09:02 +0000 Subject: [PATCH 06/48] Hopefully a fix for the 4001 bug we have been experiencing. --- packages/server/src/app.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 10eec1e66f..58526cdf4c 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -1,4 +1,5 @@ const Koa = require("koa") +const destroyable = require("server-destroy") const electron = require("electron") const koaBody = require("koa-body") const logger = require("koa-pino-logger") @@ -44,6 +45,7 @@ if (electron.app && electron.app.isPackaged) { } const server = http.createServer(app.callback()) +destroyable(server) server.on("close", () => console.log("Server Closed")) @@ -51,3 +53,14 @@ module.exports = server.listen(env.PORT || 4001, () => { console.log(`Budibase running on ${JSON.stringify(server.address())}`) automations.init() }) + +process.on("uncaughtException", err => { + console.error(err) + server.close() + server.destroy() +}) + +process.on("SIGTERM", () => { + server.close() + server.destroy() +}) From ffad7bb3d064990eefd302330ad2edea10004982 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 4 Dec 2020 17:10:01 +0000 Subject: [PATCH 07/48] Some work towards replication of couch locally. --- .../src/api/controllers/deploy/awsDeploy.js | 18 ++++++++++- .../src/api/controllers/deploy/index.js | 32 +++---------------- .../src/api/controllers/deploy/selfDeploy.js | 22 ++++++++++++- .../src/api/controllers/deploy/utils.js | 9 ++++++ packages/server/src/environment.js | 1 + 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/awsDeploy.js b/packages/server/src/api/controllers/deploy/awsDeploy.js index 433913644a..26b3ea2b21 100644 --- a/packages/server/src/api/controllers/deploy/awsDeploy.js +++ b/packages/server/src/api/controllers/deploy/awsDeploy.js @@ -1,7 +1,9 @@ const AWS = require("aws-sdk") const fetch = require("node-fetch") const env = require("../../../environment") -const { deployToObjectStore } = require("./utils") +const { deployToObjectStore, performReplication } = require("./utils") +const CouchDB = require("pouchdb") +const PouchDB = require("../../../db") /** * Verifies the users API key and @@ -83,3 +85,17 @@ exports.deploy = async function(deployment) { }) await deployToObjectStore(appId, s3Client, metadata) } + +exports.replicateDb = async function(deployment) { + const appId = deployment.getAppId() + const { session } = deployment.getVerification() + const localDb = new PouchDB(appId) + const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, { + fetch: function(url, opts) { + opts.headers.set("Cookie", `${session};`) + return CouchDB.fetch(url, opts) + }, + }) + + return performReplication(localDb, remoteDb) +} diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index a37fad62a0..b0cc8f1133 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -1,11 +1,10 @@ -const CouchDB = require("pouchdb") const PouchDB = require("../../../db") const env = require("../../../environment") const deployment = env.SELF_HOSTED ? require("./selfDeploy") : require("./awsDeploy") -const { deploy, preDeployment, postDeployment } = deployment +const { deploy, preDeployment, postDeployment, replicateDb } = deployment const Deployment = require("./Deployment") // the max time we can wait for an invalidation to complete before considering it failed @@ -34,29 +33,6 @@ async function checkAllDeployments(deployments) { return { updated, deployments } } -function replicate(local, remote) { - return new Promise((resolve, reject) => { - const replication = local.sync(remote) - - replication.on("complete", () => resolve()) - replication.on("error", err => reject(err)) - }) -} - -async function replicateCouch(deployment) { - const appId = deployment.getAppId() - const { session } = deployment.getVerification() - const localDb = new PouchDB(appId) - const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, { - fetch: function(url, opts) { - opts.headers.set("Cookie", `${session};`) - return CouchDB.fetch(url, opts) - }, - }) - - return replicate(localDb, remoteDb) -} - async function storeLocalDeploymentHistory(deployment) { const appId = deployment.getAppId() const deploymentJSON = deployment.getJSON() @@ -96,9 +72,9 @@ async function deployApp(deployment) { await deploy(deployment) - // replicate the DB to the couchDB cluster in prod - console.log("Replicating local PouchDB to remote..") - await replicateCouch(deployment) + // replicate the DB to the main couchDB cluster + console.log("Replicating local PouchDB to CouchDB..") + await replicateDb(deployment) await postDeployment(deployment) diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index a18736a5f4..0b26af4e36 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -1,6 +1,8 @@ const env = require("../../../environment") const AWS = require("aws-sdk") -const { deployToObjectStore } = require("./utils") +const { deployToObjectStore, performReplication } = require("./utils") +const CouchDB = require("pouchdb") +const PouchDB = require("../../../db") const APP_BUCKET = "app-assets" @@ -25,7 +27,25 @@ exports.deploy = async function(deployment) { Bucket: APP_BUCKET, }, }) + // checking the bucket exists + try { + await objClient.headBucket({ Bucket: APP_BUCKET }).promise() + } catch (err) { + // bucket doesn't exist create it + if (err.statusCode === 404) { + await objClient.createBucket({ Bucket: APP_BUCKET }).promise() + } else { + throw err + } + } // no metadata, aws has account ID in metadata const metadata = {} await deployToObjectStore(appId, objClient, metadata) } + +exports.replicateDb = async function(deployment) { + const appId = deployment.getAppId() + const localDb = new PouchDB(appId) + const remoteDb = new CouchDB(`${env.COUCH_DB_URL}/${appId}`) + return performReplication(localDb, remoteDb) +} diff --git a/packages/server/src/api/controllers/deploy/utils.js b/packages/server/src/api/controllers/deploy/utils.js index f0a4b44e1d..4ee1f3a3f2 100644 --- a/packages/server/src/api/controllers/deploy/utils.js +++ b/packages/server/src/api/controllers/deploy/utils.js @@ -91,3 +91,12 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) { throw err } } + +exports.performReplication = (local, remote) => { + return new Promise((resolve, reject) => { + const replication = local.sync(remote) + + replication.on("complete", () => resolve()) + replication.on("error", err => reject(err)) + }) +} diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 578645ec66..0c67f8c747 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -36,6 +36,7 @@ module.exports = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, + // self hosting features MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, _set(key, value) { From 4ce920df01ff1982d29c3a8beba38f66a6baa4e8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Dec 2020 10:52:18 +0000 Subject: [PATCH 08/48] Updating row controller to make sure that all user requests (bar deletion) are passed through correctly to the user controller so that any logic such as removing user password can be correctly held in the user controller logic. --- packages/server/src/api/controllers/row.js | 102 +++++------------- packages/server/src/api/controllers/user.js | 11 +- .../server/src/api/routes/tests/row.spec.js | 2 - packages/server/src/utilities/index.js | 75 +++++++++++++ 4 files changed, 109 insertions(+), 81 deletions(-) diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 61dd666f3e..41d2ae15b1 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -9,7 +9,7 @@ const { ViewNames, } = require("../../db/utils") const usersController = require("./user") -const { cloneDeep } = require("lodash") +const { coerceRowValues } = require("../../utilities") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -29,11 +29,24 @@ validateJs.extend(validateJs.validators.datetime, { }, }) -// lots of row functionality too specific to pass to user controller, simply handle the -// password deletion here -function removePassword(tableId, row) { +async function findRow(db, appId, tableId, rowId) { + let row if (tableId === ViewNames.USERS) { - delete row.password + let ctx = { + params: { + userId: rowId, + }, + user: { + appId, + }, + } + await usersController.find(ctx) + row = ctx.body + } else { + row = await db.get(rowId) + } + if (row.tableId !== tableId) { + throw "Supplied tableId does not match the rows tableId" } return row } @@ -224,13 +237,12 @@ exports.fetchTableRows = async function(ctx) { exports.find = async function(ctx) { const appId = ctx.user.appId const db = new CouchDB(appId) - let row = await db.get(ctx.params.rowId) - if (row.tableId !== ctx.params.tableId) { - ctx.throw(400, "Supplied tableId does not match the rows tableId") - return + try { + const row = await findRow(db, appId, ctx.params.tableId, ctx.params.rowId) + ctx.body = await linkRows.attachLinkInfo(appId, row) + } catch (err) { + ctx.throw(400, err) } - row = removePassword(ctx.params.tableId, row) - ctx.body = await linkRows.attachLinkInfo(appId, row) } exports.destroy = async function(ctx) { @@ -296,8 +308,10 @@ exports.fetchEnrichedRow = async function(ctx) { return } // need table to work out where links go in row - let [table, row] = await Promise.all([db.get(tableId), db.get(rowId)]) - row = removePassword(tableId, row) + let [table, row] = await Promise.all([ + db.get(tableId), + findRow(db, appId, tableId, rowId), + ]) // get the link docs const linkVals = await linkRows.getLinkDocuments({ appId, @@ -327,68 +341,6 @@ exports.fetchEnrichedRow = async function(ctx) { ctx.status = 200 } -function coerceRowValues(record, table) { - const row = cloneDeep(record) - for (let [key, value] of Object.entries(row)) { - const field = table.schema[key] - if (!field) continue - - // eslint-disable-next-line no-prototype-builtins - if (TYPE_TRANSFORM_MAP[field.type].hasOwnProperty(value)) { - row[key] = TYPE_TRANSFORM_MAP[field.type][value] - } else if (TYPE_TRANSFORM_MAP[field.type].parse) { - row[key] = TYPE_TRANSFORM_MAP[field.type].parse(value) - } - } - return row -} - -const TYPE_TRANSFORM_MAP = { - link: { - "": [], - [null]: [], - [undefined]: undefined, - }, - options: { - "": "", - [null]: "", - [undefined]: undefined, - }, - string: { - "": "", - [null]: "", - [undefined]: undefined, - }, - longform: { - "": "", - [null]: "", - [undefined]: undefined, - }, - number: { - "": null, - [null]: null, - [undefined]: undefined, - parse: n => parseFloat(n), - }, - datetime: { - "": null, - [undefined]: undefined, - [null]: null, - }, - attachment: { - "": [], - [null]: [], - [undefined]: undefined, - }, - boolean: { - "": null, - [null]: null, - [undefined]: undefined, - true: true, - false: false, - }, -} - async function bulkDelete(ctx) { const appId = ctx.user.appId const { rows } = ctx.request.body diff --git a/packages/server/src/api/controllers/user.js b/packages/server/src/api/controllers/user.js index 7ca2e05015..9ef7c2281f 100644 --- a/packages/server/src/api/controllers/user.js +++ b/packages/server/src/api/controllers/user.js @@ -89,9 +89,12 @@ exports.destroy = async function(ctx) { exports.find = async function(ctx) { const database = new CouchDB(ctx.user.appId) - const user = await database.get(generateUserID(ctx.params.email)) - ctx.body = { - email: user.email, - _rev: user._rev, + let lookup = ctx.params.email + ? generateUserID(ctx.params.email) + : ctx.params.userId + const user = await database.get(lookup) + if (user) { + delete user.password } + ctx.body = user } diff --git a/packages/server/src/api/routes/tests/row.spec.js b/packages/server/src/api/routes/tests/row.spec.js index 5972e90ca9..0698250050 100644 --- a/packages/server/src/api/routes/tests/row.spec.js +++ b/packages/server/src/api/routes/tests/row.spec.js @@ -51,8 +51,6 @@ describe("/rows", () => { describe("save, load, update, delete", () => { - - it("returns a success message when the row is created", async () => { const res = await createRow() expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`) diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 9fc3863118..8a43e9d27a 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -1,9 +1,59 @@ const env = require("../environment") const { DocumentTypes, SEPARATOR } = require("../db/utils") const fs = require("fs") +const { cloneDeep } = require("lodash/fp") const APP_PREFIX = DocumentTypes.APP + SEPARATOR +/** + * A map of how we convert various properties in rows to each other based on the row type. + */ +const TYPE_TRANSFORM_MAP = { + link: { + "": [], + [null]: [], + [undefined]: undefined, + }, + options: { + "": "", + [null]: "", + [undefined]: undefined, + }, + string: { + "": "", + [null]: "", + [undefined]: undefined, + }, + longform: { + "": "", + [null]: "", + [undefined]: undefined, + }, + number: { + "": null, + [null]: null, + [undefined]: undefined, + parse: n => parseFloat(n), + }, + datetime: { + "": null, + [undefined]: undefined, + [null]: null, + }, + attachment: { + "": [], + [null]: [], + [undefined]: undefined, + }, + boolean: { + "": null, + [null]: null, + [undefined]: undefined, + true: true, + false: false, + }, +} + function confirmAppId(possibleAppId) { return possibleAppId && possibleAppId.startsWith(APP_PREFIX) ? possibleAppId @@ -93,3 +143,28 @@ exports.walkDir = (dirPath, callback) => { } } } + +/** + * This will coerce the values in a row to the correct types based on the type transform map and the + * table schema. + * @param {object} row The row which is to be coerced to correct values based on schema, this input + * row will not be updated. + * @param {object} table The table that has been retrieved from DB, this must contain the expected + * schema for the rows. + * @returns {object} The updated row will be returned with all values coerced. + */ +exports.coerceRowValues = (row, table) => { + const clonedRow = cloneDeep(row) + for (let [key, value] of Object.entries(clonedRow)) { + const field = table.schema[key] + if (!field) continue + + // eslint-disable-next-line no-prototype-builtins + if (TYPE_TRANSFORM_MAP[field.type].hasOwnProperty(value)) { + clonedRow[key] = TYPE_TRANSFORM_MAP[field.type][value] + } else if (TYPE_TRANSFORM_MAP[field.type].parse) { + clonedRow[key] = TYPE_TRANSFORM_MAP[field.type].parse(value) + } + } + return clonedRow +} From 1cbe3771abe1a8cc4df11f8308342945a0a61c6e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Dec 2020 17:04:05 +0000 Subject: [PATCH 09/48] Adding in self hosted API key. --- packages/server/src/app.js | 6 ++- packages/server/src/middleware/authorized.js | 6 +-- packages/server/src/middleware/usageQuota.js | 7 ++-- packages/server/src/selfhost/README.md | 7 ++++ packages/server/src/selfhost/index.js | 39 +++++++++++++++++++ .../server/src/utilities/security/apikey.js | 22 +++++++++++ 6 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 packages/server/src/selfhost/README.md create mode 100644 packages/server/src/selfhost/index.js create mode 100644 packages/server/src/utilities/security/apikey.js diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 58526cdf4c..a5ef3132af 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -9,6 +9,7 @@ const env = require("./environment") const eventEmitter = require("./events") const automations = require("./automations/index") const Sentry = require("@sentry/node") +const selfhost = require("./selfhost") const app = new Koa() @@ -49,9 +50,12 @@ destroyable(server) server.on("close", () => console.log("Server Closed")) -module.exports = server.listen(env.PORT || 4001, () => { +module.exports = server.listen(env.PORT || 4001, async () => { console.log(`Budibase running on ${JSON.stringify(server.address())}`) automations.init() + if (env.SELF_HOSTED) { + await selfhost.init() + } }) process.on("uncaughtException", err => { diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index e959e05f9d..fff66a68d6 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -7,7 +7,7 @@ const { doesHavePermission, } = require("../utilities/security/permissions") const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") +const { getAPIKey } = require("../utilities/security/apikey") const { AuthTypes } = require("../constants") const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER] @@ -21,9 +21,7 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => { } if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) { // api key header passed by external webhook - const apiKeyInfo = await apiKeyTable.get({ - primary: ctx.headers["x-api-key"], - }) + const apiKeyInfo = await getAPIKey(ctx.headers["x-api-key"]) if (apiKeyInfo) { ctx.auth = { diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 39e387bd5a..e980afe678 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -43,6 +43,10 @@ module.exports = async (ctx, next) => { return } } + // if running in builder or a self hosted cloud usage quotas should not be executed + if (!env.CLOUD || env.SELF_HOSTED) { + return next() + } // update usage for uploads to be the total size if (property === usageQuota.Properties.UPLOAD) { const files = @@ -51,9 +55,6 @@ module.exports = async (ctx, next) => { : [ctx.request.files.file] usage = files.map(file => file.size).reduce((total, size) => total + size) } - if (!env.CLOUD) { - return next() - } try { await usageQuota.update(ctx.auth.apiKey, property, usage) return next() diff --git a/packages/server/src/selfhost/README.md b/packages/server/src/selfhost/README.md new file mode 100644 index 0000000000..a02743a58c --- /dev/null +++ b/packages/server/src/selfhost/README.md @@ -0,0 +1,7 @@ +### Self hosting +This directory contains utilities that are needed for self hosted platforms to operate. +These will mostly be utilities, necessary to the operation of the server e.g. storing self +hosting specific options and attributes to CouchDB. + +All the internal operations should be exposed through the `index.js` so importing +the self host directory should give you everything you need. \ No newline at end of file diff --git a/packages/server/src/selfhost/index.js b/packages/server/src/selfhost/index.js new file mode 100644 index 0000000000..05c9bdc6b2 --- /dev/null +++ b/packages/server/src/selfhost/index.js @@ -0,0 +1,39 @@ +const CouchDB = require("../db") +const env = require("../environment") +const newid = require("../db/newid") + +const SELF_HOST_DB = "self-host-db" +const SELF_HOST_DOC = "self-host-info" + +async function createSelfHostDB(db) { + await db.put({ + _id: "_design/database", + views: {}, + }) + const selfHostInfo = { + _id: SELF_HOST_DOC, + apiKeyId: newid(), + } + await db.put(selfHostInfo) + return selfHostInfo +} + +exports.init = async () => { + if (!env.SELF_HOSTED) { + return + } + const db = new CouchDB(SELF_HOST_DB) + try { + await db.get(SELF_HOST_DOC) + } catch (err) { + // failed to retrieve + if (err.status === 404) { + await createSelfHostDB(db) + } + } +} + +exports.getSelfHostInfo = async () => { + const db = new CouchDB(SELF_HOST_DB) + return db.get(SELF_HOST_DOC) +} diff --git a/packages/server/src/utilities/security/apikey.js b/packages/server/src/utilities/security/apikey.js new file mode 100644 index 0000000000..b2fd230130 --- /dev/null +++ b/packages/server/src/utilities/security/apikey.js @@ -0,0 +1,22 @@ +const { apiKeyTable } = require("../../db/dynamoClient") +const env = require("../../environment") +const { getSelfHostInfo } = require("../../selfhost") + +/** + * This file purely exists so that we can centralise all logic pertaining to API keys, as their usage differs + * in our Cloud environment versus self hosted. + */ + +exports.getAPIKey = async apiKeyId => { + if (env.CLOUD && !env.SELF_HOSTED) { + return apiKeyTable.get({ + primary: apiKeyId, + }) + } + if (env.SELF_HOSTED) { + const selfHostInfo = await getSelfHostInfo() + // if the api key supplied is correct then return structure similar + return apiKeyId === selfHostInfo.apiKeyId ? { pk: apiKeyId } : null + } + return null +} From 7b21acd8bd647d7760a3fc95553411212feac004 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Dec 2020 17:10:53 +0000 Subject: [PATCH 10/48] Updating API key controller in self-host mode to return self host API key. --- packages/server/src/api/controllers/apikeys.js | 13 ++++++++++--- packages/server/src/middleware/authorized.js | 6 ++---- packages/server/src/selfhost/index.js | 5 +++++ packages/server/src/utilities/security/apikey.js | 13 +++++++------ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/apikeys.js b/packages/server/src/api/controllers/apikeys.js index 96754f17cc..1cd69c54df 100644 --- a/packages/server/src/api/controllers/apikeys.js +++ b/packages/server/src/api/controllers/apikeys.js @@ -3,13 +3,20 @@ const { join } = require("../../utilities/centralPath") const readline = require("readline") const { budibaseAppsDir } = require("../../utilities/budibaseDir") const env = require("../../environment") +const selfhost = require("../../selfhost") const ENV_FILE_PATH = "/.env" exports.fetch = async function(ctx) { ctx.status = 200 - ctx.body = { - budibase: env.BUDIBASE_API_KEY, - userId: env.USERID_API_KEY, + if (env.SELF_HOSTED) { + ctx.body = { + selfhost: await selfhost.getSelfHostAPIKey(), + } + } else { + ctx.body = { + budibase: env.BUDIBASE_API_KEY, + userId: env.USERID_API_KEY, + } } } diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index fff66a68d6..ad2c4344fa 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -7,7 +7,7 @@ const { doesHavePermission, } = require("../utilities/security/permissions") const env = require("../environment") -const { getAPIKey } = require("../utilities/security/apikey") +const { isAPIKeyValid } = require("../utilities/security/apikey") const { AuthTypes } = require("../constants") const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER] @@ -21,9 +21,7 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => { } if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) { // api key header passed by external webhook - const apiKeyInfo = await getAPIKey(ctx.headers["x-api-key"]) - - if (apiKeyInfo) { + if (await isAPIKeyValid(ctx.headers["x-api-key"])) { ctx.auth = { authenticated: AuthTypes.EXTERNAL, apiKey: ctx.headers["x-api-key"], diff --git a/packages/server/src/selfhost/index.js b/packages/server/src/selfhost/index.js index 05c9bdc6b2..f77d1f0b6c 100644 --- a/packages/server/src/selfhost/index.js +++ b/packages/server/src/selfhost/index.js @@ -37,3 +37,8 @@ exports.getSelfHostInfo = async () => { const db = new CouchDB(SELF_HOST_DB) return db.get(SELF_HOST_DOC) } + +exports.getSelfHostAPIKey = async () => { + const info = await exports.getSelfHostInfo() + return info ? info.apiKeyId : null +} diff --git a/packages/server/src/utilities/security/apikey.js b/packages/server/src/utilities/security/apikey.js index b2fd230130..c8965cee43 100644 --- a/packages/server/src/utilities/security/apikey.js +++ b/packages/server/src/utilities/security/apikey.js @@ -1,22 +1,23 @@ const { apiKeyTable } = require("../../db/dynamoClient") const env = require("../../environment") -const { getSelfHostInfo } = require("../../selfhost") +const { getSelfHostAPIKey } = require("../../selfhost") /** * This file purely exists so that we can centralise all logic pertaining to API keys, as their usage differs * in our Cloud environment versus self hosted. */ -exports.getAPIKey = async apiKeyId => { +exports.isAPIKeyValid = async apiKeyId => { if (env.CLOUD && !env.SELF_HOSTED) { - return apiKeyTable.get({ + let apiKeyInfo = await apiKeyTable.get({ primary: apiKeyId, }) + return apiKeyInfo != null } if (env.SELF_HOSTED) { - const selfHostInfo = await getSelfHostInfo() + const selfHostKey = await getSelfHostAPIKey() // if the api key supplied is correct then return structure similar - return apiKeyId === selfHostInfo.apiKeyId ? { pk: apiKeyId } : null + return apiKeyId === selfHostKey ? { pk: apiKeyId } : null } - return null + return false } From 90cfad1dad8e9bcaec92bf1924a815516f3f8040 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Dec 2020 18:02:10 +0000 Subject: [PATCH 11/48] Linting. --- packages/standard-components/src/Card.svelte | 12 ++++++++++-- .../standard-components/src/CardHorizontal.svelte | 14 +++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/standard-components/src/Card.svelte b/packages/standard-components/src/Card.svelte index e8248b7a01..74121590e3 100644 --- a/packages/standard-components/src/Card.svelte +++ b/packages/standard-components/src/Card.svelte @@ -29,11 +29,19 @@ class="container" use:styleable={$component.styles} style="--cardWidth: {cardWidth}"> - {#if showImage}{/if} + {#if showImage} + + {/if}

{heading}

{description}

- {linkText} + {linkText}
diff --git a/packages/standard-components/src/CardHorizontal.svelte b/packages/standard-components/src/CardHorizontal.svelte index f0741fd7de..5d7596e296 100644 --- a/packages/standard-components/src/CardHorizontal.svelte +++ b/packages/standard-components/src/CardHorizontal.svelte @@ -24,15 +24,23 @@ use:styleable={$component.styles} class="container" style="--cardWidth: {cardWidth}"> - {#if showImage}{/if} + {#if showImage} + + {/if}

{heading}

{description}

-
From 0e854473f0127bd1fa83b452d6c18390a9efa000 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 11 Dec 2020 16:38:40 +0000 Subject: [PATCH 12/48] Putting together the basics of the compose system, this is now functional, along with some handy scripts to make it work. This requires minimum docker 1.25.0 to run due to the use of --env-file. --- hosting/docker-compose.yml | 67 +++++++++++++++++++++++++++++++ hosting/hosting.properties | 12 +++--- hosting/install-docker-compose.sh | 4 ++ hosting/nginx.conf | 65 ++++++++++++++++++++++++++++++ hosting/server | 1 + hosting/start.sh | 20 +-------- hosting/utils/testing.sh | 20 +++++++++ packages/server/Dockerfile | 8 +++- 8 files changed, 170 insertions(+), 27 deletions(-) create mode 100644 hosting/docker-compose.yml create mode 100755 hosting/install-docker-compose.sh create mode 100644 hosting/nginx.conf create mode 120000 hosting/server create mode 100755 hosting/utils/testing.sh diff --git a/hosting/docker-compose.yml b/hosting/docker-compose.yml new file mode 100644 index 0000000000..1b1f608f91 --- /dev/null +++ b/hosting/docker-compose.yml @@ -0,0 +1,67 @@ +version: "3" + +services: + app-service: + build: ./server + volumes: + - ./server:/app + ports: + - "4001:4001" + environment: + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + SELF_HOSTED: ${SELF_HOSTED} + COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 + BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} + depends_on: + - nginx-service + - minio-service + - couch-init + + minio-service: + image: minio/minio:RELEASE.2020-12-10T01-54-29Z + volumes: + - data1:/data + expose: + - "9000" + environment: + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + command: server /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + nginx-service: + image: nginx:1.19.2-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + depends_on: + - minio-service + + couchdb-service: + image: apache/couchdb:3.0 + environment: + - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} + - COUCHDB_USER=${COUCH_DB_USER} + ports: + - "5984:5984" + - "4369:4369" + - "9100:9100" + volumes: + - couchdb_data:/couchdb + + couch-init: + image: curlimages/curl + depends_on: + - couchdb-service + command: ["sh","-c","sleep 10 && curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984/_users && curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984/_replicator; fg ; "] + +volumes: + couchdb_data: + driver: local + data1: diff --git a/hosting/hosting.properties b/hosting/hosting.properties index ce3ab326c9..cf818d0eb3 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -1,6 +1,6 @@ -minio_access_key=budibase -minio_secret_key=budibase -minio_port=9000 -# specify the following below settings if switching credentials -#minio_secret_key_old=minioadmin -#minio_access_key_old=minioadmin +SELF_HOSTED=1 +MINIO_ACCESS_KEY=budibase +MINIO_SECRET_KEY=budibase +COUCH_DB_PASSWORD=budibase +COUCH_DB_USER=budibase +BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/hosting/install-docker-compose.sh b/hosting/install-docker-compose.sh new file mode 100755 index 0000000000..6d466a7655 --- /dev/null +++ b/hosting/install-docker-compose.sh @@ -0,0 +1,4 @@ +#!/bin/bash +sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose diff --git a/hosting/nginx.conf b/hosting/nginx.conf new file mode 100644 index 0000000000..9ca4f1d630 --- /dev/null +++ b/hosting/nginx.conf @@ -0,0 +1,65 @@ + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio:9000; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } +} diff --git a/hosting/server b/hosting/server new file mode 120000 index 0000000000..785f5d338c --- /dev/null +++ b/hosting/server @@ -0,0 +1 @@ +../packages/server/ \ No newline at end of file diff --git a/hosting/start.sh b/hosting/start.sh index 94f2fe8896..b32098a3b7 100755 --- a/hosting/start.sh +++ b/hosting/start.sh @@ -1,20 +1,2 @@ #!/bin/bash - -function dockerInstalled { - echo "Checking docker installation..." - if [ ! -x "$(command -v docker)" ]; then - echo "Please install docker to continue" - exit -1 - fi -} - -dockerInstalled - -source "${BASH_SOURCE%/*}/hosting.properties" - -opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key" -if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then - opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old" -fi - -docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data +docker-compose --env-file hosting.properties up diff --git a/hosting/utils/testing.sh b/hosting/utils/testing.sh new file mode 100755 index 0000000000..94f2fe8896 --- /dev/null +++ b/hosting/utils/testing.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function dockerInstalled { + echo "Checking docker installation..." + if [ ! -x "$(command -v docker)" ]; then + echo "Please install docker to continue" + exit -1 + fi +} + +dockerInstalled + +source "${BASH_SOURCE%/*}/hosting.properties" + +opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key" +if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then + opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old" +fi + +docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 96ae3b7a30..9a6b058a17 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -4,12 +4,16 @@ WORKDIR /app ENV CLOUD=1 ENV COUCH_DB_URL=https://couchdb.budi.live:5984 -env BUDIBASE_ENVIRONMENT=PRODUCTION +ENV BUDIBASE_ENVIRONMENT=PRODUCTION # copy files and install dependencies COPY . ./ -RUN yarn +RUN yarn EXPOSE 4001 +# have to add node environment production after install +# due to this causing yarn to stop installing dev dependencies +# which are actually needed to get this environment up and running +ENV NODE_ENV=production CMD ["yarn", "run:docker"] From 2546346f3645d1d103dfb40d2262ea41bbba72cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Dec 2020 15:56:33 +0000 Subject: [PATCH 13/48] Adding the ability to change default logo URL for new apps in self hosting. --- hosting/docker-compose.yml | 3 ++- hosting/hosting.properties | 2 +- packages/server/src/constants/layouts.js | 5 +++-- packages/server/src/constants/screens.js | 4 ++-- packages/server/src/environment.js | 1 + packages/server/src/utilities/index.js | 13 +++++++++++++ 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/hosting/docker-compose.yml b/hosting/docker-compose.yml index 1b1f608f91..13102a28d1 100644 --- a/hosting/docker-compose.yml +++ b/hosting/docker-compose.yml @@ -10,9 +10,10 @@ services: environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} - SELF_HOSTED: ${SELF_HOSTED} + SELF_HOSTED: 1 COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} + LOGO_URL: ${LOGO_URL} depends_on: - nginx-service - minio-service diff --git a/hosting/hosting.properties b/hosting/hosting.properties index cf818d0eb3..8ffe9f1473 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -1,6 +1,6 @@ -SELF_HOSTED=1 MINIO_ACCESS_KEY=budibase MINIO_SECRET_KEY=budibase COUCH_DB_PASSWORD=budibase COUCH_DB_USER=budibase BUDIBASE_ENVIRONMENT=PRODUCTION +LOGO_URL=https://logoipsum.com/logo/logo-15.svg diff --git a/packages/server/src/constants/layouts.js b/packages/server/src/constants/layouts.js index f2ad5b2675..2472e3289e 100644 --- a/packages/server/src/constants/layouts.js +++ b/packages/server/src/constants/layouts.js @@ -1,3 +1,5 @@ +const { getLogoUrl } = require("../utilities") + const BASE_LAYOUT_PROP_IDS = { PRIVATE: "layout_private_master", PUBLIC: "layout_public_master", @@ -88,8 +90,7 @@ const BASE_LAYOUTS = [ active: {}, selected: {}, }, - logoUrl: - "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg", + logoUrl: getLogoUrl(), title: "", backgroundColor: "", color: "", diff --git a/packages/server/src/constants/screens.js b/packages/server/src/constants/screens.js index e08eaa8195..a1a96c994f 100644 --- a/packages/server/src/constants/screens.js +++ b/packages/server/src/constants/screens.js @@ -1,5 +1,6 @@ const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles") const { BASE_LAYOUT_PROP_IDS } = require("./layouts") +const { getLogoUrl } = require("../utilities") exports.createHomeScreen = () => ({ description: "", @@ -138,8 +139,7 @@ exports.createLoginScreen = app => ({ active: {}, selected: {}, }, - logo: - "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg", + logo: getLogoUrl(), title: `Log in to ${app.name}`, buttonText: "Log In", _children: [], diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 0c67f8c747..cca3c4fbac 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -39,6 +39,7 @@ module.exports = { // self hosting features MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, + LOGO_URL: process.env.LOGO_URL, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 8a43e9d27a..c8e2e57bff 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -168,3 +168,16 @@ exports.coerceRowValues = (row, table) => { } return clonedRow } + +/** + * Gets the correct link to the logo URL depending on if running in Cloud or if running in self hosting. + * @returns {string} A URL which links to the correct default logo for new apps. + */ +exports.getLogoUrl = () => { + const BB_LOGO_URL = + "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" + if (env.SELF_HOSTED) { + return env.LOGO_URL || BB_LOGO_URL + } + return BB_LOGO_URL +} From f5a9c68c699841e1aa72ba55d2cb6d9260deb4d3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Dec 2020 18:31:48 +0000 Subject: [PATCH 14/48] WIP - starting to work on deployment and hosting so that they don't use static URLs anymore to reach assets, instead using environment variables to determine what to use. --- hosting/docker-compose.yml | 1 + hosting/hosting.properties | 1 + .../server/src/api/controllers/application.js | 3 + .../src/api/controllers/deploy/selfDeploy.js | 3 +- .../server/src/api/controllers/hosting.js | 39 +++++++++++++ .../src/api/controllers/static/index.js | 29 +++++++++- .../static/templates/BudibaseApp.svelte | 5 +- packages/server/src/api/routes/hosting.js | 13 +++++ packages/server/src/constants/index.js | 2 + packages/server/src/environment.js | 2 + .../server/src/utilities/builder/hosting.js | 57 +++++++++++++++++++ 11 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/api/controllers/hosting.js create mode 100644 packages/server/src/api/routes/hosting.js create mode 100644 packages/server/src/utilities/builder/hosting.js diff --git a/hosting/docker-compose.yml b/hosting/docker-compose.yml index 13102a28d1..72a055e91a 100644 --- a/hosting/docker-compose.yml +++ b/hosting/docker-compose.yml @@ -10,6 +10,7 @@ services: environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_URL: http://minio-service:9000 SELF_HOSTED: 1 COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 8ffe9f1473..9e4dfe1dfc 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -3,4 +3,5 @@ MINIO_SECRET_KEY=budibase COUCH_DB_PASSWORD=budibase COUCH_DB_USER=budibase BUDIBASE_ENVIRONMENT=PRODUCTION +HOSTING_URL="http://localhost:4001" LOGO_URL=https://logoipsum.com/logo/logo-15.svg diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 293c72d7c3..1d259ac425 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -150,6 +150,9 @@ exports.create = async function(ctx) { name: ctx.request.body.name, template: ctx.request.body.template, instance: instance, + deployment: { + type: "cloud", + }, } const instanceDb = new CouchDB(appId) await instanceDb.put(newApplication) diff --git a/packages/server/src/api/controllers/deploy/selfDeploy.js b/packages/server/src/api/controllers/deploy/selfDeploy.js index 0b26af4e36..e20462a236 100644 --- a/packages/server/src/api/controllers/deploy/selfDeploy.js +++ b/packages/server/src/api/controllers/deploy/selfDeploy.js @@ -20,7 +20,7 @@ exports.postDeployment = async function() { exports.deploy = async function(deployment) { const appId = deployment.getAppId() var objClient = new AWS.S3({ - endpoint: "http://localhost:9000", + endpoint: env.MINIO_URL, s3ForcePathStyle: true, // needed with minio? signatureVersion: "v4", params: { @@ -46,6 +46,7 @@ exports.deploy = async function(deployment) { exports.replicateDb = async function(deployment) { const appId = deployment.getAppId() const localDb = new PouchDB(appId) + const remoteDb = new CouchDB(`${env.COUCH_DB_URL}/${appId}`) return performReplication(localDb, remoteDb) } diff --git a/packages/server/src/api/controllers/hosting.js b/packages/server/src/api/controllers/hosting.js new file mode 100644 index 0000000000..4e95405738 --- /dev/null +++ b/packages/server/src/api/controllers/hosting.js @@ -0,0 +1,39 @@ +const CouchDB = require("../../db") +const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") +const { + getHostingInfo, + HostingTypes, + getAppServerUrl, +} = require("../../utilities/builder/hosting") + +exports.fetchInfo = async ctx => { + ctx.body = { + types: Object.values(HostingTypes), + } +} + +exports.save = async ctx => { + const db = new CouchDB(BUILDER_CONFIG_DB) + const { type, appServerUrl, objectStoreUrl, useHttps } = ctx.request.body + if (type === HostingTypes.CLOUD) { + ctx.throw(400, "Cannot update Cloud hosting information") + } + await db.put({ + _id: HOSTING_DOC, + type, + appServerUrl, + objectStoreUrl, + useHttps, + }) + ctx.body = "Hosting information saved successfully" +} + +exports.fetch = async ctx => { + ctx.body = await getHostingInfo() +} + +exports.fetchUrls = async ctx => { + ctx.body = { + appServer: getAppServerUrl(ctx.appId), + } +} diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index f884725b29..468a896a96 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -17,6 +17,22 @@ const setBuilderToken = require("../../../utilities/builder/setBuilderToken") const fileProcessor = require("../../../utilities/fileProcessor") const env = require("../../../environment") +function appServerUrl(appId) { + if (env.SELF_HOSTED) { + return env.HOSTING_URL + } else { + return `https://${appId}.app.budi.live` + } +} + +function objectStoreUrl() { + if (env.SELF_HOSTED) { + return env.MINIO_URL + } else { + return "https://cdn.app.budi.live/assets" + } +} + // this was the version before we started versioning the component library const COMP_LIB_BASE_APP_VERSION = "0.2.5" @@ -148,6 +164,7 @@ exports.serveApp = async function(ctx) { title: appInfo.name, production: env.CLOUD, appId: ctx.params.appId, + appServerUrl: appServerUrl(ctx.params.appId), }) const template = handlebars.compile( @@ -166,8 +183,9 @@ exports.serveAttachment = async function(ctx) { const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments") // Serve from CloudFront + // TODO: need to replace this with link to self hosted object store if (env.CLOUD) { - const S3_URL = `https://cdn.app.budi.live/assets/${appId}/attachments/${ctx.file}` + const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file) const response = await fetch(S3_URL) const body = await response.text() ctx.set("Content-Type", response.headers.get("Content-Type")) @@ -213,7 +231,14 @@ exports.serveComponentLibrary = async function(ctx) { componentLib += `-${COMP_LIB_BASE_APP_VERSION}` } const S3_URL = encodeURI( - `https://${appId}.app.budi.live/assets/${componentLib}/${ctx.query.library}/dist/index.js` + join( + appServerUrl(appId), + "assets", + componentLib, + ctx.query.library, + "dist", + "index.js" + ) ) const response = await fetch(S3_URL) const body = await response.text() diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte index 3641c26e93..0729737a44 100644 --- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte +++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte @@ -4,12 +4,11 @@ export let appId export let production - - export const PRODUCTION_ASSETS_URL = `https://${appId}.app.budi.live` + export let appServerUrl function publicPath(path) { if (production) { - return `${PRODUCTION_ASSETS_URL}/assets/${appId}/${path}` + return `${appServerUrl}/assets/${appId}/${path}` } return `/assets/${path}` diff --git a/packages/server/src/api/routes/hosting.js b/packages/server/src/api/routes/hosting.js new file mode 100644 index 0000000000..b8d4256bcd --- /dev/null +++ b/packages/server/src/api/routes/hosting.js @@ -0,0 +1,13 @@ +const Router = require("@koa/router") +const controller = require("../controllers/hosting") +const authorized = require("../../middleware/authorized") +const { BUILDER } = require("../../utilities/security/permissions") + +const router = Router() + +router + .fetch("/api/hosting/info", authorized(BUILDER), controller.fetchInfo) + .fetch("/api/hosting", authorized(BUILDER), controller.fetch) + .post("/api/hosting", authorized(BUILDER), controller.save) + +module.exports = router diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 0ae805c064..5a97a62af6 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -41,3 +41,5 @@ const USERS_TABLE_SCHEMA = { exports.AuthTypes = AuthTypes exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA +exports.BUILDER_CONFIG_DB = "builder-config-db" +exports.HOSTING_DOC = "hosting-doc" diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index cca3c4fbac..19d3cf1c57 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -39,6 +39,8 @@ module.exports = { // self hosting features MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, + MINIO_URL: process.MINIO_URL, + HOSTING_URL: process.HOSTING_URL, LOGO_URL: process.env.LOGO_URL, _set(key, value) { process.env[key] = value diff --git a/packages/server/src/utilities/builder/hosting.js b/packages/server/src/utilities/builder/hosting.js new file mode 100644 index 0000000000..97af87d3e2 --- /dev/null +++ b/packages/server/src/utilities/builder/hosting.js @@ -0,0 +1,57 @@ +const CouchDB = require("../../db") +const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") +const { join } = require("path") + +function getProtocol(hostingInfo) { + return hostingInfo.useHttps ? "https://" : "http://" +} + +exports.HostingTypes = { + CLOUD: "cloud", + SELF: "self", +} + +exports.getHostingInfo = async () => { + const db = new CouchDB(BUILDER_CONFIG_DB) + let doc + try { + doc = await db.get(HOSTING_DOC) + } catch (err) { + // don't write this doc, want to be able to update these default props + // for our servers with a new release without needing to worry about state of + // PouchDB in peoples installations + doc = { + _id: HOSTING_DOC, + type: exports.HostingTypes.CLOUD, + appServerUrl: "app.budi.live", + objectStoreUrl: "cdn.app.budi.live", + templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com", + useHttps: true, + } + } + return doc +} + +exports.getAppServerUrl = async (appId) => { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + let url + if (hostingInfo.type === "cloud") { + url = `${protocol}${appId}.${hostingInfo.appServerUrl}` + } else { + url = `${protocol}${hostingInfo.appServerUrl}` + } + return url +} + +exports.getTemplatesUrl = async (appId, type, name) => { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + let path + if (type && name) { + path = join("templates", type, `${name}.tar.gz`) + } else { + path = "manifest.json" + } + return join(`${protocol}${hostingInfo.templatesUrl}`, path) +} From 21978007cfed9b65c94c0cd110e37b97f5ff0468 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Dec 2020 16:41:55 +0000 Subject: [PATCH 15/48] Adding feature to allow configured the builder to point at a self hosted system instead of production (for deployment). --- packages/builder/src/builderStore/index.js | 5 +- .../builder/src/builderStore/store/hosting.js | 35 +++++++++++++ .../start/BuilderSettingsButton.svelte | 32 ++++++++++++ .../start/BuilderSettingsModal.svelte | 51 +++++++++++++++++++ packages/builder/src/pages/_layout.svelte | 48 +++++++++-------- .../server/src/api/controllers/hosting.js | 4 +- packages/server/src/api/routes/hosting.js | 4 +- packages/server/src/api/routes/index.js | 2 + 8 files changed, 156 insertions(+), 25 deletions(-) create mode 100644 packages/builder/src/builderStore/store/hosting.js create mode 100644 packages/builder/src/components/start/BuilderSettingsButton.svelte create mode 100644 packages/builder/src/components/start/BuilderSettingsModal.svelte diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index 887ef733e4..8b6666f3ab 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -1,6 +1,8 @@ import { getFrontendStore } from "./store/frontend" import { getBackendUiStore } from "./store/backend" -import { getAutomationStore } from "./store/automation/" +import { getAutomationStore } from "./store/automation" +import { getHostingStore } from "./store/hosting" + import { getThemeStore } from "./store/theme" import { derived } from "svelte/store" import analytics from "analytics" @@ -11,6 +13,7 @@ export const store = getFrontendStore() export const backendUiStore = getBackendUiStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() +export const hostingStore = getHostingStore() export const currentAsset = derived(store, $store => { const layout = $store.layouts diff --git a/packages/builder/src/builderStore/store/hosting.js b/packages/builder/src/builderStore/store/hosting.js new file mode 100644 index 0000000000..0e46a42c3b --- /dev/null +++ b/packages/builder/src/builderStore/store/hosting.js @@ -0,0 +1,35 @@ +import { writable } from "svelte/store" +import api from "../api" + +const INITIAL_BACKEND_UI_STATE = { + hostingInfo: {}, + appUrl: "", +} + +export const getHostingStore = () => { + const store = writable({...INITIAL_BACKEND_UI_STATE}) + store.actions = { + fetch: async () => { + const response = await api.get("/api/hosting/") + const info = await response.json() + store.update(state => { + state.hostingInfo = info + return state + }) + return info + }, + save: async hostingInfo => { + const response = await api.post("/api/hosting", hostingInfo) + const revision = (await response.json()).rev + store.update(state => { + state.hostingInfo = { + ...hostingInfo, + _rev: revision, + } + return state + }) + } + } + + return store +} diff --git a/packages/builder/src/components/start/BuilderSettingsButton.svelte b/packages/builder/src/components/start/BuilderSettingsButton.svelte new file mode 100644 index 0000000000..7252ab4ded --- /dev/null +++ b/packages/builder/src/components/start/BuilderSettingsButton.svelte @@ -0,0 +1,32 @@ + + +
+ +
+ + + + + diff --git a/packages/builder/src/components/start/BuilderSettingsModal.svelte b/packages/builder/src/components/start/BuilderSettingsModal.svelte new file mode 100644 index 0000000000..3d58d65e9b --- /dev/null +++ b/packages/builder/src/components/start/BuilderSettingsModal.svelte @@ -0,0 +1,51 @@ + + + +
Hosting
+

This section contains settings that relate to the deployment and hosting of apps made in this builder.

+ + {#if selfhosted} + + + + {/if} + + + diff --git a/packages/builder/src/pages/_layout.svelte b/packages/builder/src/pages/_layout.svelte index 3f8b36d8f4..13c6bed988 100644 --- a/packages/builder/src/pages/_layout.svelte +++ b/packages/builder/src/pages/_layout.svelte @@ -7,6 +7,9 @@ CommunityIcon, BugIcon, } from "components/common/Icons" + import BuilderSettingsButton from "components/start/BuilderSettingsButton.svelte" + + let modal
@@ -16,27 +19,30 @@
- - - - - - +
+ + + + + +
+
+ +
-
@@ -76,8 +82,10 @@ } .nav-section { - margin: 20px 0px; + margin: 20px 0 0 0; display: flex; flex-direction: column; + justify-content: space-between; + height: 100%; } diff --git a/packages/server/src/api/controllers/hosting.js b/packages/server/src/api/controllers/hosting.js index 4e95405738..f2a2ac61cb 100644 --- a/packages/server/src/api/controllers/hosting.js +++ b/packages/server/src/api/controllers/hosting.js @@ -18,14 +18,14 @@ exports.save = async ctx => { if (type === HostingTypes.CLOUD) { ctx.throw(400, "Cannot update Cloud hosting information") } - await db.put({ + const response = await db.put({ _id: HOSTING_DOC, type, appServerUrl, objectStoreUrl, useHttps, }) - ctx.body = "Hosting information saved successfully" + ctx.body = response } exports.fetch = async ctx => { diff --git a/packages/server/src/api/routes/hosting.js b/packages/server/src/api/routes/hosting.js index b8d4256bcd..783259a514 100644 --- a/packages/server/src/api/routes/hosting.js +++ b/packages/server/src/api/routes/hosting.js @@ -6,8 +6,8 @@ const { BUILDER } = require("../../utilities/security/permissions") const router = Router() router - .fetch("/api/hosting/info", authorized(BUILDER), controller.fetchInfo) - .fetch("/api/hosting", authorized(BUILDER), controller.fetch) + .get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo) + .get("/api/hosting", authorized(BUILDER), controller.fetch) .post("/api/hosting", authorized(BUILDER), controller.save) module.exports = router diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index 2a8669b042..455f0fe989 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -17,6 +17,7 @@ const templatesRoutes = require("./templates") const analyticsRoutes = require("./analytics") const routingRoutes = require("./routing") const permissionRoutes = require("./permission") +const hostingRoutes = require("./hosting") exports.mainRoutes = [ deployRoutes, @@ -34,6 +35,7 @@ exports.mainRoutes = [ webhookRoutes, routingRoutes, permissionRoutes, + hostingRoutes, // these need to be handled last as they still use /api/:tableId // this could be breaking as koa may recognise other routes as this tableRoutes, From c65ed9ad462a573587d9a2f48bb6a4dc3156f257 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Dec 2020 16:42:40 +0000 Subject: [PATCH 16/48] Linting. --- packages/builder/src/builderStore/store/hosting.js | 4 ++-- .../components/start/BuilderSettingsButton.svelte | 2 +- .../src/components/start/BuilderSettingsModal.svelte | 12 +++++++++--- packages/builder/src/pages/_layout.svelte | 2 +- packages/server/src/utilities/builder/hosting.js | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/builderStore/store/hosting.js b/packages/builder/src/builderStore/store/hosting.js index 0e46a42c3b..3e452920d2 100644 --- a/packages/builder/src/builderStore/store/hosting.js +++ b/packages/builder/src/builderStore/store/hosting.js @@ -7,7 +7,7 @@ const INITIAL_BACKEND_UI_STATE = { } export const getHostingStore = () => { - const store = writable({...INITIAL_BACKEND_UI_STATE}) + const store = writable({ ...INITIAL_BACKEND_UI_STATE }) store.actions = { fetch: async () => { const response = await api.get("/api/hosting/") @@ -28,7 +28,7 @@ export const getHostingStore = () => { } return state }) - } + }, } return store diff --git a/packages/builder/src/components/start/BuilderSettingsButton.svelte b/packages/builder/src/components/start/BuilderSettingsButton.svelte index 7252ab4ded..5f0008e375 100644 --- a/packages/builder/src/components/start/BuilderSettingsButton.svelte +++ b/packages/builder/src/components/start/BuilderSettingsButton.svelte @@ -7,7 +7,7 @@
diff --git a/packages/builder/src/components/start/BuilderSettingsModal.svelte b/packages/builder/src/components/start/BuilderSettingsModal.svelte index 3d58d65e9b..7a4ee41c31 100644 --- a/packages/builder/src/components/start/BuilderSettingsModal.svelte +++ b/packages/builder/src/components/start/BuilderSettingsModal.svelte @@ -24,13 +24,19 @@ onMount(async () => { hostingInfo = await hostingStore.actions.fetch() selfhosted = hostingInfo.type === "self" - }) - +
Hosting
-

This section contains settings that relate to the deployment and hosting of apps made in this builder.

+

+ This section contains settings that relate to the deployment and hosting of + apps made in this builder. +

{#if selfhosted} diff --git a/packages/builder/src/pages/_layout.svelte b/packages/builder/src/pages/_layout.svelte index 13c6bed988..ac45ff8372 100644 --- a/packages/builder/src/pages/_layout.svelte +++ b/packages/builder/src/pages/_layout.svelte @@ -39,7 +39,7 @@ href="https://github.com/Budibase/budibase/issues/new/choose" />
- +
diff --git a/packages/server/src/utilities/builder/hosting.js b/packages/server/src/utilities/builder/hosting.js index 97af87d3e2..404e1ead94 100644 --- a/packages/server/src/utilities/builder/hosting.js +++ b/packages/server/src/utilities/builder/hosting.js @@ -32,7 +32,7 @@ exports.getHostingInfo = async () => { return doc } -exports.getAppServerUrl = async (appId) => { +exports.getAppServerUrl = async appId => { const hostingInfo = await exports.getHostingInfo() const protocol = getProtocol(hostingInfo) let url From cc34838f451256709f4245583e256185a7f2f6a5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Dec 2020 17:27:45 +0000 Subject: [PATCH 17/48] Updating deployment service, checking in builder the builder settings info stored in DB before deploying. --- .../src/builderStore/store/frontend.js | 2 ++ .../builder/src/builderStore/store/hosting.js | 6 ++-- .../automation/Shared/WebhookDisplay.svelte | 5 +-- .../deploy/DeploymentHistory.svelte | 5 +-- .../src/api/controllers/deploy/index.js | 31 ++++++++++++------- .../server/src/api/controllers/hosting.js | 12 +++---- packages/server/src/api/routes/hosting.js | 1 + 7 files changed, 35 insertions(+), 27 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 3b02edcfdb..dc168ac97c 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -8,6 +8,7 @@ import { import { allScreens, backendUiStore, + hostingStore, currentAsset, mainLayout, selectedComponent, @@ -69,6 +70,7 @@ export const getFrontendStore = () => { appInstance: pkg.application.instance, })) + await hostingStore.actions.fetch() await backendUiStore.actions.database.select(pkg.application.instance) }, routing: { diff --git a/packages/builder/src/builderStore/store/hosting.js b/packages/builder/src/builderStore/store/hosting.js index 3e452920d2..722efb6dbd 100644 --- a/packages/builder/src/builderStore/store/hosting.js +++ b/packages/builder/src/builderStore/store/hosting.js @@ -10,10 +10,11 @@ export const getHostingStore = () => { const store = writable({ ...INITIAL_BACKEND_UI_STATE }) store.actions = { fetch: async () => { - const response = await api.get("/api/hosting/") - const info = await response.json() + const responses = await Promise.all([api.get("/api/hosting/"), api.get("/api/hosting/urls")]) + const [info, urls] = await Promise.all(responses.map(resp => resp.json())) store.update(state => { state.hostingInfo = info + state.appUrl = urls.appServer return state }) return info @@ -30,6 +31,5 @@ export const getHostingStore = () => { }) }, } - return store } diff --git a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte index 3299206e78..2edcbec3f1 100644 --- a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte +++ b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte @@ -1,16 +1,17 @@ -
- -
-
- -
-
- - -
- {#if $themeStore.darkMode && !showAdvanced} -
- -
- {/if} - {#if $themeStore.darkMode && showAdvanced} - - - -
- -
- {/if} +
+
+ +
+ {#if $themeStore.darkMode && !showAdvanced} +
+
- + {/if} + {#if $themeStore.darkMode && showAdvanced} + + + +
+ +
+ {/if}
+ \ No newline at end of file diff --git a/packages/builder/src/components/settings/ThemeEditorDropdown.svelte b/packages/builder/src/components/settings/ThemeEditorDropdown.svelte new file mode 100644 index 0000000000..3ca620b5d0 --- /dev/null +++ b/packages/builder/src/components/settings/ThemeEditorDropdown.svelte @@ -0,0 +1,56 @@ + + +
+ +
+
+ +
+ + +
+ +
+ + diff --git a/packages/builder/src/components/start/BuilderSettingsModal.svelte b/packages/builder/src/components/start/BuilderSettingsModal.svelte index d101369fba..302b385af8 100644 --- a/packages/builder/src/components/start/BuilderSettingsModal.svelte +++ b/packages/builder/src/components/start/BuilderSettingsModal.svelte @@ -2,6 +2,7 @@ import { notifier } from "builderStore/store/notifications" import { hostingStore } from "builderStore" import { Input, ModalContent, Toggle } from "@budibase/bbui" + import ThemeEditor from "components/settings/ThemeEditor.svelte" import analytics from "analytics" import { onMount } from "svelte" @@ -32,6 +33,8 @@ confirmText="Save" onConfirm={save} showConfirmButton={selfhosted}> +
Theme
+
Hosting

This section contains settings that relate to the deployment and hosting of diff --git a/packages/builder/src/pages/[application]/_reset.svelte b/packages/builder/src/pages/[application]/_reset.svelte index 74367c794d..a02287a89e 100644 --- a/packages/builder/src/pages/[application]/_reset.svelte +++ b/packages/builder/src/pages/[application]/_reset.svelte @@ -2,7 +2,7 @@ import { store, automationStore, backendUiStore } from "builderStore" import { Button } from "@budibase/bbui" import SettingsLink from "components/settings/Link.svelte" - import ThemeEditor from "components/settings/ThemeEditor.svelte" + import ThemeEditorDropdown from "components/settings/ThemeEditorDropdown.svelte" import FeedbackNavLink from "components/userInterface/Feedback/FeedbackNavLink.svelte" import { get } from "builderStore/api" import { isActive, goto, layout } from "@sveltech/routify" @@ -67,7 +67,7 @@ {/each}

- +
Date: Tue, 5 Jan 2021 16:04:12 +0000 Subject: [PATCH 28/48] Initial state of swapping to envoy from nginx, this isn't fully functional, still need to make the builder use it and need to get URL re-writing working. --- ...docker-compose.yml => docker-compose.yaml} | 23 +++-- hosting/envoy.yaml | 91 +++++++++++++++++++ hosting/hosting.properties | 1 + hosting/nginx.conf | 64 ------------- 4 files changed, 105 insertions(+), 74 deletions(-) rename hosting/{docker-compose.yml => docker-compose.yaml} (84%) create mode 100644 hosting/envoy.yaml delete mode 100644 hosting/nginx.conf diff --git a/hosting/docker-compose.yml b/hosting/docker-compose.yaml similarity index 84% rename from hosting/docker-compose.yml rename to hosting/docker-compose.yaml index 65d89d2b0f..184e340ce4 100644 --- a/hosting/docker-compose.yml +++ b/hosting/docker-compose.yaml @@ -6,13 +6,13 @@ services: volumes: - ./server:/app ports: - - "${APP_PORT}:${APP_PORT}" + - "${APP_PORT}:4002" environment: SELF_HOSTED: 1 COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} LOGO_URL: ${LOGO_URL} - PORT: ${APP_PORT} + PORT: 4002 HOSTING_URL: ${HOSTING_URL} MINIO_PORT: ${MINIO_PORT} JWT_SECRET: ${JWT_SECRET} @@ -22,19 +22,18 @@ services: worker-service: build: ./worker ports: - - "${WORKER_PORT}:${WORKER_PORT}" + - "${WORKER_PORT}:4003" environment: SELF_HOSTED: 1, DEPLOYMENT_API_KEY: ${WORKER_API_KEY} - PORT: ${WORKER_PORT} + PORT: 4003 MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} - RAW_MINIO_URL: http://nginx-service:5001 + RAW_MINIO_URL: http://minio-service:9000 COUCH_DB_USERNAME: ${COUCH_DB_USER} COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} RAW_COUCH_DB_URL: http://couchdb-service:5984 depends_on: - - nginx-service - minio-service - couch-init @@ -54,14 +53,18 @@ services: timeout: 20s retries: 3 - nginx-service: - image: nginx:1.19.2-alpine + proxy-service: + image: envoyproxy/envoy:v1.16-latest volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./envoy.yaml:/etc/envoy/envoy.yaml ports: - - "${MINIO_PORT}:5001" + - "${MAIN_PORT}:10000" + - "9901:9901" depends_on: - minio-service + - worker-service + - app-service + - couchdb-service couchdb-service: image: apache/couchdb:3.0 diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml new file mode 100644 index 0000000000..f9bbc49025 --- /dev/null +++ b/hosting/envoy.yaml @@ -0,0 +1,91 @@ +admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: { address: 0.0.0.0, port_value: 9901 } + +static_resources: + listeners: + - name: main_listener + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: auto + route_config: + name: local_route + virtual_hosts: + - name: local_services + domains: ["*"] + routes: + - match: { prefix: "/app" } + route: { cluster: app-service } + - match: { prefix: "/obj" } + route: { cluster: minio-service } + - match: { prefix: "/worker" } + route: { cluster: worker-service } + - match: { prefix: "/db" } + route: { cluster: couchdb-service } + http_filters: + - name: envoy.filters.http.router + + clusters: + - name: app-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: app-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: app-service + port_value: 4002 + + - name: minio-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: minio-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: minio-service + port_value: 9000 + + - name: worker-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: worker-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: worker-service + port_value: 4003 + + - name: couchdb-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: couchdb-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: couchdb-service + port_value: 5984 + diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 35df12f07b..4f57f0278d 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -6,6 +6,7 @@ WORKER_API_KEY=budibase BUDIBASE_ENVIRONMENT=PRODUCTION HOSTING_URL=http://localhost LOGO_URL=https://logoipsum.com/logo/logo-15.svg +MAIN_PORT=10000 APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 diff --git a/hosting/nginx.conf b/hosting/nginx.conf deleted file mode 100644 index 4782d7a0b1..0000000000 --- a/hosting/nginx.conf +++ /dev/null @@ -1,64 +0,0 @@ - -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - # include /etc/nginx/conf.d/*.conf; - - upstream minio { - # if adding more minio services to cluster add them here - server minio-service:9000; - } - - server { - listen 5001; - listen [::]:5001; - server_name localhost; - - # To allow special characters in headers - ignore_invalid_headers off; - client_max_body_size 100m; - # To disable buffering - proxy_buffering off; - - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_connect_timeout 300; - # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 - proxy_http_version 1.1; - proxy_set_header Connection ""; - chunked_transfer_encoding off; - - proxy_pass http://minio; - } - } -} From 40a6348e7c0c07f5f7cd15df1a43a78e7aee09f5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Jan 2021 18:47:51 +0000 Subject: [PATCH 29/48] Version working with reverse proxy, need to just change settings menu in builder now to use this properly. --- hosting/docker-compose.yaml | 2 +- hosting/envoy.yaml | 31 ++++++++++++++++++++++--------- packages/client/src/api/api.js | 3 ++- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 184e340ce4..cf86d3cd98 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -42,7 +42,7 @@ services: volumes: - data1:/data ports: - - "9000:9000" + - "${MINIO_PORT}:9000" environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index f9bbc49025..a6b110428e 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -13,7 +13,7 @@ static_resources: - name: envoy.filters.network.http_connection_manager typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http + stat_prefix: ingress codec_type: auto route_config: name: local_route @@ -21,14 +21,27 @@ static_resources: - name: local_services domains: ["*"] routes: - - match: { prefix: "/app" } - route: { cluster: app-service } - - match: { prefix: "/obj" } - route: { cluster: minio-service } - - match: { prefix: "/worker" } - route: { cluster: worker-service } - - match: { prefix: "/db" } - route: { cluster: couchdb-service } + - match: { prefix: "/app/" } + route: + cluster: app-service + prefix_rewrite: "/" + + - match: { prefix: "/worker/" } + route: + cluster: worker-service + prefix_rewrite: "/" + + - match: { prefix: "/db/" } + route: + prefix_rewrite: "/" + cluster: couchdb-service + + # minio is on the default route because this works + # best, minio + AWS SDK doesn't handle path proxy + - match: { prefix: "/" } + route: + cluster: minio-service + http_filters: - name: envoy.filters.http.router diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js index ea88787865..c339bf8ac6 100644 --- a/packages/client/src/api/api.js +++ b/packages/client/src/api/api.js @@ -9,7 +9,8 @@ let cache = {} * Makes a fully formatted URL based on the SDK configuration. */ const makeFullURL = path => { - return `/${path}`.replace("//", "/") + const isProxy = window.location.pathname.startsWith("/app/") + return `${isProxy ? "/app/" : "/"}${path}`.replace("//", "/") } /** From 287bcd01d25c696f5a428366e4f8e9cc98d69d8a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Jan 2021 10:59:10 +0000 Subject: [PATCH 30/48] Fixing up settings to only take the one URL now, the main hosting URL. --- .../start/BuilderSettingsModal.svelte | 5 +-- .../server/src/utilities/builder/hosting.js | 31 +++++++++---------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/packages/builder/src/components/start/BuilderSettingsModal.svelte b/packages/builder/src/components/start/BuilderSettingsModal.svelte index 302b385af8..6ddad0c787 100644 --- a/packages/builder/src/components/start/BuilderSettingsModal.svelte +++ b/packages/builder/src/components/start/BuilderSettingsModal.svelte @@ -42,10 +42,7 @@

{#if selfhosted} - - - - + {/if} diff --git a/packages/server/src/utilities/builder/hosting.js b/packages/server/src/utilities/builder/hosting.js index 0e5e07b26f..f7a3528733 100644 --- a/packages/server/src/utilities/builder/hosting.js +++ b/packages/server/src/utilities/builder/hosting.js @@ -5,6 +5,14 @@ function getProtocol(hostingInfo) { return hostingInfo.useHttps ? "https://" : "http://" } +async function getURLWithPath(pathIfSelfHosted) { + const hostingInfo = await exports.getHostingInfo() + const protocol = getProtocol(hostingInfo) + const path = + hostingInfo.type === exports.HostingTypes.SELF ? pathIfSelfHosted : "" + return `${protocol}${hostingInfo.hostingUrl}${path}` +} + exports.HostingTypes = { CLOUD: "cloud", SELF: "self", @@ -22,10 +30,7 @@ exports.getHostingInfo = async () => { doc = { _id: HOSTING_DOC, type: exports.HostingTypes.CLOUD, - appUrl: "app.budi.live", - workerUrl: "", - minioUrl: "", - couchUrl: "", + hostingUrl: "app.budi.live", templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com", useHttps: true, } @@ -37,30 +42,24 @@ exports.getAppUrl = async appId => { const hostingInfo = await exports.getHostingInfo() const protocol = getProtocol(hostingInfo) let url - if (hostingInfo.type === "cloud") { - url = `${protocol}${appId}.${hostingInfo.appUrl}` + if (hostingInfo.type === exports.HostingTypes.CLOUD) { + url = `${protocol}${appId}.${hostingInfo.hostingUrl}` } else { - url = `${protocol}${hostingInfo.appUrl}` + url = `${protocol}${hostingInfo.hostingUrl}/app` } return url } exports.getWorkerUrl = async () => { - const hostingInfo = await exports.getHostingInfo() - const protocol = getProtocol(hostingInfo) - return `${protocol}${hostingInfo.workerUrl}` + return getURLWithPath("/worker") } exports.getMinioUrl = async () => { - const hostingInfo = await exports.getHostingInfo() - const protocol = getProtocol(hostingInfo) - return `${protocol}${hostingInfo.minioUrl}` + return getURLWithPath("/") } exports.getCouchUrl = async () => { - const hostingInfo = await exports.getHostingInfo() - const protocol = getProtocol(hostingInfo) - return `${protocol}${hostingInfo.couchUrl}` + return getURLWithPath("/db") } exports.getTemplatesUrl = async (appId, type, name) => { From 882cfa700bc76ff8a24e2c5b71d9b04a08147ff8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Jan 2021 16:58:29 +0000 Subject: [PATCH 31/48] Updating worker to support using a self host key, a basic level of security, stopping builder from asking for API key if currently configured for self hosting, made the default values for self hosting make sense for a basic local installation, this should be final. --- hosting/docker-compose.yaml | 8 ++-- hosting/envoy.yaml | 2 +- hosting/hosting.properties | 23 +++++++++--- hosting/start.sh | 2 +- .../start/BuilderSettingsModal.svelte | 29 +++++++++++---- .../components/start/CreateAppModal.svelte | 30 +++++++++------ .../src/api/controllers/deploy/index.js | 5 ++- .../src/api/controllers/deploy/selfDeploy.js | 37 +++++++++++-------- .../src/api/controllers/deploy/utils.js | 3 +- .../server/src/api/controllers/hosting.js | 16 +++++--- .../src/api/controllers/static/index.js | 2 +- packages/server/src/environment.js | 2 +- .../server/src/utilities/builder/hosting.js | 10 ++++- packages/worker/src/api/controllers/deploy.js | 21 ++++++----- packages/worker/src/environment.js | 1 + packages/worker/src/middleware/check-key.js | 9 ++++- 16 files changed, 133 insertions(+), 67 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index cf86d3cd98..cfdbc48e5f 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -14,8 +14,8 @@ services: LOGO_URL: ${LOGO_URL} PORT: 4002 HOSTING_URL: ${HOSTING_URL} - MINIO_PORT: ${MINIO_PORT} JWT_SECRET: ${JWT_SECRET} + PROXY_PORT: ${MAIN_PORT} depends_on: - worker-service @@ -33,6 +33,7 @@ services: COUCH_DB_USERNAME: ${COUCH_DB_USER} COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} RAW_COUCH_DB_URL: http://couchdb-service:5984 + SELF_HOST_KEY: ${HOSTING_KEY} depends_on: - minio-service - couch-init @@ -40,7 +41,7 @@ services: minio-service: image: minio/minio volumes: - - data1:/data + - minio_data:/data ports: - "${MINIO_PORT}:9000" environment: @@ -89,4 +90,5 @@ services: volumes: couchdb_data: driver: local - data1: + minio_data: + driver: local diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index a6b110428e..4e0e233430 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -33,8 +33,8 @@ static_resources: - match: { prefix: "/db/" } route: - prefix_rewrite: "/" cluster: couchdb-service + prefix_rewrite: "/" # minio is on the default route because this works # best, minio + AWS SDK doesn't handle path proxy diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 4f57f0278d..5907d62819 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -1,14 +1,27 @@ +# Use the main port in the builder for your self hosting URL, e.g. localhost:10000 +MAIN_PORT=10000 + +# Use this password when configuring your self hosting settings +# This should be updated +HOSTING_KEY=budibase + +# This section contains customisation options +HOSTING_URL=http://localhost +LOGO_URL=https://logoipsum.com/logo/logo-15.svg +HOSTING_URL=http://localhost + +# This section contains all secrets pertaining to the system +# These should be updated +JWT_SECRET=testsecret MINIO_ACCESS_KEY=budibase MINIO_SECRET_KEY=budibase COUCH_DB_PASSWORD=budibase COUCH_DB_USER=budibase WORKER_API_KEY=budibase -BUDIBASE_ENVIRONMENT=PRODUCTION -HOSTING_URL=http://localhost -LOGO_URL=https://logoipsum.com/logo/logo-15.svg -MAIN_PORT=10000 + +# This section contains variables that do not need to be altered under normal circumstances APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 -JWT_SECRET=testsecret +BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/hosting/start.sh b/hosting/start.sh index b32098a3b7..803db51eda 100755 --- a/hosting/start.sh +++ b/hosting/start.sh @@ -1,2 +1,2 @@ #!/bin/bash -docker-compose --env-file hosting.properties up +docker-compose --env-file hosting.properties up --build diff --git a/packages/builder/src/components/start/BuilderSettingsModal.svelte b/packages/builder/src/components/start/BuilderSettingsModal.svelte index 6ddad0c787..0199069f73 100644 --- a/packages/builder/src/components/start/BuilderSettingsModal.svelte +++ b/packages/builder/src/components/start/BuilderSettingsModal.svelte @@ -6,14 +6,18 @@ import analytics from "analytics" import { onMount } from "svelte" - let selfhosted = false let hostingInfo + let selfhosted = false async function save() { - if (!selfhosted) { - return - } hostingInfo.type = selfhosted ? "self" : "cloud" + if (!selfhosted && hostingInfo._rev) { + hostingInfo = { + type: hostingInfo.type, + _id: hostingInfo._id, + _rev: hostingInfo._rev, + } + } try { await hostingStore.actions.save(hostingInfo) notifier.success(`Settings saved.`) @@ -22,6 +26,15 @@ } } + function updateSelfHosting(event) { + + if (hostingInfo.type === "cloud" && event.target.checked) { + hostingInfo.hostingUrl = "localhost:10000" + hostingInfo.useHttps = false + hostingInfo.selfHostKey = "budibase" + } + } + onMount(async () => { hostingInfo = await hostingStore.actions.fetch() selfhosted = hostingInfo.type === "self" @@ -31,8 +44,7 @@ + onConfirm={save}>
Theme
Hosting
@@ -40,9 +52,10 @@ This section contains settings that relate to the deployment and hosting of apps made in this builder.

- + {#if selfhosted} - + + {/if} diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index b88397b6c4..39ca5032a4 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -1,6 +1,6 @@ - +
Theme
Hosting
@@ -52,7 +48,11 @@ This section contains settings that relate to the deployment and hosting of apps made in this builder.

- + {#if selfhosted} diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index 39ca5032a4..2b54575795 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -1,6 +1,11 @@ + {{{body}}} \ No newline at end of file From 85320ea938dd6a1c515a8ff4b94ff1281c60113b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 7 Jan 2021 15:37:41 +0000 Subject: [PATCH 38/48] Some minor updates, making template object store URL in self hosting relative so everything just goes through the proxy, and fixing issue with API key being required to create apps in self hosting. --- hosting/docker-compose.yaml | 2 - hosting/hosting.properties | 2 - .../components/start/CreateAppModal.svelte | 46 +++++++++++-------- .../src/api/controllers/static/index.js | 6 +-- packages/server/src/environment.js | 2 - 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index b0e3941c93..32659ec53c 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -11,9 +11,7 @@ services: BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} LOGO_URL: ${LOGO_URL} PORT: 4002 - HOSTING_URL: ${HOSTING_URL} JWT_SECRET: ${JWT_SECRET} - PROXY_PORT: ${MAIN_PORT} depends_on: - worker-service diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 5907d62819..5776f108e3 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -6,9 +6,7 @@ MAIN_PORT=10000 HOSTING_KEY=budibase # This section contains customisation options -HOSTING_URL=http://localhost LOGO_URL=https://logoipsum.com/logo/logo-15.svg -HOSTING_URL=http://localhost # This section contains all secrets pertaining to the system # These should be updated diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index 1f27f18aaa..84d1ef7fa3 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -29,6 +29,7 @@ let lastApiKey let fetchApiKeyPromise const validateApiKey = async apiKey => { + if (isApiKeyValid) return true if (!apiKey) return false // make sure we only fetch once, unless API Key is changed @@ -45,27 +46,31 @@ return isApiKeyValid } + const apiValidation = { + apiKey: string() + .required("Please enter your API key.") + .test("valid-apikey", "This API key is invalid", validateApiKey), + } + const infoValidation = { + applicationName: string().required("Your application must have a name."), + } + const userValidation = { + email: string() + .email() + .required("Your application needs a first user."), + password: string().required( + "Please enter a password for your first user." + ), + roleId: string().required("You need to select a role for your user."), + } + let submitting = false let errors = {} let validationErrors = {} let validationSchemas = [ - { - apiKey: string() - .required("Please enter your API key.") - .test("valid-apikey", "This API key is invalid", validateApiKey), - }, - { - applicationName: string().required("Your application must have a name."), - }, - { - email: string() - .email() - .required("Your application needs a first user."), - password: string().required( - "Please enter a password for your first user." - ), - roleId: string().required("You need to select a role for your user."), - }, + apiValidation, + infoValidation, + userValidation, ] function buildStep(component) { @@ -82,9 +87,12 @@ let hostingInfo = await hostingStore.actions.fetch() // re-init the steps based on whether self hosting or cloud hosted if (hostingInfo.type === "self") { + isApiKeyValid = true steps = [buildStep(Info), buildStep(User)] - } else { - steps = [buildStep(API), buildStep(Info), buildStep(User)] + validationSchemas = [ + infoValidation, + userValidation, + ] } }) diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index a13ab7ac41..52e12d92e7 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -19,8 +19,8 @@ const env = require("../../../environment") function objectStoreUrl() { if (env.SELF_HOSTED) { - // TODO: need a better way to handle this, probably reverse proxy - return `${env.HOSTING_URL}:${env.PROXY_PORT}/app-assets/assets` + // can use a relative url for this as all goes through the proxy (this is hosted in minio) + return `/app-assets/assets` } else { return "https://cdn.app.budi.live/assets" } @@ -157,7 +157,7 @@ exports.serveApp = async function(ctx) { title: appInfo.name, production: env.CLOUD, appId: ctx.params.appId, - objectStoreUrl: objectStoreUrl(ctx.params.appId), + objectStoreUrl: objectStoreUrl(), }) const template = handlebars.compile( diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 9d1bccf7c1..8e129913fd 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -37,8 +37,6 @@ module.exports = { DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, // self hosting features - HOSTING_URL: process.env.HOSTING_URL, - PROXY_PORT: process.env.PROXY_PORT, LOGO_URL: process.env.LOGO_URL, _set(key, value) { process.env[key] = value From 63c092799fbd9e64885d361c3e2f59fe164193d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 7 Jan 2021 15:39:49 +0000 Subject: [PATCH 39/48] Linting. --- .../src/components/start/CreateAppModal.svelte | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index 84d1ef7fa3..1ca192a6c9 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -58,20 +58,14 @@ email: string() .email() .required("Your application needs a first user."), - password: string().required( - "Please enter a password for your first user." - ), + password: string().required("Please enter a password for your first user."), roleId: string().required("You need to select a role for your user."), } let submitting = false let errors = {} let validationErrors = {} - let validationSchemas = [ - apiValidation, - infoValidation, - userValidation, - ] + let validationSchemas = [apiValidation, infoValidation, userValidation] function buildStep(component) { return { @@ -89,10 +83,7 @@ if (hostingInfo.type === "self") { isApiKeyValid = true steps = [buildStep(Info), buildStep(User)] - validationSchemas = [ - infoValidation, - userValidation, - ] + validationSchemas = [infoValidation, userValidation] } }) From c82d2b209b0c8a37c77fb09843008f5d05cec67b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 8 Jan 2021 10:36:41 +0000 Subject: [PATCH 40/48] Removing swap file. --- hosting/scripts/.release-to-docker-hub.sh.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hosting/scripts/.release-to-docker-hub.sh.swp diff --git a/hosting/scripts/.release-to-docker-hub.sh.swp b/hosting/scripts/.release-to-docker-hub.sh.swp deleted file mode 100644 index e7830335a2abae303fc88182df1d5856eaa6eef0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI&y-ve05C`xp8v`GLktL<9PD&Z57}*#)wv1Ii+$L&TTd`A^U}fZicmNh2gM|?x zW-c^QDYV^-{*g{@oU?uYNg1j*X!fo;PTOq~U;9MQZ=e11>k~Ta5RH?O6p@@3kJep9 z9;b<3lG)r(FV*ex?7M10FUX@L@Qs}Rh?TL)WaybNOSCm!CR51?!YUD|a3nJk=YebD z Date: Fri, 8 Jan 2021 10:46:40 +0000 Subject: [PATCH 41/48] Removing envoy admin feature (pointless, can add if needed, adds security risk). --- hosting/envoy.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 4e0e233430..b355dd804b 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -1,8 +1,3 @@ -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: { address: 0.0.0.0, port_value: 9901 } - static_resources: listeners: - name: main_listener From b4a3f2282c40524f8857f4af1177f535f2e348ed Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 8 Jan 2021 10:53:20 +0000 Subject: [PATCH 42/48] Changing scripts directory around a bit for hosting, make it clear shell scripts are for unix/linux. --- .../scripts/{ => linux}/install-docker-compose.sh | 0 hosting/scripts/linux/install-docker.sh | 14 ++++++++++++++ .../scripts/{ => linux}/release-to-docker-hub.sh | 0 3 files changed, 14 insertions(+) rename hosting/scripts/{ => linux}/install-docker-compose.sh (100%) create mode 100755 hosting/scripts/linux/install-docker.sh rename hosting/scripts/{ => linux}/release-to-docker-hub.sh (100%) diff --git a/hosting/scripts/install-docker-compose.sh b/hosting/scripts/linux/install-docker-compose.sh similarity index 100% rename from hosting/scripts/install-docker-compose.sh rename to hosting/scripts/linux/install-docker-compose.sh diff --git a/hosting/scripts/linux/install-docker.sh b/hosting/scripts/linux/install-docker.sh new file mode 100755 index 0000000000..dccf688de1 --- /dev/null +++ b/hosting/scripts/linux/install-docker.sh @@ -0,0 +1,14 @@ +#!/bin/bash +sudo apt-get install \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io diff --git a/hosting/scripts/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh similarity index 100% rename from hosting/scripts/release-to-docker-hub.sh rename to hosting/scripts/linux/release-to-docker-hub.sh From 711e44a3b8bc1689a3ad14307585b72f0107b841 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 8 Jan 2021 11:59:42 +0000 Subject: [PATCH 43/48] Updating envoy to handle API request, proxy to app-service, doesn't have to come from /app/api. --- hosting/envoy.yaml | 4 ++++ hosting/scripts/linux/install-docker.sh | 18 +++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index b355dd804b..3fe8454305 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -20,6 +20,10 @@ static_resources: route: cluster: app-service prefix_rewrite: "/" + + # special case for when API requests are made, can just forward, not to minio + - match: { prefix: "/api/" } + cluster: app-service - match: { prefix: "/worker/" } route: diff --git a/hosting/scripts/linux/install-docker.sh b/hosting/scripts/linux/install-docker.sh index dccf688de1..a71809c31f 100755 --- a/hosting/scripts/linux/install-docker.sh +++ b/hosting/scripts/linux/install-docker.sh @@ -1,14 +1,6 @@ #!/bin/bash -sudo apt-get install \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - software-properties-common -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" -sudo apt-get update -sudo apt-get install docker-ce docker-ce-cli containerd.io +echo "**** WARNING - not for production environments ****" +# warning this is a convience script, for production installations install docker +# properly for your environment! +curl -fsSL https://get.docker.com -o get-docker.sh +sudo sh get-docker.sh From ae5d970a92b737d209ef294f6e5393e64c60b262 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 8 Jan 2021 12:10:21 +0000 Subject: [PATCH 44/48] Missing route prop. --- hosting/envoy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 3fe8454305..3c816cb1ca 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -23,6 +23,7 @@ static_resources: # special case for when API requests are made, can just forward, not to minio - match: { prefix: "/api/" } + route: cluster: app-service - match: { prefix: "/worker/" } From e94cdf1f8e2d6ae7ac03de4386d667454eb04c8f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 11 Jan 2021 09:38:40 +0000 Subject: [PATCH 45/48] Remove hostname check when creating API endpoint URLs now that they are proxied --- packages/client/src/api/api.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js index 99b085b2fb..a875a61299 100644 --- a/packages/client/src/api/api.js +++ b/packages/client/src/api/api.js @@ -3,14 +3,6 @@ */ let cache = {} -/** - * Makes a fully formatted URL based on the SDK configuration. - */ -const makeFullURL = path => { - const isProxy = window.location.pathname.startsWith("/app/") - return `${isProxy ? "/app/" : "/"}${path}`.replace("//", "/") -} - /** * Handler for API errors. */ @@ -81,8 +73,7 @@ const makeCachedApiCall = async params => { */ const requestApiCall = method => async params => { const { url, cache = false } = params - const fullURL = makeFullURL(url) - const enrichedParams = { ...params, method, url: fullURL } + const enrichedParams = { ...params, method, url } return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams) } From ef0787917adc506bcfea90994f0e5c6305eaba20 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 11 Jan 2021 10:01:02 +0000 Subject: [PATCH 46/48] Fixing removal of full path function (adding back original functionality). --- packages/client/src/api/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.js index a875a61299..1e73baf602 100644 --- a/packages/client/src/api/api.js +++ b/packages/client/src/api/api.js @@ -73,7 +73,8 @@ const makeCachedApiCall = async params => { */ const requestApiCall = method => async params => { const { url, cache = false } = params - const enrichedParams = { ...params, method, url } + const fixedUrl = `/${url}`.replace("//", "/") + const enrichedParams = { ...params, method, fixedUrl } return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams) } From 3ec01a6fe5accee47bad23ef6bc01fa1a5becde7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 11 Jan 2021 11:24:53 +0000 Subject: [PATCH 47/48] Adding docker image release to release GH action. --- .github/workflows/release.yml | 9 +++++++++ hosting/scripts/linux/release-to-docker-hub.sh | 2 +- package.json | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 817cd17652..10beaf4fe6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,3 +61,12 @@ jobs: # macOS notarization API key API_KEY_ID: ${{ secrets.api_key_id }} API_KEY_ISSUER_ID: ${{ secrets.api_key_issuer_id }} + + - name: Build/release Docker images + uses: actions/checkout@v2 + env: + DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} + run: | + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + run: yarn build:docker diff --git a/hosting/scripts/linux/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh index bbe2d773f9..b1921916e4 100755 --- a/hosting/scripts/linux/release-to-docker-hub.sh +++ b/hosting/scripts/linux/release-to-docker-hub.sh @@ -1,5 +1,5 @@ #!/bin/bash -pushd ../build +pushd ../../build docker-compose build --force app-service docker-compose build --force worker-service docker tag build_app-service budibase/budibase-apps:latest diff --git a/package.json b/package.json index d58e36517d..fd601dd099 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ "lint:fix": "eslint --fix packages", "format": "prettier --write \"{,!(node_modules)/**/}*.{js,jsx,svelte}\"", "test:e2e": "lerna run cy:test", - "test:e2e:ci": "lerna run cy:ci" + "test:e2e:ci": "lerna run cy:ci", + "build:docker": "cd hosting/scripts/linux/ && ./release-to-docker-hub.sh && cd -" }, "dependencies": { "@fortawesome/fontawesome": "^1.1.8" From bf50f9af89fac5d92fb77c79acba443292f1c2e6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 11 Jan 2021 11:35:44 +0000 Subject: [PATCH 48/48] Making sure docker image build only runs on linux and fixing syntax error. --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 10beaf4fe6..68e71f4cf7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,10 +63,10 @@ jobs: API_KEY_ISSUER_ID: ${{ secrets.api_key_issuer_id }} - name: Build/release Docker images - uses: actions/checkout@v2 + # only run the docker image build on linux, easiest way + if: startsWith(matrix.os, 'ubuntu') env: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD run: yarn build:docker