From 7c4ec2f23ad2d00ac17ba573c3a09696bd0496dc Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 27 May 2021 11:11:44 +0100 Subject: [PATCH 1/5] execute bash commands in automations --- packages/server/src/automations/actions.js | 3 + packages/server/src/automations/steps/bash.js | 56 +++++++++++++++++++ .../src/automations/steps/executeScript.js | 2 +- 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/automations/steps/bash.js diff --git a/packages/server/src/automations/actions.js b/packages/server/src/automations/actions.js index ad102e7b67..762ce677a7 100644 --- a/packages/server/src/automations/actions.js +++ b/packages/server/src/automations/actions.js @@ -4,6 +4,7 @@ const createRow = require("./steps/createRow") const updateRow = require("./steps/updateRow") const deleteRow = require("./steps/deleteRow") const executeScript = require("./steps/executeScript") +const bash = require("./steps/bash") const executeQuery = require("./steps/executeQuery") const outgoingWebhook = require("./steps/outgoingWebhook") const env = require("../environment") @@ -21,6 +22,7 @@ const BUILTIN_ACTIONS = { DELETE_ROW: deleteRow.run, OUTGOING_WEBHOOK: outgoingWebhook.run, EXECUTE_SCRIPT: executeScript.run, + EXECUTE_BASH: bash.run, EXECUTE_QUERY: executeQuery.run, } const BUILTIN_DEFINITIONS = { @@ -32,6 +34,7 @@ const BUILTIN_DEFINITIONS = { OUTGOING_WEBHOOK: outgoingWebhook.definition, EXECUTE_SCRIPT: executeScript.definition, EXECUTE_QUERY: executeQuery.definition, + EXECUTE_BASH: bash.definition, } let MANIFEST = null diff --git a/packages/server/src/automations/steps/bash.js b/packages/server/src/automations/steps/bash.js new file mode 100644 index 0000000000..e71c70368b --- /dev/null +++ b/packages/server/src/automations/steps/bash.js @@ -0,0 +1,56 @@ +const scriptController = require("../../api/controllers/script") +const { execSync } = require("child_process") +const { processStringSync } = require("@budibase/string-templates") + +module.exports.definition = { + name: "Bash Scripting", + tagline: "Execute a bash command", + icon: "ri-terminal-box-line", + description: "Run a bash script", + type: "ACTION", + stepId: "EXECUTE_BASH", + inputs: {}, + schema: { + inputs: { + properties: { + code: { + type: "string", + customType: "code", + title: "Code", + }, + }, + required: ["code"], + }, + outputs: { + properties: { + stdout: { + type: "string", + description: "Standard output of your bash command or script.", + }, + }, + }, + required: ["stdout"], + }, +} + +module.exports.run = async function ({ inputs, context }) { + if (inputs.code == null) { + return { + stdout: "Budibase bash automation failed: Invalid inputs", + } + } + + try { + const command = processStringSync(inputs.code, context) + const stdout = execSync(command, { timeout: 1000 }) + return { + stdout, + } + } catch (err) { + console.error(err) + return { + success: false, + response: err, + } + } +} diff --git a/packages/server/src/automations/steps/executeScript.js b/packages/server/src/automations/steps/executeScript.js index 5bea1ab151..33ffd3ee8e 100644 --- a/packages/server/src/automations/steps/executeScript.js +++ b/packages/server/src/automations/steps/executeScript.js @@ -1,7 +1,7 @@ const scriptController = require("../../api/controllers/script") module.exports.definition = { - name: "Scripting", + name: "JS Scripting", tagline: "Execute JavaScript Code", icon: "ri-terminal-box-line", description: "Run a piece of JavaScript code in your automation", From 5be2182fef2676392bc8c71ce1c8e8dd9c9374a3 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 27 May 2021 11:47:02 +0100 Subject: [PATCH 2/5] piping errors to stderr when timeout exceeded --- packages/server/src/automations/steps/bash.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/server/src/automations/steps/bash.js b/packages/server/src/automations/steps/bash.js index e71c70368b..eb7ce605d2 100644 --- a/packages/server/src/automations/steps/bash.js +++ b/packages/server/src/automations/steps/bash.js @@ -42,7 +42,14 @@ module.exports.run = async function ({ inputs, context }) { try { const command = processStringSync(inputs.code, context) - const stdout = execSync(command, { timeout: 1000 }) + + let stdout + try { + stdout = execSync(command, { timeout: 500 }) + } catch (err) { + stdout = err.message + } + return { stdout, } From 0c90f9ffc5fda6f8193444c1ba74cd7f9fa39a2d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 28 May 2021 10:09:32 +0100 Subject: [PATCH 3/5] adding watchtower to docker config --- hosting/docker-compose.yaml | 19 ++++ hosting/envoy.yaml | 19 ++++ packages/auth/src/index.js | 3 +- packages/auth/src/middleware/auditLog.js | 10 ++ packages/auth/src/middleware/index.js | 2 + .../src/pages/builder/portal/_layout.svelte | 4 + .../portal/settings/organisation.svelte | 9 +- .../builder/portal/settings/update.svelte | 70 +++++++++++++ packages/server/src/api/index.js | 3 +- packages/server/src/app.js | 18 ++-- packages/server/src/automations/steps/bash.js | 1 - packages/server/src/utilities/redis.js | 4 +- packages/worker/package.json | 1 + .../worker/src/api/controllers/admin/debug.js | 4 + .../src/api/controllers/admin/updates.js | 1 + packages/worker/src/api/index.js | 3 +- packages/worker/src/api/routes/admin/debug.js | 9 ++ .../worker/src/api/routes/admin/updates.js | 9 ++ packages/worker/src/api/routes/index.js | 2 + packages/worker/src/utilities/redis.js | 4 +- packages/worker/yarn.lock | 97 ++++++++++++++++++- 21 files changed, 267 insertions(+), 25 deletions(-) create mode 100644 packages/auth/src/middleware/auditLog.js create mode 100644 packages/builder/src/pages/builder/portal/settings/update.svelte create mode 100644 packages/worker/src/api/controllers/admin/debug.js create mode 100644 packages/worker/src/api/controllers/admin/updates.js create mode 100644 packages/worker/src/api/routes/admin/debug.js create mode 100644 packages/worker/src/api/routes/admin/updates.js diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 0cd7bc92bf..c672f911a2 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -24,6 +24,8 @@ services: ENABLE_ANALYTICS: "true" REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} + volumes: + - ./logs:/logs depends_on: - worker-service @@ -46,6 +48,8 @@ services: INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} + volumes: + - ./logs:/logs depends_on: - redis-service - minio-service @@ -109,6 +113,21 @@ services: - "${REDIS_PORT}:6379" volumes: - redis_data:/data + + watchtower-service: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --debug --http-api-update budibase/apps budibase/worker + environment: + - WATCHTOWER_HTTP_API=true + - WATCHTOWER_HTTP_API_TOKEN=budibase + - WATCHTOWER_CLEANUP=true + labels: + - "com.centurylinklabs.watchtower.enable=false" + ports: + - 6666:8080 + volumes: couchdb3_data: diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 463b32ab60..6e143ac843 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -38,6 +38,11 @@ static_resources: route: cluster: worker-service + - match: { prefix: "/v1/update" } + route: + cluster: watchtower-service + + - match: { path: "/" } route: cluster: app-service @@ -123,3 +128,17 @@ static_resources: address: couchdb-service port_value: 5984 + - name: watchtower-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: watchtower-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: watchtower-service + port_value: 6666 + diff --git a/packages/auth/src/index.js b/packages/auth/src/index.js index d2ba86a524..9582d6ffd6 100644 --- a/packages/auth/src/index.js +++ b/packages/auth/src/index.js @@ -2,7 +2,7 @@ const passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy const { StaticDatabases } = require("./db/utils") -const { jwt, local, authenticated, google } = require("./middleware") +const { jwt, local, authenticated, google, auditLog } = require("./middleware") const { setDB, getDB } = require("./db") // Strategies @@ -45,6 +45,7 @@ module.exports = { passport, google, jwt: require("jsonwebtoken"), + auditLog, }, StaticDatabases, constants: require("./constants"), diff --git a/packages/auth/src/middleware/auditLog.js b/packages/auth/src/middleware/auditLog.js new file mode 100644 index 0000000000..919d7257ba --- /dev/null +++ b/packages/auth/src/middleware/auditLog.js @@ -0,0 +1,10 @@ +module.exports = async (ctx, next) => { + ctx.log.info({ + userId: ctx.user && ctx.user._id, + ip: ctx.ip, + url: ctx.originalUrl, + origin: ctx.origin, + method: ctx.method, + }) + return next() +} diff --git a/packages/auth/src/middleware/index.js b/packages/auth/src/middleware/index.js index 519233eda4..2a249ce0f9 100644 --- a/packages/auth/src/middleware/index.js +++ b/packages/auth/src/middleware/index.js @@ -2,10 +2,12 @@ const jwt = require("./passport/jwt") const local = require("./passport/local") const google = require("./passport/google") const authenticated = require("./authenticated") +const auditLog = require("./auditLog") module.exports = { google, jwt, local, authenticated, + auditLog, } diff --git a/packages/builder/src/pages/builder/portal/_layout.svelte b/packages/builder/src/pages/builder/portal/_layout.svelte index ba5da18d98..3d9ae71fff 100644 --- a/packages/builder/src/pages/builder/portal/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/_layout.svelte @@ -43,6 +43,10 @@ title: "Theming", href: "/builder/portal/settings/theming", }, + { + title: "Update", + href: "/builder/portal/settings/update", + }, ]) } else { menu = menu.concat([ diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index 046f55615b..9cb448fb17 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -12,7 +12,7 @@ notifications, } from "@budibase/bbui" import { auth, organisation } from "stores/portal" - import { post } from "builderStore/api" + import { post, get } from "builderStore/api" import analytics from "analytics" import { writable } from "svelte/store" import { redirect } from "@roxi/routify" @@ -70,6 +70,7 @@ loading = false } + {#if $auth.isAdmin} @@ -130,10 +131,10 @@ +
+ +
-
- -
{/if} diff --git a/packages/builder/src/pages/builder/portal/settings/update.svelte b/packages/builder/src/pages/builder/portal/settings/update.svelte new file mode 100644 index 0000000000..a0eaa987cd --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/update.svelte @@ -0,0 +1,70 @@ + + +{#if $auth.isAdmin} + + + Update + + Keep your budibase installation up to date to take advantage of the latest features, security updates and much more. + + + +
+
+ +
+
+ +{/if} + + diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 332b917a76..6c4188a5dc 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -1,5 +1,5 @@ const Router = require("@koa/router") -const { buildAuthMiddleware } = require("@budibase/auth").auth +const { buildAuthMiddleware, auditLog } = require("@budibase/auth").auth const currentApp = require("../middleware/currentapp") const compress = require("koa-compress") const zlib = require("zlib") @@ -37,6 +37,7 @@ router }) ) .use(currentApp) + .use(auditLog) // error handling middleware router.use(async (ctx, next) => { diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 50df056b2a..5772eefad3 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -5,7 +5,7 @@ require("@budibase/auth").init(CouchDB) const Koa = require("koa") const destroyable = require("server-destroy") const koaBody = require("koa-body") -const logger = require("koa-pino-logger") +const pino = require("koa-pino-logger") const http = require("http") const api = require("./api") const eventEmitter = require("./events") @@ -28,14 +28,14 @@ app.use( }) ) -app.use( - logger({ - prettyPrint: { - levelFirst: true, - }, - level: env.LOG_LEVEL || "error", - }) -) +let logger = pino({ + prettyPrint: { + levelFirst: true, + }, + level: env.LOG_LEVEL || "error", +}) + +app.use(logger) if (!env.isTest()) { const bullApp = bullboard.init() diff --git a/packages/server/src/automations/steps/bash.js b/packages/server/src/automations/steps/bash.js index eb7ce605d2..76d6713c5b 100644 --- a/packages/server/src/automations/steps/bash.js +++ b/packages/server/src/automations/steps/bash.js @@ -1,4 +1,3 @@ -const scriptController = require("../../api/controllers/script") const { execSync } = require("child_process") const { processStringSync } = require("@budibase/string-templates") diff --git a/packages/server/src/utilities/redis.js b/packages/server/src/utilities/redis.js index ae18b82e02..e1fa632003 100644 --- a/packages/server/src/utilities/redis.js +++ b/packages/server/src/utilities/redis.js @@ -12,8 +12,8 @@ exports.init = async () => { } exports.shutdown = async () => { - await devAppClient.finish() - await debounceClient.finish() + if (devAppClient) await devAppClient.finish() + if (debounceClient) await debounceClient.finish() } exports.doesUserHaveLock = async (devAppId, user) => { diff --git a/packages/worker/package.json b/packages/worker/package.json index 7a51c8ac49..b234990966 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -26,6 +26,7 @@ "@koa/router": "^8.0.0", "aws-sdk": "^2.811.0", "bcryptjs": "^2.4.3", + "dockerode": "^3.3.0", "dotenv": "^8.2.0", "got": "^11.8.1", "joi": "^17.4.0", diff --git a/packages/worker/src/api/controllers/admin/debug.js b/packages/worker/src/api/controllers/admin/debug.js new file mode 100644 index 0000000000..a96bf00c83 --- /dev/null +++ b/packages/worker/src/api/controllers/admin/debug.js @@ -0,0 +1,4 @@ +exports.fetchDebugLogs = async ctx => { + // read them from file + // serve them +} diff --git a/packages/worker/src/api/controllers/admin/updates.js b/packages/worker/src/api/controllers/admin/updates.js new file mode 100644 index 0000000000..a41beaf50c --- /dev/null +++ b/packages/worker/src/api/controllers/admin/updates.js @@ -0,0 +1 @@ +exports.updateSystem = async ctx => {} diff --git a/packages/worker/src/api/index.js b/packages/worker/src/api/index.js index 1b2586a6b6..d456778d54 100644 --- a/packages/worker/src/api/index.js +++ b/packages/worker/src/api/index.js @@ -2,7 +2,7 @@ const Router = require("@koa/router") const compress = require("koa-compress") const zlib = require("zlib") const { routes } = require("./routes") -const { buildAuthMiddleware } = require("@budibase/auth").auth +const { buildAuthMiddleware, auditLog } = require("@budibase/auth").auth const PUBLIC_ENDPOINTS = [ { @@ -62,6 +62,7 @@ router } return next() }) + .use(auditLog) // error handling middleware router.use(async (ctx, next) => { diff --git a/packages/worker/src/api/routes/admin/debug.js b/packages/worker/src/api/routes/admin/debug.js new file mode 100644 index 0000000000..ee8e490d0e --- /dev/null +++ b/packages/worker/src/api/routes/admin/debug.js @@ -0,0 +1,9 @@ +const Router = require("@koa/router") +const controller = require("../../controllers/admin/debug") +const adminOnly = require("../../../middleware/adminOnly") + +const router = Router() + +router.get("/api/admin/debug/logs", adminOnly, controller.fetchDebugLogs) + +module.exports = router diff --git a/packages/worker/src/api/routes/admin/updates.js b/packages/worker/src/api/routes/admin/updates.js new file mode 100644 index 0000000000..a89ce8b68a --- /dev/null +++ b/packages/worker/src/api/routes/admin/updates.js @@ -0,0 +1,9 @@ +const Router = require("@koa/router") +const controller = require("../../controllers/admin/updates") +const adminOnly = require("../../../middleware/adminOnly") + +const router = Router() + +router.get("/api/admin/update", adminOnly, controller.updateSystem) + +module.exports = router diff --git a/packages/worker/src/api/routes/index.js b/packages/worker/src/api/routes/index.js index 8b232f7b7c..ae83a714db 100644 --- a/packages/worker/src/api/routes/index.js +++ b/packages/worker/src/api/routes/index.js @@ -5,6 +5,7 @@ const templateRoutes = require("./admin/templates") const emailRoutes = require("./admin/email") const authRoutes = require("./admin/auth") const roleRoutes = require("./admin/roles") +const updatesRoutes = require("./admin/updates") const appRoutes = require("./app") exports.routes = [ @@ -16,4 +17,5 @@ exports.routes = [ templateRoutes, emailRoutes, roleRoutes, + updatesRoutes, ] diff --git a/packages/worker/src/utilities/redis.js b/packages/worker/src/utilities/redis.js index 28162a0c14..0701e500dd 100644 --- a/packages/worker/src/utilities/redis.js +++ b/packages/worker/src/utilities/redis.js @@ -51,8 +51,8 @@ exports.init = async () => { * make sure redis connection is closed. */ exports.shutdown = async () => { - await pwResetClient.finish() - await invitationClient.finish() + if (pwResetClient) await pwResetClient.finish() + if (invitationClient) await invitationClient.finish() } /** diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index ce1185f832..9f17e18bc9 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -910,7 +910,7 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= -asn1@~0.2.3: +asn1@~0.2.0, asn1@~0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== @@ -1076,7 +1076,7 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" -bcrypt-pbkdf@^1.0.0: +bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= @@ -1093,6 +1093,15 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bl@^4.0.3: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" + integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== + dependencies: + buffer "^5.5.0" + inherits "^2.0.4" + readable-stream "^3.4.0" + boxen@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" @@ -1337,6 +1346,11 @@ chokidar@^3.2.2: optionalDependencies: fsevents "~2.3.1" +chownr@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" + integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + ci-info@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" @@ -1773,6 +1787,24 @@ diff-sequences@^26.6.2: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.6.2.tgz#48ba99157de1923412eed41db6b6d4aa9ca7c0b1" integrity sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q== +docker-modem@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-3.0.0.tgz#cb912ad8daed42f858269fb3be6944df281ec12d" + integrity sha512-WwFajJ8I5geZ/dDZ5FDMDA6TBkWa76xWwGIGw8uzUjNUGCN0to83wJ8Oi1AxrJTC0JBn+7fvIxUctnawtlwXeg== + dependencies: + debug "^4.1.1" + readable-stream "^3.5.0" + split-ca "^1.0.1" + ssh2 "^0.8.7" + +dockerode@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-3.3.0.tgz#bedaf48ef9fa9124275a54a9881a92374c51008e" + integrity sha512-St08lfOjpYCOXEM8XA0VLu3B3hRjtddODphNW5GFoA0AS3JHgoPQKOz0Qmdzg3P+hUPxhb02g1o1Cu1G+U3lRg== + dependencies: + docker-modem "^3.0.0" + tar-fs "~2.0.1" + domexception@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" @@ -1857,7 +1889,7 @@ encoding-down@^6.3.0: level-codec "^9.0.0" level-errors "^2.0.0" -end-of-stream@^1.1.0: +end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== @@ -2238,6 +2270,11 @@ fresh@~0.5.2: resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -3998,6 +4035,11 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" +mkdirp-classic@^0.5.2: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + mkdirp@^0.5.0: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" @@ -4829,7 +4871,7 @@ readable-stream@1.1.14: isarray "0.0.1" string_decoder "~0.10.x" -"readable-stream@2 || 3", readable-stream@^3.0.0, readable-stream@^3.4.0, readable-stream@^3.6.0: +"readable-stream@2 || 3", readable-stream@^3.0.0, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -5326,6 +5368,11 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.7.tgz#e9c18a410e5ed7e12442a549fbd8afa767038d65" integrity sha512-U+MTEOO0AiDzxwFvoa4JVnMV6mZlJKk2sBLt90s7G0Gd0Mlknc7kxEn3nuDPNZRta7O2uy8oLcZLVT+4sqNZHQ== +split-ca@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + integrity sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY= + split-string@^3.0.1, split-string@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" @@ -5345,6 +5392,22 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +ssh2-streams@~0.4.10: + version "0.4.10" + resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34" + integrity sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ== + dependencies: + asn1 "~0.2.0" + bcrypt-pbkdf "^1.0.2" + streamsearch "~0.1.2" + +ssh2@^0.8.7: + version "0.8.9" + resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.8.9.tgz#54da3a6c4ba3daf0d8477a538a481326091815f3" + integrity sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw== + dependencies: + ssh2-streams "~0.4.10" + sshpk@^1.7.0: version "1.16.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" @@ -5390,6 +5453,11 @@ stealthy-require@^1.1.1: resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks= +streamsearch@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a" + integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo= + string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -5529,6 +5597,27 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== +tar-fs@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" + integrity sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.0.0" + +tar-stream@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + term-size@^2.1.0: version "2.2.1" resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.1.tgz#2a6a54840432c2fb6320fea0f415531e90189f54" From aacaca3d4e13d114951a9b36cf3dcc8d87daf650 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Jun 2021 20:17:06 +0100 Subject: [PATCH 4/5] update functionality complete --- hosting/docker-compose.dev.yaml | 14 ++++++++++++ hosting/docker-compose.yaml | 2 +- hosting/envoy.dev.yaml.hbs | 3 +-- hosting/envoy.yaml | 2 +- .../src/pages/builder/portal/_layout.svelte | 2 +- .../portal/settings/organisation.svelte | 1 - .../builder/portal/settings/update.svelte | 22 +++++++++---------- packages/server/src/app.js | 16 +++++++------- .../worker/src/api/controllers/admin/debug.js | 4 ---- .../src/api/controllers/admin/updates.js | 1 - packages/worker/src/api/routes/admin/debug.js | 9 -------- packages/worker/src/api/routes/index.js | 2 -- 12 files changed, 37 insertions(+), 41 deletions(-) delete mode 100644 packages/worker/src/api/controllers/admin/debug.js delete mode 100644 packages/worker/src/api/controllers/admin/updates.js delete mode 100644 packages/worker/src/api/routes/admin/debug.js diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 3b99ef796c..1eddbf9271 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -64,6 +64,20 @@ services: - "${REDIS_PORT}:6379" volumes: - redis_data:/data + + watchtower-service: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --debug --http-api-update budibase/apps budibase/worker + environment: + - WATCHTOWER_HTTP_API=true + - WATCHTOWER_HTTP_API_TOKEN=budibase + - WATCHTOWER_CLEANUP=true + labels: + - "com.centurylinklabs.watchtower.enable=false" + ports: + - 6161:8080 volumes: couchdb3_data: diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index c672f911a2..b03b433cef 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -126,7 +126,7 @@ services: labels: - "com.centurylinklabs.watchtower.enable=false" ports: - - 6666:8080 + - 6161:8080 volumes: diff --git a/hosting/envoy.dev.yaml.hbs b/hosting/envoy.dev.yaml.hbs index 054ceb9d3f..6a1d758895 100644 --- a/hosting/envoy.dev.yaml.hbs +++ b/hosting/envoy.dev.yaml.hbs @@ -132,5 +132,4 @@ static_resources: address: socket_address: address: {{ address }} - port_value: 4002 - + port_value: 4002 \ No newline at end of file diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 6e143ac843..601506c622 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -140,5 +140,5 @@ static_resources: address: socket_address: address: watchtower-service - port_value: 6666 + port_value: 6161 diff --git a/packages/builder/src/pages/builder/portal/_layout.svelte b/packages/builder/src/pages/builder/portal/_layout.svelte index e6f7da4371..4bc0b46167 100644 --- a/packages/builder/src/pages/builder/portal/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/_layout.svelte @@ -44,7 +44,7 @@ href: "/builder/portal/settings/theming", }, { - title: "Update", + title: "Updates", href: "/builder/portal/settings/update", }, ]) diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index 9cb448fb17..f8f4c8a7dd 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -70,7 +70,6 @@ loading = false } - {#if $auth.isAdmin} diff --git a/packages/builder/src/pages/builder/portal/settings/update.svelte b/packages/builder/src/pages/builder/portal/settings/update.svelte index a0eaa987cd..5835f9d0dc 100644 --- a/packages/builder/src/pages/builder/portal/settings/update.svelte +++ b/packages/builder/src/pages/builder/portal/settings/update.svelte @@ -27,17 +27,16 @@ async function updateBudibase() { try { notifications.info("Updating budibase..") - // const response = await fetch("http://localhost:6666/v1/update", { - // headers: { - // Authorization: "Bearer budibase" - // } - // }) + const response = await fetch("/v1/update", { + headers: { + Authorization: "Bearer budibase", + }, + }) notifications.success("Your budibase installation is up to date.") } catch (err) { notifications.error(`Error installing budibase update ${err}`) } } - {#if $auth.isAdmin} @@ -45,15 +44,16 @@ Update - Keep your budibase installation up to date to take advantage of the latest features, security updates and much more. + Keep your budibase installation up to date to take advantage of the + latest features, security updates and much more. -
-
- -
+
+
+
+
{/if} diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 5772eefad3..d7fc248bba 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -28,14 +28,14 @@ app.use( }) ) -let logger = pino({ - prettyPrint: { - levelFirst: true, - }, - level: env.LOG_LEVEL || "error", -}) - -app.use(logger) +app.use( + pino({ + prettyPrint: { + levelFirst: true, + }, + level: env.LOG_LEVEL || "error", + }) +) if (!env.isTest()) { const bullApp = bullboard.init() diff --git a/packages/worker/src/api/controllers/admin/debug.js b/packages/worker/src/api/controllers/admin/debug.js deleted file mode 100644 index a96bf00c83..0000000000 --- a/packages/worker/src/api/controllers/admin/debug.js +++ /dev/null @@ -1,4 +0,0 @@ -exports.fetchDebugLogs = async ctx => { - // read them from file - // serve them -} diff --git a/packages/worker/src/api/controllers/admin/updates.js b/packages/worker/src/api/controllers/admin/updates.js deleted file mode 100644 index a41beaf50c..0000000000 --- a/packages/worker/src/api/controllers/admin/updates.js +++ /dev/null @@ -1 +0,0 @@ -exports.updateSystem = async ctx => {} diff --git a/packages/worker/src/api/routes/admin/debug.js b/packages/worker/src/api/routes/admin/debug.js deleted file mode 100644 index ee8e490d0e..0000000000 --- a/packages/worker/src/api/routes/admin/debug.js +++ /dev/null @@ -1,9 +0,0 @@ -const Router = require("@koa/router") -const controller = require("../../controllers/admin/debug") -const adminOnly = require("../../../middleware/adminOnly") - -const router = Router() - -router.get("/api/admin/debug/logs", adminOnly, controller.fetchDebugLogs) - -module.exports = router diff --git a/packages/worker/src/api/routes/index.js b/packages/worker/src/api/routes/index.js index ae83a714db..8b232f7b7c 100644 --- a/packages/worker/src/api/routes/index.js +++ b/packages/worker/src/api/routes/index.js @@ -5,7 +5,6 @@ const templateRoutes = require("./admin/templates") const emailRoutes = require("./admin/email") const authRoutes = require("./admin/auth") const roleRoutes = require("./admin/roles") -const updatesRoutes = require("./admin/updates") const appRoutes = require("./app") exports.routes = [ @@ -17,5 +16,4 @@ exports.routes = [ templateRoutes, emailRoutes, roleRoutes, - updatesRoutes, ] From e927f44c12b9c220c4c1c983592b7e5d05a3cdfb Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Jun 2021 20:22:56 +0100 Subject: [PATCH 5/5] removing superfluous controllers --- hosting/docker-compose.dev.yaml | 14 -------------- hosting/envoy.dev.yaml.hbs | 2 +- hosting/envoy.yaml | 9 ++++----- packages/auth/src/middleware/auditLog.js | 8 +------- packages/worker/package.json | 1 - packages/worker/src/api/routes/admin/updates.js | 9 --------- 6 files changed, 6 insertions(+), 37 deletions(-) delete mode 100644 packages/worker/src/api/routes/admin/updates.js diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 1eddbf9271..3b99ef796c 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -64,20 +64,6 @@ services: - "${REDIS_PORT}:6379" volumes: - redis_data:/data - - watchtower-service: - image: containrrr/watchtower - volumes: - - /var/run/docker.sock:/var/run/docker.sock - command: --debug --http-api-update budibase/apps budibase/worker - environment: - - WATCHTOWER_HTTP_API=true - - WATCHTOWER_HTTP_API_TOKEN=budibase - - WATCHTOWER_CLEANUP=true - labels: - - "com.centurylinklabs.watchtower.enable=false" - ports: - - 6161:8080 volumes: couchdb3_data: diff --git a/hosting/envoy.dev.yaml.hbs b/hosting/envoy.dev.yaml.hbs index 6a1d758895..080311ee49 100644 --- a/hosting/envoy.dev.yaml.hbs +++ b/hosting/envoy.dev.yaml.hbs @@ -132,4 +132,4 @@ static_resources: address: socket_address: address: {{ address }} - port_value: 4002 \ No newline at end of file + port_value: 4002 diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 601506c622..20e912434a 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -21,6 +21,10 @@ static_resources: cluster: app-service prefix_rewrite: "/" + - match: { path: "/v1/update" } + route: + cluster: watchtower-service + - match: { prefix: "/builder/" } route: cluster: app-service @@ -38,11 +42,6 @@ static_resources: route: cluster: worker-service - - match: { prefix: "/v1/update" } - route: - cluster: watchtower-service - - - match: { path: "/" } route: cluster: app-service diff --git a/packages/auth/src/middleware/auditLog.js b/packages/auth/src/middleware/auditLog.js index 919d7257ba..c9063ae2e0 100644 --- a/packages/auth/src/middleware/auditLog.js +++ b/packages/auth/src/middleware/auditLog.js @@ -1,10 +1,4 @@ module.exports = async (ctx, next) => { - ctx.log.info({ - userId: ctx.user && ctx.user._id, - ip: ctx.ip, - url: ctx.originalUrl, - origin: ctx.origin, - method: ctx.method, - }) + // Placeholder for audit log middleware return next() } diff --git a/packages/worker/package.json b/packages/worker/package.json index fe289844f0..da59c55c59 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -26,7 +26,6 @@ "@koa/router": "^8.0.0", "aws-sdk": "^2.811.0", "bcryptjs": "^2.4.3", - "dockerode": "^3.3.0", "dotenv": "^8.2.0", "got": "^11.8.1", "joi": "^17.4.0", diff --git a/packages/worker/src/api/routes/admin/updates.js b/packages/worker/src/api/routes/admin/updates.js deleted file mode 100644 index a89ce8b68a..0000000000 --- a/packages/worker/src/api/routes/admin/updates.js +++ /dev/null @@ -1,9 +0,0 @@ -const Router = require("@koa/router") -const controller = require("../../controllers/admin/updates") -const adminOnly = require("../../../middleware/adminOnly") - -const router = Router() - -router.get("/api/admin/update", adminOnly, controller.updateSystem) - -module.exports = router