MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updating hosting to break up single file into lots of independent commands - this was an attempt to improve syntax. Adding in single image handling, as well as full support for watch directory.

This commit is contained in:
mike12345567 2022-09-28 19:11:22 +01:00
parent 3bb4b3ed3a
commit a464aceffb
12 changed files with 332 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/cli/src/constants.js
View File

@ -11,7 +11,6 @@ exports.CommandWords = {
exports.InitTypes = {
QUICK: "quick",
DIGITAL_OCEAN: "do",
SINGLE: "single",
}
exports.AnalyticsEvents = {

216
packages/cli/src/hosting/index.js
View File

@ -1,211 +1,11 @@
const Command = require("../structures/Command")
const { CommandWords, InitTypes, AnalyticsEvents } = require("../constants")
const { lookpath } = require("lookpath")
const { resolve } = require("path")
const {
downloadFile,
logErrorToFile,
success,
info,
error,
parseEnv,
} = require("../utils")
const { confirmation } = require("../questions")
const fs = require("fs")
const compose = require("docker-compose")
const makeFiles = require("./makeFiles")
const axios = require("axios")
const { captureEvent } = require("../events")
const yaml = require("yaml")
const BUDIBASE_SERVICES = ["app-service", "worker-service", "proxy-service"]
const ERROR_FILE = "docker-error.log"
const FILE_URLS = [
"https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
]
const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
async function downloadFiles() {
const promises = []
for (let url of FILE_URLS) {
const fileName = url.split("/").slice(-1)[0]
promises.push(downloadFile(url, `./${fileName}`))
}
await Promise.all(promises)
}
async function checkDockerConfigured() {
const error =
"docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
const docker = await lookpath("docker")
const compose = await lookpath("docker-compose")
if (!docker || !compose) {
throw error
}
}
function checkInitComplete() {
if (!fs.existsSync(makeFiles.filePath)) {
throw "Please run the hosting --init command before any other hosting command."
}
}
async function handleError(func) {
try {
await func()
} catch (err) {
if (err && err.err) {
logErrorToFile(ERROR_FILE, err.err)
}
throw `Failed to start - logs written to file: ${ERROR_FILE}`
}
}
async function init(type) {
const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
await checkDockerConfigured()
if (!isQuick) {
const shouldContinue = await confirmation(
"This will create multiple files in current directory, should continue?"
)
if (!shouldContinue) {
console.log("Stopping.")
return
}
}
captureEvent(AnalyticsEvents.SelfHostInit, {
type,
})
await downloadFiles()
const config = isQuick ? makeFiles.QUICK_CONFIG : {}
if (type === InitTypes.DIGITAL_OCEAN) {
try {
const output = await axios.get(DO_USER_DATA_URL)
const response = parseEnv(output.data)
for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
if (response[key]) {
config[value] = response[key]
}
}
} catch (err) {
// don't need to handle error, just don't do anything
}
}
await makeFiles.makeEnv(config)
}
async function start() {
await checkDockerConfigured()
checkInitComplete()
console.log(
info(
"Starting services, this may take a moment - first time this may take a few minutes to download images."
)
)
const port = makeFiles.getEnvProperty("MAIN_PORT")
await handleError(async () => {
// need to log as it makes it more clear
await compose.upAll({ cwd: "./", log: true })
})
console.log(
success(
`Services started, please go to http://localhost:${port} for next steps.`
)
)
}
async function status() {
await checkDockerConfigured()
checkInitComplete()
console.log(info("Budibase status"))
await handleError(async () => {
const response = await compose.ps()
console.log(response.out)
})
}
async function stop() {
await checkDockerConfigured()
checkInitComplete()
console.log(info("Stopping services, this may take a moment."))
await handleError(async () => {
await compose.stop()
})
console.log(success("Services have been stopped successfully."))
}
async function update() {
await checkDockerConfigured()
checkInitComplete()
if (await confirmation("Do you wish to update you docker-compose.yaml?")) {
await downloadFiles()
}
await handleError(async () => {
const status = await compose.ps()
const parts = status.out.split("\n")
const isUp = parts[2] && parts[2].indexOf("Up") !== -1
if (isUp) {
console.log(info("Stopping services, this may take a moment."))
await compose.stop()
}
console.log(info("Beginning update, this may take a few minutes."))
await compose.pullMany(BUDIBASE_SERVICES, { log: true })
if (isUp) {
console.log(success("Update complete, restarting services..."))
await start()
}
})
}
async function watchPlugins(pluginPath) {
const PLUGIN_PATH = "/plugins"
// get absolute path
pluginPath = resolve(pluginPath)
if (!fs.existsSync(pluginPath)) {
console.log(
error(
`The directory "${pluginPath}" does not exist, please create and then try again.`
)
)
return
}
const opts = ["docker-compose.yaml", "docker-compose.yml"]
let dockerFilePath = opts.find(name => fs.existsSync(name))
if (!dockerFilePath) {
console.log(error("Unable to locate docker-compose YAML."))
return
}
const dockerYaml = fs.readFileSync(dockerFilePath, "utf8")
const parsedYaml = yaml.parse(dockerYaml)
let service,
serviceList = Object.keys(parsedYaml.services)
if (parsedYaml.services["app-service"]) {
service = parsedYaml.services["app-service"]
} else if (serviceList.length === 1) {
service = parsedYaml.services[serviceList[0]]
}
if (!service) {
console.log(
error(
"Unable to locate service within compose file, is it a valid Budibase configuration?"
)
)
return
}
// set environment variable
service.environment["PLUGINS_DIR"] = PLUGIN_PATH
// add volumes to parsed yaml
if (!service.volumes) {
service.volumes = []
}
const found = service.volumes.find(vol => vol.includes(PLUGIN_PATH))
if (found) {
service.volumes.splice(service.volumes.indexOf(found), 1)
}
service.volumes.push(`${pluginPath}:${PLUGIN_PATH}`)
fs.writeFileSync(dockerFilePath, yaml.stringify(parsedYaml))
console.log(success("Docker compose configuration has been updated!"))
}
const { CommandWords } = require("../constants")
const { init } = require("./init")
const { start } = require("./start")
const { stop } = require("./stop")
const { status } = require("./status")
const { update } = require("./update")
const { watchPlugins } = require("./watch")
const command = new Command(`${CommandWords.HOSTING}`)
.addHelp("Controls self hosting on the Budibase platform.")
@ -239,6 +39,6 @@ const command = new Command(`${CommandWords.HOSTING}`)
"Add plugin directory watching to a Budibase install.",
watchPlugins
)
.addSubOption("--dev", "")
.addSubOption("--single", "Specify this with init to use the single image.")
exports.command = command

63
packages/cli/src/hosting/init.js Normal file
View File

@ -0,0 +1,63 @@
const { InitTypes, AnalyticsEvents } = require("../constants")
const { confirmation } = require("../questions")
const { captureEvent } = require("../events")
const makeFiles = require("./makeFiles")
const axios = require("axios")
const { parseEnv } = require("../utils")
const { checkDockerConfigured, downloadFiles } = require("./utils")
const { watchPlugins } = require("./watch")
const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
async function getInitConfig(type, isQuick) {
const config = isQuick ? makeFiles.QUICK_CONFIG : {}
if (type === InitTypes.DIGITAL_OCEAN) {
try {
const output = await axios.get(DO_USER_DATA_URL)
const response = parseEnv(output.data)
for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
if (response[key]) {
config[value] = response[key]
}
}
} catch (err) {
// don't need to handle error, just don't do anything
}
}
return config
}
exports.init = async opts => {
let type, isSingle, watchDir
if (typeof opts === "string") {
type = opts
} else {
type = opts.init
isSingle = opts.single
watchDir = opts.watchPluginDir
}
const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
await checkDockerConfigured()
if (!isQuick) {
const shouldContinue = await confirmation(
"This will create multiple files in current directory, should continue?"
)
if (!shouldContinue) {
console.log("Stopping.")
return
}
}
captureEvent(AnalyticsEvents.SelfHostInit, {
type,
})
const config = await getInitConfig(type, isQuick)
if (!isSingle) {
await downloadFiles()
await makeFiles.makeEnv(config)
} else {
await makeFiles.makeSingleCompose(config)
if (watchDir) {
await watchPlugins(watchDir)
}
}
}

20
packages/cli/src/hosting/makeFiles.js
View File

@ -4,6 +4,7 @@ const fs = require("fs")
const path = require("path")
const randomString = require("randomstring")
const yaml = require("yaml")
const { getAppService } = require("./utils")
const SINGLE_IMAGE = "budibase/budibase:latest"
const VOL_NAME = "budibase_data"
@ -16,12 +17,13 @@ function getSecrets() {
"MINIO_ACCESS_KEY",
"MINIO_SECRET_KEY",
"COUCH_DB_PASSWORD",
"COUCH_DB_USER",
"REDIS_PASSWORD",
"INTERNAL_API_KEY",
]
const obj = {}
secrets.forEach(secret => (obj[secret] = randomString.generate()))
// hard code to admin
obj["COUCH_DB_USER"] = "admin"
return obj
}
@ -70,10 +72,13 @@ function getEnv(port) {
].join("\n")
}
module.exports.filePath = ENV_PATH
exports.ENV_PATH = ENV_PATH
exports.COMPOSE_PATH = COMPOSE_PATH
module.exports.ConfigMap = {
MAIN_PORT: "port",
}
module.exports.QUICK_CONFIG = {
key: "budibase",
port: 10000,
@ -112,3 +117,14 @@ module.exports.getEnvProperty = property => {
}
return property.split("=")[1].split("\n")[0]
}
module.exports.getComposeProperty = property => {
const appService = getAppService(COMPOSE_PATH)
if (property === "port" && Array.isArray(appService.ports)) {
const port = appService.ports[0]
return port.split(":")[0]
} else if (appService.environment) {
return appService.environment[property]
}
return null
}

34
packages/cli/src/hosting/start.js Normal file
View File

@ -0,0 +1,34 @@
const {
checkDockerConfigured,
checkInitComplete,
handleError,
} = require("./utils")
const { info, success } = require("../utils")
const makeFiles = require("./makeFiles")
const compose = require("docker-compose")
const fs = require("fs")
exports.start = async () => {
await checkDockerConfigured()
checkInitComplete()
console.log(
info(
"Starting services, this may take a moment - first time this may take a few minutes to download images."
)
)
let port
if (fs.existsSync(makeFiles.ENV_PATH)) {
port = makeFiles.getEnvProperty("MAIN_PORT")
} else {
port = makeFiles.getComposeProperty("port")
}
await handleError(async () => {
// need to log as it makes it more clear
await compose.upAll({ cwd: "./", log: true })
})
console.log(
success(
`Services started, please go to http://localhost:${port} for next steps.`
)
)
}

17
packages/cli/src/hosting/status.js Normal file
View File

@ -0,0 +1,17 @@
const {
checkDockerConfigured,
checkInitComplete,
handleError,
} = require("./utils")
const { info } = require("../utils")
const compose = require("docker-compose")
exports.status = async () => {
await checkDockerConfigured()
checkInitComplete()
console.log(info("Budibase status"))
await handleError(async () => {
const response = await compose.ps()
console.log(response.out)
})
}

17
packages/cli/src/hosting/stop.js Normal file
View File

@ -0,0 +1,17 @@
const {
checkDockerConfigured,
checkInitComplete,
handleError,
} = require("./utils")
const { info, success } = require("../utils")
const compose = require("docker-compose")
exports.stop = async () => {
await checkDockerConfigured()
checkInitComplete()
console.log(info("Stopping services, this may take a moment."))
await handleError(async () => {
await compose.stop()
})
console.log(success("Services have been stopped successfully."))
}

48
packages/cli/src/hosting/update.js Normal file
View File

@ -0,0 +1,48 @@
const {
checkDockerConfigured,
checkInitComplete,
downloadFiles,
handleError,
getServices,
} = require("./utils")
const { confirmation } = require("../questions")
const compose = require("docker-compose")
const { COMPOSE_PATH } = require("./makeFiles")
const { info, success } = require("../utils")
const { start } = require("./start")
const BB_COMPOSE_SERVICES = ["app-service", "worker-service", "proxy-service"]
const BB_SINGLE_SERVICE = ["budibase"]
exports.update = async () => {
const isSingle = Object.keys(getServices(COMPOSE_PATH)).length === 1
await checkDockerConfigured()
checkInitComplete()
if (
!isSingle &&
(await confirmation("Do you wish to update you docker-compose.yaml?"))
) {
await downloadFiles()
}
await handleError(async () => {
const status = await compose.ps()
const parts = status.out.split("\n")
const isUp = parts[2] && parts[2].indexOf("Up") !== -1
if (isUp) {
console.log(info("Stopping services, this may take a moment."))
await compose.stop()
}
console.log(info("Beginning update, this may take a few minutes."))
let services
if (isSingle) {
services = BB_SINGLE_SERVICE
} else {
services = BB_COMPOSE_SERVICES
}
await compose.pullMany(services, { log: true })
if (isUp) {
console.log(success("Update complete, restarting services..."))
await start()
}
})
}

67
packages/cli/src/hosting/utils.js Normal file
View File

@ -0,0 +1,67 @@
const { lookpath } = require("lookpath")
const fs = require("fs")
const makeFiles = require("./makeFiles")
const { logErrorToFile, downloadFile } = require("../utils")
const yaml = require("yaml")
const ERROR_FILE = "docker-error.log"
const FILE_URLS = [
"https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
]
exports.downloadFiles = async () => {
const promises = []
for (let url of FILE_URLS) {
const fileName = url.split("/").slice(-1)[0]
promises.push(downloadFile(url, `./${fileName}`))
}
await Promise.all(promises)
}
exports.checkDockerConfigured = async () => {
const error =
"docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
const docker = await lookpath("docker")
const compose = await lookpath("docker-compose")
if (!docker || !compose) {
throw error
}
}
exports.checkInitComplete = () => {
if (
!fs.existsSync(makeFiles.ENV_PATH) &&
!fs.existsSync(makeFiles.COMPOSE_PATH)
) {
throw "Please run the hosting --init command before any other hosting command."
}
}
exports.handleError = async func => {
try {
await func()
} catch (err) {
if (err && err.err) {
logErrorToFile(ERROR_FILE, err.err)
}
throw `Failed to start - logs written to file: ${ERROR_FILE}`
}
}
exports.getServices = path => {
const dockerYaml = fs.readFileSync(path, "utf8")
const parsedYaml = yaml.parse(dockerYaml)
return parsedYaml.services
}
exports.getAppService = path => {
const services = exports.getServices(path),
serviceList = Object.keys(services)
let service
if (services["app-service"]) {
service = services["app-service"]
} else if (serviceList.length === 1) {
service = services[serviceList[0]]
}
return service
}

49
packages/cli/src/hosting/watch.js Normal file
View File

@ -0,0 +1,49 @@
const { resolve } = require("path")
const fs = require("fs")
const { error, success } = require("../utils")
const yaml = require("yaml")
const { getAppService } = require("./utils")
exports.watchPlugins = async pluginPath => {
const PLUGIN_PATH = "/plugins"
// get absolute path
pluginPath = resolve(pluginPath)
if (!fs.existsSync(pluginPath)) {
console.log(
error(
`The directory "${pluginPath}" does not exist, please create and then try again.`
)
)
return
}
const opts = ["docker-compose.yaml", "docker-compose.yml"]
let dockerFilePath = opts.find(name => fs.existsSync(name))
if (!dockerFilePath) {
console.log(error("Unable to locate docker-compose YAML."))
return
}
const dockerYaml = fs.readFileSync(dockerFilePath, "utf8")
const parsedYaml = yaml.parse(dockerYaml)
const service = getAppService(dockerFilePath)
if (!service) {
console.log(
error(
"Unable to locate service within compose file, is it a valid Budibase configuration?"
)
)
return
}
// set environment variable
service.environment["PLUGINS_DIR"] = PLUGIN_PATH
// add volumes to parsed yaml
if (!service.volumes) {
service.volumes = []
}
const found = service.volumes.find(vol => vol.includes(PLUGIN_PATH))
if (found) {
service.volumes.splice(service.volumes.indexOf(found), 1)
}
service.volumes.push(`${pluginPath}:${PLUGIN_PATH}`)
fs.writeFileSync(dockerFilePath, yaml.stringify(parsedYaml))
console.log(success("Docker compose configuration has been updated!"))
}

17
packages/cli/src/structures/Command.js
View File

@ -39,10 +39,7 @@ class Command {
command = command.description(getHelpDescription(thisCmd.help))
}
for (let opt of thisCmd.opts) {
command = command.option(
`${opt.command}`,
getSubHelpDescription(opt.help)
)
command = command.option(opt.command, getSubHelpDescription(opt.help))
}
command.helpOption(
"--help",
@ -50,19 +47,25 @@ class Command {
)
command.action(async options => {
try {
let executed = false
let executed = false,
found = false
for (let opt of thisCmd.opts) {
let lookup = opt.command.split(" ")[0].replace("--", "")
// need to handle how commander converts watch-plugin-dir to watchPluginDir
lookup = this.convertToCommander(lookup)
if (!executed && options[lookup]) {
found = !executed && options[lookup]
if (found && opt.func) {
const input =
Object.keys(options).length > 1 ? options : options[lookup]
await opt.func(input)
executed = true
}
}
if (!executed) {
if (found && !executed) {
console.log(
error(`${Object.keys(options)[0]} is an option, not an operation.`)
)
} else if (!executed) {
console.log(error(`Unknown ${this.command} option.`))
command.help()
}

3
packages/cli/src/structures/ConfigManager.js
View File

@ -33,11 +33,10 @@ class ConfigManager {
}
setValue(key, value) {
const updated = {
this.config = {
...this.config,
[key]: value,
}
this.config = updated
}
removeKey(key) {