Updating hosting to break up single file into lots of independent commands - this was an attempt to improve syntax. Adding in single image handling, as well as full support for watch directory.

packages/cli/src/constants.js

@@ -11,7 +11,6 @@ exports.CommandWords = {
 exports.InitTypes = {
   QUICK: "quick",
   DIGITAL_OCEAN: "do",
-  SINGLE: "single",
 }
 
 exports.AnalyticsEvents = {

packages/cli/src/hosting/index.js

@@ -1,211 +1,11 @@
 const Command = require("../structures/Command")
-const { CommandWords, InitTypes, AnalyticsEvents } = require("../constants")
+const { CommandWords } = require("../constants")
-const { lookpath } = require("lookpath")
+const { init } = require("./init")
-const { resolve } = require("path")
+const { start } = require("./start")
-const {
+const { stop } = require("./stop")
-  downloadFile,
+const { status } = require("./status")
-  logErrorToFile,
+const { update } = require("./update")
-  success,
+const { watchPlugins } = require("./watch")
-  info,
-  error,
-  parseEnv,
-} = require("../utils")
-const { confirmation } = require("../questions")
-const fs = require("fs")
-const compose = require("docker-compose")
-const makeFiles = require("./makeFiles")
-const axios = require("axios")
-const { captureEvent } = require("../events")
-const yaml = require("yaml")
-
-const BUDIBASE_SERVICES = ["app-service", "worker-service", "proxy-service"]
-const ERROR_FILE = "docker-error.log"
-const FILE_URLS = [
-  "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
-]
-const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
-
-async function downloadFiles() {
-  const promises = []
-  for (let url of FILE_URLS) {
-    const fileName = url.split("/").slice(-1)[0]
-    promises.push(downloadFile(url, `./${fileName}`))
-  }
-  await Promise.all(promises)
-}
-
-async function checkDockerConfigured() {
-  const error =
-    "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
-  const docker = await lookpath("docker")
-  const compose = await lookpath("docker-compose")
-  if (!docker || !compose) {
-    throw error
-  }
-}
-
-function checkInitComplete() {
-  if (!fs.existsSync(makeFiles.filePath)) {
-    throw "Please run the hosting --init command before any other hosting command."
-  }
-}
-
-async function handleError(func) {
-  try {
-    await func()
-  } catch (err) {
-    if (err && err.err) {
-      logErrorToFile(ERROR_FILE, err.err)
-    }
-    throw `Failed to start - logs written to file: ${ERROR_FILE}`
-  }
-}
-
-async function init(type) {
-  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
-  await checkDockerConfigured()
-  if (!isQuick) {
-    const shouldContinue = await confirmation(
-      "This will create multiple files in current directory, should continue?"
-    )
-    if (!shouldContinue) {
-      console.log("Stopping.")
-      return
-    }
-  }
-  captureEvent(AnalyticsEvents.SelfHostInit, {
-    type,
-  })
-  await downloadFiles()
-  const config = isQuick ? makeFiles.QUICK_CONFIG : {}
-  if (type === InitTypes.DIGITAL_OCEAN) {
-    try {
-      const output = await axios.get(DO_USER_DATA_URL)
-      const response = parseEnv(output.data)
-      for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
-        if (response[key]) {
-          config[value] = response[key]
-        }
-      }
-    } catch (err) {
-      // don't need to handle error, just don't do anything
-    }
-  }
-  await makeFiles.makeEnv(config)
-}
-
-async function start() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(
-    info(
-      "Starting services, this may take a moment - first time this may take a few minutes to download images."
-    )
-  )
-  const port = makeFiles.getEnvProperty("MAIN_PORT")
-  await handleError(async () => {
-    // need to log as it makes it more clear
-    await compose.upAll({ cwd: "./", log: true })
-  })
-  console.log(
-    success(
-      `Services started, please go to http://localhost:${port} for next steps.`
-    )
-  )
-}
-
-async function status() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(info("Budibase status"))
-  await handleError(async () => {
-    const response = await compose.ps()
-    console.log(response.out)
-  })
-}
-
-async function stop() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(info("Stopping services, this may take a moment."))
-  await handleError(async () => {
-    await compose.stop()
-  })
-  console.log(success("Services have been stopped successfully."))
-}
-
-async function update() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  if (await confirmation("Do you wish to update you docker-compose.yaml?")) {
-    await downloadFiles()
-  }
-  await handleError(async () => {
-    const status = await compose.ps()
-    const parts = status.out.split("\n")
-    const isUp = parts[2] && parts[2].indexOf("Up") !== -1
-    if (isUp) {
-      console.log(info("Stopping services, this may take a moment."))
-      await compose.stop()
-    }
-    console.log(info("Beginning update, this may take a few minutes."))
-    await compose.pullMany(BUDIBASE_SERVICES, { log: true })
-    if (isUp) {
-      console.log(success("Update complete, restarting services..."))
-      await start()
-    }
-  })
-}
-
-async function watchPlugins(pluginPath) {
-  const PLUGIN_PATH = "/plugins"
-  // get absolute path
-  pluginPath = resolve(pluginPath)
-  if (!fs.existsSync(pluginPath)) {
-    console.log(
-      error(
-        `The directory "${pluginPath}" does not exist, please create and then try again.`
-      )
-    )
-    return
-  }
-  const opts = ["docker-compose.yaml", "docker-compose.yml"]
-  let dockerFilePath = opts.find(name => fs.existsSync(name))
-  if (!dockerFilePath) {
-    console.log(error("Unable to locate docker-compose YAML."))
-    return
-  }
-  const dockerYaml = fs.readFileSync(dockerFilePath, "utf8")
-  const parsedYaml = yaml.parse(dockerYaml)
-  let service,
-    serviceList = Object.keys(parsedYaml.services)
-  if (parsedYaml.services["app-service"]) {
-    service = parsedYaml.services["app-service"]
-  } else if (serviceList.length === 1) {
-    service = parsedYaml.services[serviceList[0]]
-  }
-  if (!service) {
-    console.log(
-      error(
-        "Unable to locate service within compose file, is it a valid Budibase configuration?"
-      )
-    )
-    return
-  }
-  // set environment variable
-  service.environment["PLUGINS_DIR"] = PLUGIN_PATH
-  // add volumes to parsed yaml
-  if (!service.volumes) {
-    service.volumes = []
-  }
-  const found = service.volumes.find(vol => vol.includes(PLUGIN_PATH))
-  if (found) {
-    service.volumes.splice(service.volumes.indexOf(found), 1)
-  }
-  service.volumes.push(`${pluginPath}:${PLUGIN_PATH}`)
-  fs.writeFileSync(dockerFilePath, yaml.stringify(parsedYaml))
-  console.log(success("Docker compose configuration has been updated!"))
-}
 
 const command = new Command(`${CommandWords.HOSTING}`)
   .addHelp("Controls self hosting on the Budibase platform.")
@@ -239,6 +39,6 @@ const command = new Command(`${CommandWords.HOSTING}`)
     "Add plugin directory watching to a Budibase install.",
     watchPlugins
   )
-  .addSubOption("--dev", "")
+  .addSubOption("--single", "Specify this with init to use the single image.")
 
 exports.command = command

packages/cli/src/hosting/init.js

@@ -0,0 +1,63 @@
+const { InitTypes, AnalyticsEvents } = require("../constants")
+const { confirmation } = require("../questions")
+const { captureEvent } = require("../events")
+const makeFiles = require("./makeFiles")
+const axios = require("axios")
+const { parseEnv } = require("../utils")
+const { checkDockerConfigured, downloadFiles } = require("./utils")
+const { watchPlugins } = require("./watch")
+
+const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
+
+async function getInitConfig(type, isQuick) {
+  const config = isQuick ? makeFiles.QUICK_CONFIG : {}
+  if (type === InitTypes.DIGITAL_OCEAN) {
+    try {
+      const output = await axios.get(DO_USER_DATA_URL)
+      const response = parseEnv(output.data)
+      for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
+        if (response[key]) {
+          config[value] = response[key]
+        }
+      }
+    } catch (err) {
+      // don't need to handle error, just don't do anything
+    }
+  }
+  return config
+}
+
+exports.init = async opts => {
+  let type, isSingle, watchDir
+  if (typeof opts === "string") {
+    type = opts
+  } else {
+    type = opts.init
+    isSingle = opts.single
+    watchDir = opts.watchPluginDir
+  }
+  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
+  await checkDockerConfigured()
+  if (!isQuick) {
+    const shouldContinue = await confirmation(
+      "This will create multiple files in current directory, should continue?"
+    )
+    if (!shouldContinue) {
+      console.log("Stopping.")
+      return
+    }
+  }
+  captureEvent(AnalyticsEvents.SelfHostInit, {
+    type,
+  })
+  const config = await getInitConfig(type, isQuick)
+  if (!isSingle) {
+    await downloadFiles()
+    await makeFiles.makeEnv(config)
+  } else {
+    await makeFiles.makeSingleCompose(config)
+    if (watchDir) {
+      await watchPlugins(watchDir)
+    }
+  }
+}

packages/cli/src/hosting/makeFiles.js

@@ -4,6 +4,7 @@ const fs = require("fs")
 const path = require("path")
 const randomString = require("randomstring")
 const yaml = require("yaml")
+const { getAppService } = require("./utils")
 
 const SINGLE_IMAGE = "budibase/budibase:latest"
 const VOL_NAME = "budibase_data"
@@ -16,12 +17,13 @@ function getSecrets() {
     "MINIO_ACCESS_KEY",
     "MINIO_SECRET_KEY",
     "COUCH_DB_PASSWORD",
-    "COUCH_DB_USER",
     "REDIS_PASSWORD",
     "INTERNAL_API_KEY",
   ]
   const obj = {}
   secrets.forEach(secret => (obj[secret] = randomString.generate()))
+  // hard code to admin
+  obj["COUCH_DB_USER"] = "admin"
   return obj
 }
 
@@ -70,10 +72,13 @@ function getEnv(port) {
   ].join("\n")
 }
 
-module.exports.filePath = ENV_PATH
+exports.ENV_PATH = ENV_PATH
+exports.COMPOSE_PATH = COMPOSE_PATH
+
 module.exports.ConfigMap = {
   MAIN_PORT: "port",
 }
+
 module.exports.QUICK_CONFIG = {
   key: "budibase",
   port: 10000,
@@ -112,3 +117,14 @@ module.exports.getEnvProperty = property => {
   }
   return property.split("=")[1].split("\n")[0]
 }
+
+module.exports.getComposeProperty = property => {
+  const appService = getAppService(COMPOSE_PATH)
+  if (property === "port" && Array.isArray(appService.ports)) {
+    const port = appService.ports[0]
+    return port.split(":")[0]
+  } else if (appService.environment) {
+    return appService.environment[property]
+  }
+  return null
+}

packages/cli/src/hosting/start.js

@@ -0,0 +1,34 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  handleError,
+} = require("./utils")
+const { info, success } = require("../utils")
+const makeFiles = require("./makeFiles")
+const compose = require("docker-compose")
+const fs = require("fs")
+
+exports.start = async () => {
+  await checkDockerConfigured()
+  checkInitComplete()
+  console.log(
+    info(
+      "Starting services, this may take a moment - first time this may take a few minutes to download images."
+    )
+  )
+  let port
+  if (fs.existsSync(makeFiles.ENV_PATH)) {
+    port = makeFiles.getEnvProperty("MAIN_PORT")
+  } else {
+    port = makeFiles.getComposeProperty("port")
+  }
+  await handleError(async () => {
+    // need to log as it makes it more clear
+    await compose.upAll({ cwd: "./", log: true })
+  })
+  console.log(
+    success(
+      `Services started, please go to http://localhost:${port} for next steps.`
+    )
+  )
+}

packages/cli/src/hosting/status.js

@@ -0,0 +1,17 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  handleError,
+} = require("./utils")
+const { info } = require("../utils")
+const compose = require("docker-compose")
+
+exports.status = async () => {
+  await checkDockerConfigured()
+  checkInitComplete()
+  console.log(info("Budibase status"))
+  await handleError(async () => {
+    const response = await compose.ps()
+    console.log(response.out)
+  })
+}

packages/cli/src/hosting/stop.js

@@ -0,0 +1,17 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  handleError,
+} = require("./utils")
+const { info, success } = require("../utils")
+const compose = require("docker-compose")
+
+exports.stop = async () => {
+  await checkDockerConfigured()
+  checkInitComplete()
+  console.log(info("Stopping services, this may take a moment."))
+  await handleError(async () => {
+    await compose.stop()
+  })
+  console.log(success("Services have been stopped successfully."))
+}

packages/cli/src/hosting/update.js

@@ -0,0 +1,48 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  downloadFiles,
+  handleError,
+  getServices,
+} = require("./utils")
+const { confirmation } = require("../questions")
+const compose = require("docker-compose")
+const { COMPOSE_PATH } = require("./makeFiles")
+const { info, success } = require("../utils")
+const { start } = require("./start")
+
+const BB_COMPOSE_SERVICES = ["app-service", "worker-service", "proxy-service"]
+const BB_SINGLE_SERVICE = ["budibase"]
+
+exports.update = async () => {
+  const isSingle = Object.keys(getServices(COMPOSE_PATH)).length === 1
+  await checkDockerConfigured()
+  checkInitComplete()
+  if (
+    !isSingle &&
+    (await confirmation("Do you wish to update you docker-compose.yaml?"))
+  ) {
+    await downloadFiles()
+  }
+  await handleError(async () => {
+    const status = await compose.ps()
+    const parts = status.out.split("\n")
+    const isUp = parts[2] && parts[2].indexOf("Up") !== -1
+    if (isUp) {
+      console.log(info("Stopping services, this may take a moment."))
+      await compose.stop()
+    }
+    console.log(info("Beginning update, this may take a few minutes."))
+    let services
+    if (isSingle) {
+      services = BB_SINGLE_SERVICE
+    } else {
+      services = BB_COMPOSE_SERVICES
+    }
+    await compose.pullMany(services, { log: true })
+    if (isUp) {
+      console.log(success("Update complete, restarting services..."))
+      await start()
+    }
+  })
+}

packages/cli/src/hosting/utils.js

@@ -0,0 +1,67 @@
+const { lookpath } = require("lookpath")
+const fs = require("fs")
+const makeFiles = require("./makeFiles")
+const { logErrorToFile, downloadFile } = require("../utils")
+const yaml = require("yaml")
+
+const ERROR_FILE = "docker-error.log"
+const FILE_URLS = [
+  "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
+]
+
+exports.downloadFiles = async () => {
+  const promises = []
+  for (let url of FILE_URLS) {
+    const fileName = url.split("/").slice(-1)[0]
+    promises.push(downloadFile(url, `./${fileName}`))
+  }
+  await Promise.all(promises)
+}
+
+exports.checkDockerConfigured = async () => {
+  const error =
+    "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
+  const docker = await lookpath("docker")
+  const compose = await lookpath("docker-compose")
+  if (!docker || !compose) {
+    throw error
+  }
+}
+
+exports.checkInitComplete = () => {
+  if (
+    !fs.existsSync(makeFiles.ENV_PATH) &&
+    !fs.existsSync(makeFiles.COMPOSE_PATH)
+  ) {
+    throw "Please run the hosting --init command before any other hosting command."
+  }
+}
+
+exports.handleError = async func => {
+  try {
+    await func()
+  } catch (err) {
+    if (err && err.err) {
+      logErrorToFile(ERROR_FILE, err.err)
+    }
+    throw `Failed to start - logs written to file: ${ERROR_FILE}`
+  }
+}
+
+exports.getServices = path => {
+  const dockerYaml = fs.readFileSync(path, "utf8")
+  const parsedYaml = yaml.parse(dockerYaml)
+  return parsedYaml.services
+}
+
+exports.getAppService = path => {
+  const services = exports.getServices(path),
+    serviceList = Object.keys(services)
+  let service
+  if (services["app-service"]) {
+    service = services["app-service"]
+  } else if (serviceList.length === 1) {
+    service = services[serviceList[0]]
+  }
+  return service
+}

packages/cli/src/hosting/watch.js

@@ -0,0 +1,49 @@
+const { resolve } = require("path")
+const fs = require("fs")
+const { error, success } = require("../utils")
+const yaml = require("yaml")
+const { getAppService } = require("./utils")
+
+exports.watchPlugins = async pluginPath => {
+  const PLUGIN_PATH = "/plugins"
+  // get absolute path
+  pluginPath = resolve(pluginPath)
+  if (!fs.existsSync(pluginPath)) {
+    console.log(
+      error(
+        `The directory "${pluginPath}" does not exist, please create and then try again.`
+      )
+    )
+    return
+  }
+  const opts = ["docker-compose.yaml", "docker-compose.yml"]
+  let dockerFilePath = opts.find(name => fs.existsSync(name))
+  if (!dockerFilePath) {
+    console.log(error("Unable to locate docker-compose YAML."))
+    return
+  }
+  const dockerYaml = fs.readFileSync(dockerFilePath, "utf8")
+  const parsedYaml = yaml.parse(dockerYaml)
+  const service = getAppService(dockerFilePath)
+  if (!service) {
+    console.log(
+      error(
+        "Unable to locate service within compose file, is it a valid Budibase configuration?"
+      )
+    )
+    return
+  }
+  // set environment variable
+  service.environment["PLUGINS_DIR"] = PLUGIN_PATH
+  // add volumes to parsed yaml
+  if (!service.volumes) {
+    service.volumes = []
+  }
+  const found = service.volumes.find(vol => vol.includes(PLUGIN_PATH))
+  if (found) {
+    service.volumes.splice(service.volumes.indexOf(found), 1)
+  }
+  service.volumes.push(`${pluginPath}:${PLUGIN_PATH}`)
+  fs.writeFileSync(dockerFilePath, yaml.stringify(parsedYaml))
+  console.log(success("Docker compose configuration has been updated!"))
+}

packages/cli/src/structures/Command.js

@@ -39,10 +39,7 @@ class Command {
       command = command.description(getHelpDescription(thisCmd.help))
     }
     for (let opt of thisCmd.opts) {
-      command = command.option(
+      command = command.option(opt.command, getSubHelpDescription(opt.help))
-        `${opt.command}`,
-        getSubHelpDescription(opt.help)
-      )
     }
     command.helpOption(
       "--help",
@@ -50,19 +47,25 @@ class Command {
     )
     command.action(async options => {
       try {
-        let executed = false
+        let executed = false,
+          found = false
         for (let opt of thisCmd.opts) {
           let lookup = opt.command.split(" ")[0].replace("--", "")
           // need to handle how commander converts watch-plugin-dir to watchPluginDir
           lookup = this.convertToCommander(lookup)
-          if (!executed && options[lookup]) {
+          found = !executed && options[lookup]
+          if (found && opt.func) {
             const input =
               Object.keys(options).length > 1 ? options : options[lookup]
             await opt.func(input)
             executed = true
           }
         }
-        if (!executed) {
+        if (found && !executed) {
+          console.log(
+            error(`${Object.keys(options)[0]} is an option, not an operation.`)
+          )
+        } else if (!executed) {
           console.log(error(`Unknown ${this.command} option.`))
           command.help()
         }

packages/cli/src/structures/ConfigManager.js

@@ -33,11 +33,10 @@ class ConfigManager {
   }
 
   setValue(key, value) {
-    const updated = {
+    this.config = {
       ...this.config,
       [key]: value,
     }
-    this.config = updated
   }
 
   removeKey(key) {