Merge pull request #1329 from Budibase/single-stack-dev

Single stack dev

hosting/docker-compose.dev.yaml

@@ -61,9 +61,6 @@ services:
     container_name: budi-redis-dev
     restart: always
     image: redis
-    environment:
-      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
-      - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${REDIS_PORT}:6379"
     volumes:

hosting/docker-compose.yaml

@@ -99,8 +99,18 @@ services:
       - couchdb-service
     command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
   
+  redis-service:
+    restart: always
+    image: redis
+    ports:
+      - "${REDIS_PORT}:6379"
+    volumes:
+      - redis_data:/data
+
 volumes:
   couchdb_data:
     driver: local
   minio_data:
     driver: local
+  redis_data:
+    driver: local

hosting/envoy.yaml

@@ -36,6 +36,11 @@ static_resources:
                   cluster: worker-service
                   prefix_rewrite: "/"
 
+              - match: { prefix: "/cache/" }
+                route:
+                  cluster: redis-service
+                  prefix_rewrite: "/"
+
               - match: { prefix: "/db/" }
                 route:
                   cluster: couchdb-service
@@ -107,3 +112,18 @@ static_resources:
                 address: couchdb-service
                 port_value: 5984
 
+  - name: redis-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: redis-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: redis-service
+                port_value: 6379
+
+

packages/builder/cypress.json

@@ -3,6 +3,7 @@
   "video": true,
   "projectId": "bmbemn",
   "env": {
-    "PORT": "4001"
+    "PORT": "4001",
+    "JWT_SECRET": "test"
   }
 }

packages/builder/cypress/setup.js

@@ -1,25 +1,26 @@
-// What this script does:
-// 1. Removes the old test folder if it exists (.budibase)
-// 2. Initialises using `.budibase`
-// 3. Runs the server using said folder
-
-const { join, resolve } = require("path")
-const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase")
 const cypressConfig = require("../cypress.json")
+const path = require("path")
 
-const homedir = join(require("os").homedir(), ".budibase")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
 
 process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
 process.env.ENABLE_ANALYTICS = "false"
 process.env.PORT = cypressConfig.env.PORT
+process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = 1
+process.env.MINIO_URL = "http://localhost:10000/"
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"
 
-async function run(dir) {
+async function run() {
-  process.env.BUDIBASE_DIR = resolve(dir)
+  // require("dotenv").config({ path: resolve(dir, ".env") })
-  require("dotenv").config({ path: resolve(dir, ".env") })
 
   // dont make this a variable or top level require
   // it will cause environment module to be loaded prematurely
@@ -27,12 +28,15 @@ async function run(dir) {
   server.on("close", () => console.log("Server Closed"))
 }
 
-initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
+run()
-  .then(() => {
+
-    delete require.cache[require.resolve("../../server/src/environment")]
+// TODO: ensure that this still works
-    const xPlatHomeDir = homedir.startsWith("~")
+// initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
-      ? join(homedir(), homedir.substring(1))
+// .then(() => {
-      : homedir
+//   delete require.cache[require.resolve("../../server/src/environment")]
-    run(xPlatHomeDir)
+//   const xPlatHomeDir = homedir.startsWith("~")
-  })
+//     ? join(homedir(), homedir.substring(1))
-  .catch(e => console.error(e))
+//     : homedir
+//   run(xPlatHomeDir)
+// })
+// .catch(e => console.error(e))

packages/server/package.json

@@ -65,6 +65,7 @@
       "!src/db/tests/**/*",
       "!src/tests/**/*",
       "!src/automations/tests/**/*",
+      "!src/utilities/fileProcessor.js",
       "!src/utilities/fileSystem/**/*"
     ],
     "coverageReporters": [

packages/server/scripts/dev/manage.js

@@ -16,17 +16,6 @@ const Commands = {
   Nuke: "nuke",
 }
 
-const managementCommand = process.argv.slice(2)[0]
-
-if (
-  !managementCommand ||
-  !Object.values(Commands).some(command => managementCommand === command)
-) {
-  throw new Error(
-    "You must supply either an 'up' or 'down' commmand to manage the budibase dev env."
-  )
-}
-
 async function init() {
   const envFilePath = path.join(process.cwd(), ".env")
   if (fs.existsSync(envFilePath)) {
@@ -54,31 +43,30 @@ async function init() {
 async function up() {
   console.log("Spinning up your budibase dev environment... 🔧✨")
   await init()
-  try {
+  await compose.upAll(CONFIG)
-    await compose.upAll(CONFIG)
-  } catch (err) {
-    console.log("Something went wrong:", err.message)
-  }
 }
 
 async function down() {
   console.log("Spinning down your budibase dev environment... 🌇")
-  try {
+  await compose.stop(CONFIG)
-    await compose.stop(CONFIG)
-  } catch (err) {
-    console.log("Something went wrong:", err.message)
-  }
 }
 
 async function nuke() {
   console.log(
     "Clearing down your budibase dev environment, including all containers and volumes... 💥"
   )
-  try {
+  await compose.down(CONFIG)
-    await compose.down(CONFIG)
+}
-  } catch (err) {
+
-    console.log("Something went wrong:", err.message)
+const managementCommand = process.argv.slice(2)[0]
-  }
+
+if (
+  !managementCommand ||
+  !Object.values(Commands).some(command => managementCommand === command)
+) {
+  throw new Error(
+    "You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment."
+  )
 }
 
 let command
@@ -100,6 +88,9 @@ command()
   .then(() => {
     console.log("Done! 🎉")
   })
-  .catch(() => {
+  .catch(err => {
-    console.log("Error while managing budibase dev environment.")
+    console.error(
+      "Something went wrong while managing budibase dev environment:",
+      err.message
+    )
   })

packages/server/scripts/initialise.js

@@ -1,17 +0,0 @@
-// const { join } = require("path")
-// const { homedir } = require("os")
-
-// // const initialiseBudibase = require("../src/utilities/initialiseBudibase")
-// // const DIRECTORY = "~/.budibase"
-
-// function run() {
-//   let opts = {}
-//   // let dir = DIRECTORY
-//   opts.quiet = true
-//   // opts.dir = dir.startsWith("~") ? join(homedir(), dir.substring(1)) : dir
-//   return initialiseBudibase(opts)
-// }
-
-// run().then(() => {
-//   console.log("Init complete.")
-// })

packages/server/src/api/routes/static.js

@@ -1,10 +1,10 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/static")
 const { budibaseTempDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/security/permissions")
 const usage = require("../../middleware/usageQuota")
+const env = require("../../environment")
 
 const router = Router()
 

packages/server/src/electron.js

@@ -1,106 +1,112 @@
-const { app, BrowserWindow, shell, dialog } = require("electron")
+// const { app, BrowserWindow, shell, dialog } = require("electron")
-const { join } = require("./utilities/centralPath")
+// const { join } = require("./utilities/centralPath")
-const isDev = require("electron-is-dev")
+// const isDev = require("electron-is-dev")
-const { autoUpdater } = require("electron-updater")
+// const { autoUpdater } = require("electron-updater")
-const unhandled = require("electron-unhandled")
+// const unhandled = require("electron-unhandled")
-const { existsSync } = require("fs-extra")
+// const { existsSync } = require("fs-extra")
-const initialiseBudibase = require("./utilities/initialiseBudibase")
+// const initialiseBudibase = require("./utilities/initialiseBudibase")
-const { budibaseAppsDir } = require("./utilities/budibaseDir")
+// const { budibaseAppsDir } = require("./utilities/budibaseDir")
-const { openNewGitHubIssue, debugInfo } = require("electron-util")
+// const { openNewGitHubIssue, debugInfo } = require("electron-util")
-const eventEmitter = require("./events")
+// const eventEmitter = require("./events")
 
-const budibaseDir = budibaseAppsDir()
+// const budibaseDir = budibaseAppsDir()
-const envFile = join(budibaseDir, ".env")
+// const envFile = join(budibaseDir, ".env")
 
-async function startApp() {
+// async function startApp() {
-  if (!existsSync(envFile)) {
+//   if (!existsSync(envFile)) {
-    await initialiseBudibase({ dir: budibaseDir })
+//     await initialiseBudibase({ dir: budibaseDir })
-  }
+//   }
-  // evict environment from cache, so it reloads when next asked
+//   // evict environment from cache, so it reloads when next asked
-  delete require.cache[require.resolve("./environment")]
+//   delete require.cache[require.resolve("./environment")]
-  unhandled({
+//   // store the port incase its going to get overridden
-    showDialog: true,
+//   const port = process.env.PORT
-    reportButton: error => {
+//   require("dotenv").config({ path: envFile })
-      openNewGitHubIssue({
+//   // overwrite the port - don't want to use dotenv for the port
-        title: error.message,
+//   require("./environment")._set("PORT", port)
-        user: "Budibase",
-        labels: ["error-report"],
-        repo: "budibase",
-        body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
-          error.stack
-        }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
-      })
-    },
-  })
 
-  let win
+//   unhandled({
+//     showDialog: true,
+//     reportButton: error => {
+//       openNewGitHubIssue({
+//         title: error.message,
+//         user: "Budibase",
+//         labels: ["error-report"],
+//         repo: "budibase",
+//         body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
+//           error.stack
+//         }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
+//       })
+//     },
+//   })
 
-  function handleRedirect(e, url) {
+//   let win
-    e.preventDefault()
-    shell.openExternal(url)
-  }
 
-  async function createWindow() {
+//   function handleRedirect(e, url) {
-    app.server = require("./app")
+//     e.preventDefault()
-    eventEmitter.on("internal:port", port => {
+//     shell.openExternal(url)
-      const APP_URL = `http://localhost:${port}/_builder`
+//   }
-      const APP_TITLE = "Budibase Builder"
-      win = new BrowserWindow({
-        width: 1920,
-        height: 1080,
-        icon: join(__dirname, "..", "build", "icons", "512x512.png"),
-      })
-      win.setTitle(APP_TITLE)
-      win.loadURL(APP_URL)
-      if (isDev) {
-        win.webContents.openDevTools()
-      } else {
-        autoUpdater.checkForUpdatesAndNotify()
-      }
 
-      // open _blank in default browser
+//   async function createWindow() {
-      win.webContents.on("new-window", handleRedirect)
+//     app.server = require("./app")
-      win.webContents.on("will-navigate", handleRedirect)
+//     eventEmitter.on("internal:port", port => {
-    })
+//       const APP_URL = `http://localhost:${port}/_builder`
-  }
+//       const APP_TITLE = "Budibase Builder"
+//       win = new BrowserWindow({
+//         width: 1920,
+//         height: 1080,
+//         icon: join(__dirname, "..", "build", "icons", "512x512.png"),
+//       })
+//       win.setTitle(APP_TITLE)
+//       win.loadURL(APP_URL)
+//       if (isDev) {
+//         win.webContents.openDevTools()
+//       } else {
+//         autoUpdater.checkForUpdatesAndNotify()
+//       }
 
-  app.whenReady().then(createWindow)
+//       // open _blank in default browser
+//       win.webContents.on("new-window", handleRedirect)
+//       win.webContents.on("will-navigate", handleRedirect)
+//     })
+//   }
 
-  // Quit when all windows are closed.
+//   app.whenReady().then(createWindow)
-  app.on("window-all-closed", () => {
-    // On macOS it is common for applications and their menu bar
-    // to stay active until the user quits explicitly with Cmd + Q
-    if (process.platform !== "darwin") {
-      app.server.close()
-      app.quit()
-    }
-  })
 
-  app.on("activate", () => {
+//   // Quit when all windows are closed.
-    // On macOS it's common to re-create a window in the app when the
+//   app.on("window-all-closed", () => {
-    // dock icon is clicked and there are no other windows open.
+//     // On macOS it is common for applications and their menu bar
-    if (win === null) createWindow()
+//     // to stay active until the user quits explicitly with Cmd + Q
-  })
+//     if (process.platform !== "darwin") {
-}
+//       app.server.close()
+//       app.quit()
+//     }
+//   })
 
-autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
+//   app.on("activate", () => {
-  const dialogOpts = {
+//     // On macOS it's common to re-create a window in the app when the
-    type: "info",
+//     // dock icon is clicked and there are no other windows open.
-    buttons: ["Restart", "Later"],
+//     if (win === null) createWindow()
-    title: "Budibase Update Available",
+//   })
-    message: process.platform === "win32" ? releaseNotes : releaseName,
+// }
-    detail:
-      "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
-  }
 
-  dialog.showMessageBox(dialogOpts).then(returnValue => {
+// autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
-    if (returnValue.response === 0) autoUpdater.quitAndInstall()
+//   const dialogOpts = {
-  })
+//     type: "info",
-})
+//     buttons: ["Restart", "Later"],
+//     title: "Budibase Update Available",
+//     message: process.platform === "win32" ? releaseNotes : releaseName,
+//     detail:
+//       "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
+//   }
 
-autoUpdater.on("error", message => {
+//   dialog.showMessageBox(dialogOpts).then(returnValue => {
-  console.error("There was a problem updating the application")
+//     if (returnValue.response === 0) autoUpdater.quitAndInstall()
-  console.error(message)
+//   })
-})
+// })
 
-startApp()
+// autoUpdater.on("error", message => {
+//   console.error("There was a problem updating the application")
+//   console.error(message)
+// })
+
+// startApp()

packages/server/src/utilities/initialiseBudibase.js

@@ -1,37 +0,0 @@
-// const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra")
-// const { join, resolve } = require("./centralPath")
-// const { processString } = require("@budibase/string-templates")
-// const uuid = require("uuid")
-
-// module.exports = async opts => {
-//   // await ensureDir(opts.dir)
-//   await setCouchDbUrl(opts)
-
-//   // need an env file
-//   await createDevEnvFile(opts)
-// }
-
-// const setCouchDbUrl = async opts => {
-//   if (!opts.couchDbUrl) {
-//     const dataDir = join(opts.dir, ".data")
-//     await ensureDir(dataDir)
-//     opts.couchDbUrl =
-//       dataDir + (dataDir.endsWith("/") || dataDir.endsWith("\\") ? "" : "/")
-//   }
-// }
-
-// const createDevEnvFile = async opts => {
-//   const destConfigFile = join(opts.dir, "./.env")
-//   let createConfig = !existsSync(destConfigFile) || opts.quiet
-//   if (createConfig) {
-//     const template = await readFile(
-//       resolve(__dirname, "..", "..", ".env.template"),
-//       {
-//         encoding: "utf8",
-//       }
-//     )
-//     opts.cookieKey1 = opts.cookieKey1 || uuid.v4()
-//     const config = await processString(template, opts)
-//     await writeFile(destConfigFile, config, { flag: "w+" })
-//   }
-// }