Merge pull request #1543 from Budibase/fix/mike-fixes

Some fixes for Redis

hosting/docker-compose.yaml

@@ -105,6 +105,8 @@ services:
     restart: always
     image: redis
     command: redis-server --requirepass ${REDIS_PASSWORD}
+    ports:
+      - "${REDIS_PORT}:6379"
     volumes:
       - redis_data:/data
 

packages/auth/src/redis/index.js

@@ -3,41 +3,86 @@ const env = require("../environment")
 const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
 const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils")
 
+const RETRY_PERIOD_MS = 2000
+const STARTUP_TIMEOUT_MS = 5000
 const CLUSTERED = false
 
 // for testing just generate the client once
-let CONNECTED = false
+let CLOSED = false
 let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null
+// if in test always connected
+let CONNECTED = !!env.isTest()
+
+function connectionError(timeout, err) {
+  // manually shut down, ignore errors
+  if (CLOSED) {
+    return
+  }
+  // always clear this on error
+  clearTimeout(timeout)
+  CONNECTED = false
+  console.error("Redis connection failed - " + err)
+  setTimeout(() => {
+    init()
+  }, RETRY_PERIOD_MS)
+}
 
 /**
  * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise
  * will return the ioredis client which will be ready to use.
- * @return {Promise<object>} The ioredis client.
  */
 function init() {
-  return new Promise((resolve, reject) => {
+  let timeout
-    // testing uses a single in memory client
+  CLOSED = false
-    if (env.isTest() || (CLIENT && CONNECTED)) {
+  // testing uses a single in memory client
-      return resolve(CLIENT)
+  if (env.isTest() || (CLIENT && CONNECTED)) {
+    return
+  }
+  // start the timer - only allowed 5 seconds to connect
+  timeout = setTimeout(() => {
+    if (!CONNECTED) {
+      connectionError(timeout)
     }
-    const { opts, host, port } = getRedisOptions(CLUSTERED)
+  }, STARTUP_TIMEOUT_MS)
-    if (CLUSTERED) {
+
-      CLIENT = new Redis.Cluster([{ host, port }], opts)
+  // disconnect any lingering client
-    } else {
+  if (CLIENT) {
-      CLIENT = new Redis(opts)
+    CLIENT.disconnect()
+  }
+  const { opts, host, port } = getRedisOptions(CLUSTERED)
+  if (CLUSTERED) {
+    CLIENT = new Redis.Cluster([{ host, port }], opts)
+  } else {
+    CLIENT = new Redis(opts)
+  }
+  // attach handlers
+  CLIENT.on("end", err => {
+    connectionError(timeout, err)
+  })
+  CLIENT.on("error", err => {
+    connectionError(timeout, err)
+  })
+  CLIENT.on("connect", () => {
+    clearTimeout(timeout)
+    CONNECTED = true
+  })
+}
+
+function waitForConnection() {
+  return new Promise(resolve => {
+    if (CLIENT == null) {
+      init()
+    } else if (CONNECTED) {
+      resolve()
+      return
     }
-    CLIENT.on("end", err => {
+    // check if the connection is ready
-      reject(err)
+    const interval = setInterval(() => {
-      CONNECTED = false
+      if (CONNECTED) {
-    })
+        clearInterval(interval)
-    CLIENT.on("error", err => {
+        resolve()
-      reject(err)
+      }
-      CONNECTED = false
+    }, 500)
-    })
-    CLIENT.on("connect", () => {
-      resolve(CLIENT)
-      CONNECTED = true
-    })
   })
 }
 
@@ -85,31 +130,32 @@ class RedisWrapper {
   }
 
   async init() {
-    this._client = await init()
+    CLOSED = false
+    init()
+    await waitForConnection()
     return this
   }
 
   async finish() {
-    this._client.disconnect()
+    CLOSED = true
+    CLIENT.disconnect()
   }
 
   async scan() {
-    const db = this._db,
+    const db = this._db
-      client = this._client
     let stream
     if (CLUSTERED) {
-      let node = client.nodes("master")
+      let node = CLIENT.nodes("master")
       stream = node[0].scanStream({ match: db + "-*", count: 100 })
     } else {
-      stream = client.scanStream({ match: db + "-*", count: 100 })
+      stream = CLIENT.scanStream({ match: db + "-*", count: 100 })
     }
     return promisifyStream(stream)
   }
 
   async get(key) {
-    const db = this._db,
+    const db = this._db
-      client = this._client
+    let response = await CLIENT.get(addDbPrefix(db, key))
-    let response = await client.get(addDbPrefix(db, key))
     // overwrite the prefixed key
     if (response != null && response.key) {
       response.key = key
@@ -123,22 +169,20 @@ class RedisWrapper {
   }
 
   async store(key, value, expirySeconds = null) {
-    const db = this._db,
+    const db = this._db
-      client = this._client
     if (typeof value === "object") {
       value = JSON.stringify(value)
     }
     const prefixedKey = addDbPrefix(db, key)
-    await client.set(prefixedKey, value)
+    await CLIENT.set(prefixedKey, value)
     if (expirySeconds) {
-      await client.expire(prefixedKey, expirySeconds)
+      await CLIENT.expire(prefixedKey, expirySeconds)
     }
   }
 
   async delete(key) {
-    const db = this._db,
+    const db = this._db
-      client = this._client
+    await CLIENT.del(addDbPrefix(db, key))
-    await client.del(addDbPrefix(db, key))
   }
 
   async clear() {

packages/builder/cypress/support/commands.js

@@ -11,12 +11,23 @@ Cypress.Commands.add("login", () => {
     if (cookie) return
 
     cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
-    cy.contains("Create Test User").click()
+
+    // cy.get("button").then(btn => {
+    //   const adminUserButton = "Create super admin user"
+    //   console.log(btn.first().first())
+    //   if (!btn.first().contains(adminUserButton)) {
+    //     // create admin user
+    //     cy.get("input").first().type("test@test.com")
+    //     cy.get('input[type="password"]').first().type("test")
+    //     cy.get('input[type="password"]').eq(1).type("test")
+    //     cy.contains(adminUserButton).click()
+    //   }
+
+    // login
     cy.get("input").first().type("test@test.com")
-
     cy.get('input[type="password"]').type("test")
-
     cy.contains("Login").click()
+    // })
   })
 })
 

packages/cli/src/hosting/index.js

@@ -101,10 +101,15 @@ async function init(type) {
 async function start() {
   await checkDockerConfigured()
   checkInitComplete()
-  console.log(info("Starting services, this may take a moment."))
+  console.log(
+    info(
+      "Starting services, this may take a moment - first time this may take a few minutes to download images."
+    )
+  )
   const port = makeEnv.get("MAIN_PORT")
   await handleError(async () => {
-    await compose.upAll({ cwd: "./", log: false })
+    // need to log as it makes it more clear
+    await compose.upAll({ cwd: "./", log: true })
   })
   console.log(
     success(

packages/server/src/api/controllers/dev.js

@@ -23,7 +23,16 @@ async function redirect(ctx, method) {
   if (cookie) {
     ctx.set("set-cookie", cookie)
   }
+  let body
+  try {
+    body = await response.json()
+  } catch (err) {
+    // don't worry about errors, likely no JSON
+  }
   ctx.status = response.status
+  if (body) {
+    ctx.body = body
+  }
   ctx.cookies
 }
 

packages/server/src/app.js

@@ -73,10 +73,11 @@ if (env.isProd()) {
 const server = http.createServer(app.callback())
 destroyable(server)
 
-server.on("close", () => {
+server.on("close", async () => {
   if (env.NODE_ENV !== "jest") {
     console.log("Server Closed")
   }
+  await redis.shutdown()
 })
 
 module.exports = server.listen(env.PORT || 0, async () => {

packages/server/src/utilities/redis.js

@@ -11,6 +11,11 @@ exports.init = async () => {
   debounceClient = await new Client(utils.Databases.DEBOUNCE).init()
 }
 
+exports.shutdown = async () => {
+  await devAppClient.finish()
+  await debounceClient.finish()
+}
+
 exports.doesUserHaveLock = async (devAppId, user) => {
   const value = await devAppClient.get(devAppId)
   if (!value) {

packages/worker/src/api/controllers/admin/configs.js

@@ -6,10 +6,8 @@ const {
   getGlobalUserParams,
   getScopedFullConfig,
 } = require("@budibase/auth").db
-const fetch = require("node-fetch")
 const { Configs } = require("../../../constants")
 const email = require("../../../utilities/email")
-const env = require("../../../environment")
 const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore
 
 const APP_PREFIX = "app_"
@@ -155,12 +153,7 @@ exports.configChecklist = async function (ctx) {
     // TODO: Watch get started video
 
     // Apps exist
-    let allDbs
+    let allDbs = await CouchDB.allDbs()
-    if (env.COUCH_DB_URL) {
-      allDbs = await (await fetch(`${env.COUCH_DB_URL}/_all_dbs`)).json()
-    } else {
-      allDbs = await CouchDB.allDbs()
-    }
     const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX))
 
     // They have set up SMTP

packages/worker/src/api/routes/admin/users.js

@@ -6,6 +6,17 @@ const Joi = require("joi")
 
 const router = Router()
 
+function buildAdminInitValidation() {
+  return joiValidator.body(
+    Joi.object({
+      email: Joi.string().required(),
+      password: Joi.string().required(),
+    })
+      .required()
+      .unknown(false)
+  )
+}
+
 function buildUserSaveValidation(isSelf = false) {
   let schema = {
     email: Joi.string().allow(null, ""),
@@ -74,7 +85,11 @@ router
     buildInviteAcceptValidation(),
     controller.inviteAccept
   )
-  .post("/api/admin/users/init", controller.adminUser)
+  .post(
+    "/api/admin/users/init",
+    buildAdminInitValidation(),
+    controller.adminUser
+  )
   .get("/api/admin/users/self", controller.getSelf)
   // admin endpoint but needs to come at end (blocks other endpoints otherwise)
   .get("/api/admin/users/:id", adminOnly, controller.find)