diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 37657ce009..0cd7bc92bf 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -105,6 +105,8 @@ services: restart: always image: redis command: redis-server --requirepass ${REDIS_PASSWORD} + ports: + - "${REDIS_PORT}:6379" volumes: - redis_data:/data diff --git a/packages/auth/src/redis/index.js b/packages/auth/src/redis/index.js index 78e3ea7acd..7d9e9ad637 100644 --- a/packages/auth/src/redis/index.js +++ b/packages/auth/src/redis/index.js @@ -3,41 +3,86 @@ const env = require("../environment") const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis") const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils") +const RETRY_PERIOD_MS = 2000 +const STARTUP_TIMEOUT_MS = 5000 const CLUSTERED = false // for testing just generate the client once -let CONNECTED = false +let CLOSED = false let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null +// if in test always connected +let CONNECTED = !!env.isTest() + +function connectionError(timeout, err) { + // manually shut down, ignore errors + if (CLOSED) { + return + } + // always clear this on error + clearTimeout(timeout) + CONNECTED = false + console.error("Redis connection failed - " + err) + setTimeout(() => { + init() + }, RETRY_PERIOD_MS) +} /** * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise * will return the ioredis client which will be ready to use. - * @return {Promise} The ioredis client. */ function init() { - return new Promise((resolve, reject) => { - // testing uses a single in memory client - if (env.isTest() || (CLIENT && CONNECTED)) { - return resolve(CLIENT) + let timeout + CLOSED = false + // testing uses a single in memory client + if (env.isTest() || (CLIENT && CONNECTED)) { + return + } + // start the timer - only allowed 5 seconds to connect + timeout = setTimeout(() => { + if (!CONNECTED) { + connectionError(timeout) } - const { opts, host, port } = getRedisOptions(CLUSTERED) - if (CLUSTERED) { - CLIENT = new Redis.Cluster([{ host, port }], opts) - } else { - CLIENT = new Redis(opts) + }, STARTUP_TIMEOUT_MS) + + // disconnect any lingering client + if (CLIENT) { + CLIENT.disconnect() + } + const { opts, host, port } = getRedisOptions(CLUSTERED) + if (CLUSTERED) { + CLIENT = new Redis.Cluster([{ host, port }], opts) + } else { + CLIENT = new Redis(opts) + } + // attach handlers + CLIENT.on("end", err => { + connectionError(timeout, err) + }) + CLIENT.on("error", err => { + connectionError(timeout, err) + }) + CLIENT.on("connect", () => { + clearTimeout(timeout) + CONNECTED = true + }) +} + +function waitForConnection() { + return new Promise(resolve => { + if (CLIENT == null) { + init() + } else if (CONNECTED) { + resolve() + return } - CLIENT.on("end", err => { - reject(err) - CONNECTED = false - }) - CLIENT.on("error", err => { - reject(err) - CONNECTED = false - }) - CLIENT.on("connect", () => { - resolve(CLIENT) - CONNECTED = true - }) + // check if the connection is ready + const interval = setInterval(() => { + if (CONNECTED) { + clearInterval(interval) + resolve() + } + }, 500) }) } @@ -85,31 +130,32 @@ class RedisWrapper { } async init() { - this._client = await init() + CLOSED = false + init() + await waitForConnection() return this } async finish() { - this._client.disconnect() + CLOSED = true + CLIENT.disconnect() } async scan() { - const db = this._db, - client = this._client + const db = this._db let stream if (CLUSTERED) { - let node = client.nodes("master") + let node = CLIENT.nodes("master") stream = node[0].scanStream({ match: db + "-*", count: 100 }) } else { - stream = client.scanStream({ match: db + "-*", count: 100 }) + stream = CLIENT.scanStream({ match: db + "-*", count: 100 }) } return promisifyStream(stream) } async get(key) { - const db = this._db, - client = this._client - let response = await client.get(addDbPrefix(db, key)) + const db = this._db + let response = await CLIENT.get(addDbPrefix(db, key)) // overwrite the prefixed key if (response != null && response.key) { response.key = key @@ -123,22 +169,20 @@ class RedisWrapper { } async store(key, value, expirySeconds = null) { - const db = this._db, - client = this._client + const db = this._db if (typeof value === "object") { value = JSON.stringify(value) } const prefixedKey = addDbPrefix(db, key) - await client.set(prefixedKey, value) + await CLIENT.set(prefixedKey, value) if (expirySeconds) { - await client.expire(prefixedKey, expirySeconds) + await CLIENT.expire(prefixedKey, expirySeconds) } } async delete(key) { - const db = this._db, - client = this._client - await client.del(addDbPrefix(db, key)) + const db = this._db + await CLIENT.del(addDbPrefix(db, key)) } async clear() { diff --git a/packages/builder/cypress/support/commands.js b/packages/builder/cypress/support/commands.js index 4f759a60ea..80d38937ac 100644 --- a/packages/builder/cypress/support/commands.js +++ b/packages/builder/cypress/support/commands.js @@ -11,12 +11,23 @@ Cypress.Commands.add("login", () => { if (cookie) return cy.visit(`localhost:${Cypress.env("PORT")}/builder`) - cy.contains("Create Test User").click() + + // cy.get("button").then(btn => { + // const adminUserButton = "Create super admin user" + // console.log(btn.first().first()) + // if (!btn.first().contains(adminUserButton)) { + // // create admin user + // cy.get("input").first().type("test@test.com") + // cy.get('input[type="password"]').first().type("test") + // cy.get('input[type="password"]').eq(1).type("test") + // cy.contains(adminUserButton).click() + // } + + // login cy.get("input").first().type("test@test.com") - cy.get('input[type="password"]').type("test") - cy.contains("Login").click() + // }) }) }) diff --git a/packages/cli/src/hosting/index.js b/packages/cli/src/hosting/index.js index 60d9f13e80..05d221435c 100644 --- a/packages/cli/src/hosting/index.js +++ b/packages/cli/src/hosting/index.js @@ -101,10 +101,15 @@ async function init(type) { async function start() { await checkDockerConfigured() checkInitComplete() - console.log(info("Starting services, this may take a moment.")) + console.log( + info( + "Starting services, this may take a moment - first time this may take a few minutes to download images." + ) + ) const port = makeEnv.get("MAIN_PORT") await handleError(async () => { - await compose.upAll({ cwd: "./", log: false }) + // need to log as it makes it more clear + await compose.upAll({ cwd: "./", log: true }) }) console.log( success( diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index 2e90fb83e7..068e1e59c0 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -23,7 +23,16 @@ async function redirect(ctx, method) { if (cookie) { ctx.set("set-cookie", cookie) } + let body + try { + body = await response.json() + } catch (err) { + // don't worry about errors, likely no JSON + } ctx.status = response.status + if (body) { + ctx.body = body + } ctx.cookies } diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 9ec6c2c687..50df056b2a 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -73,10 +73,11 @@ if (env.isProd()) { const server = http.createServer(app.callback()) destroyable(server) -server.on("close", () => { +server.on("close", async () => { if (env.NODE_ENV !== "jest") { console.log("Server Closed") } + await redis.shutdown() }) module.exports = server.listen(env.PORT || 0, async () => { diff --git a/packages/server/src/utilities/redis.js b/packages/server/src/utilities/redis.js index 8e0f774f42..ae18b82e02 100644 --- a/packages/server/src/utilities/redis.js +++ b/packages/server/src/utilities/redis.js @@ -11,6 +11,11 @@ exports.init = async () => { debounceClient = await new Client(utils.Databases.DEBOUNCE).init() } +exports.shutdown = async () => { + await devAppClient.finish() + await debounceClient.finish() +} + exports.doesUserHaveLock = async (devAppId, user) => { const value = await devAppClient.get(devAppId) if (!value) { diff --git a/packages/worker/src/api/controllers/admin/configs.js b/packages/worker/src/api/controllers/admin/configs.js index 8a6788cdfd..82466249a2 100644 --- a/packages/worker/src/api/controllers/admin/configs.js +++ b/packages/worker/src/api/controllers/admin/configs.js @@ -6,10 +6,8 @@ const { getGlobalUserParams, getScopedFullConfig, } = require("@budibase/auth").db -const fetch = require("node-fetch") const { Configs } = require("../../../constants") const email = require("../../../utilities/email") -const env = require("../../../environment") const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore const APP_PREFIX = "app_" @@ -155,12 +153,7 @@ exports.configChecklist = async function (ctx) { // TODO: Watch get started video // Apps exist - let allDbs - if (env.COUCH_DB_URL) { - allDbs = await (await fetch(`${env.COUCH_DB_URL}/_all_dbs`)).json() - } else { - allDbs = await CouchDB.allDbs() - } + let allDbs = await CouchDB.allDbs() const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX)) // They have set up SMTP diff --git a/packages/worker/src/api/routes/admin/users.js b/packages/worker/src/api/routes/admin/users.js index f334f05e7d..eff873a7b3 100644 --- a/packages/worker/src/api/routes/admin/users.js +++ b/packages/worker/src/api/routes/admin/users.js @@ -6,6 +6,17 @@ const Joi = require("joi") const router = Router() +function buildAdminInitValidation() { + return joiValidator.body( + Joi.object({ + email: Joi.string().required(), + password: Joi.string().required(), + }) + .required() + .unknown(false) + ) +} + function buildUserSaveValidation(isSelf = false) { let schema = { email: Joi.string().allow(null, ""), @@ -74,7 +85,11 @@ router buildInviteAcceptValidation(), controller.inviteAccept ) - .post("/api/admin/users/init", controller.adminUser) + .post( + "/api/admin/users/init", + buildAdminInitValidation(), + controller.adminUser + ) .get("/api/admin/users/self", controller.getSelf) // admin endpoint but needs to come at end (blocks other endpoints otherwise) .get("/api/admin/users/:id", adminOnly, controller.find)