From 1cf778845aafc7856ae7383abf2d76cbd793dea3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 4 May 2021 18:13:44 +0100 Subject: [PATCH] Fixing issues with Redis/Bull and the integration with new redis module. --- hosting/docker-compose.yaml | 2 -- hosting/envoy.dev.yaml.hbs | 19 ----------------- hosting/envoy.yaml | 20 ------------------ packages/auth/src/index.js | 4 ++++ packages/auth/src/redis/index.js | 20 +++--------------- packages/auth/src/redis/utils.js | 23 +++++++++++++++++++++ packages/server/scripts/dev/manage.js | 3 ++- packages/server/src/automations/triggers.js | 4 +++- packages/worker/scripts/dev/manage.js | 2 ++ 9 files changed, 37 insertions(+), 60 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index e062f0590e..6d9f64c07e 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -105,8 +105,6 @@ services: restart: always image: redis command: redis-server --requirepass ${REDIS_PASSWORD} - ports: - - "${REDIS_PORT}:6379" volumes: - redis_data:/data diff --git a/hosting/envoy.dev.yaml.hbs b/hosting/envoy.dev.yaml.hbs index a4e2a97118..98b554165c 100644 --- a/hosting/envoy.dev.yaml.hbs +++ b/hosting/envoy.dev.yaml.hbs @@ -21,11 +21,6 @@ static_resources: cluster: couchdb-service prefix_rewrite: "/" - - match: { prefix: "/cache/" } - route: - cluster: redis-service - prefix_rewrite: "/" - - match: { prefix: "/api/admin/" } route: cluster: worker-dev @@ -89,20 +84,6 @@ static_resources: address: couchdb-service port_value: 5984 - - name: redis-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: redis-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: redis-service - port_value: 6379 - - name: server-dev connect_timeout: 0.25s type: strict_dns diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 1fbd2070ff..d2202ff870 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -41,11 +41,6 @@ static_resources: cluster: worker-service prefix_rewrite: "/" - - match: { prefix: "/cache/" } - route: - cluster: redis-service - prefix_rewrite: "/" - - match: { prefix: "/db/" } route: cluster: couchdb-service @@ -117,18 +112,3 @@ static_resources: address: couchdb-service port_value: 5984 - - name: redis-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: redis-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: redis-service - port_value: 6379 - - diff --git a/packages/auth/src/index.js b/packages/auth/src/index.js index 348f911f80..5b3de3e336 100644 --- a/packages/auth/src/index.js +++ b/packages/auth/src/index.js @@ -28,6 +28,10 @@ module.exports = { setDB(pouch) }, db: require("./db/utils"), + redis: { + client: require("./redis"), + utils: require("./redis/utils"), + }, utils: { ...require("./utils"), ...require("./hashing"), diff --git a/packages/auth/src/redis/index.js b/packages/auth/src/redis/index.js index b81c3aee74..cdd51fc2a4 100644 --- a/packages/auth/src/redis/index.js +++ b/packages/auth/src/redis/index.js @@ -1,9 +1,6 @@ const Redis = require("ioredis") -const env = require("../environment") -const { addDbPrefix, removeDbPrefix } = require("./utils") +const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils") -const CONNECT_TIMEOUT_MS = 10000 -const SLOT_REFRESH_MS = 2000 const CLUSTERED = false let CLIENT @@ -15,21 +12,10 @@ let CLIENT */ function init() { return new Promise((resolve, reject) => { - const [host, port] = env.REDIS_URL.split(":") - const opts = { - connectTimeout: CONNECT_TIMEOUT_MS, - } + const { opts, host, port } = getRedisOptions(CLUSTERED) if (CLUSTERED) { - opts.redisOptions = {} - opts.redisOptions.tls = {} - opts.redisOptions.password = env.REDIS_PASSWORD - opts.slotsRefreshTimeout = SLOT_REFRESH_MS - opts.dnsLookup = (address, callback) => callback(null, address) - CLIENT = new Redis.Cluster([{ port, host }]) + CLIENT = new Redis.Cluster([{ host, port }], opts) } else { - opts.password = env.REDIS_PASSWORD - opts.port = port - opts.host = host CLIENT = new Redis(opts) } CLIENT.on("end", err => { diff --git a/packages/auth/src/redis/utils.js b/packages/auth/src/redis/utils.js index 90f82d041a..54b0c882cd 100644 --- a/packages/auth/src/redis/utils.js +++ b/packages/auth/src/redis/utils.js @@ -1,9 +1,32 @@ +const env = require("../environment") + +const SLOT_REFRESH_MS = 2000 +const CONNECT_TIMEOUT_MS = 10000 const SEPARATOR = "-" exports.Databases = { PW_RESETS: "pwReset", } +exports.getRedisOptions = (clustered = false) => { + const [host, port] = env.REDIS_URL.split(":") + const opts = { + connectTimeout: CONNECT_TIMEOUT_MS, + } + if (clustered) { + opts.redisOptions = {} + opts.redisOptions.tls = {} + opts.redisOptions.password = env.REDIS_PASSWORD + opts.slotsRefreshTimeout = SLOT_REFRESH_MS + opts.dnsLookup = (address, callback) => callback(null, address) + } else { + opts.host = host + opts.port = port + opts.password = env.REDIS_PASSWORD + } + return { opts, host, port } +} + exports.addDbPrefix = (db, key) => { return `${db}${SEPARATOR}${key}` } diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 305df573e4..24cac981cf 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -37,9 +37,10 @@ async function init() { PORT: 4001, MINIO_URL: "http://localhost:10000/", COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/", - REDIS_URL: "http://localhost:10000/cache/", + REDIS_URL: "localhost:6379", WORKER_URL: "http://localhost:4002", JWT_SECRET: "testsecret", + REDIS_PASSWORD: "budibase", MINIO_ACCESS_KEY: "budibase", MINIO_SECRET_KEY: "budibase", COUCH_DB_PASSWORD: "budibase", diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js index e11784eb95..77eb32377a 100644 --- a/packages/server/src/automations/triggers.js +++ b/packages/server/src/automations/triggers.js @@ -4,8 +4,10 @@ const Queue = require("bull") const { setQueues, BullAdapter } = require("bull-board") const { getAutomationParams } = require("../db/utils") const { coerce } = require("../utilities/rowProcessor") +const { utils } = require("@budibase/auth").redis -let automationQueue = new Queue("automationQueue") +const { opts } = utils.getRedisOptions() +let automationQueue = new Queue("automationQueue", { redis: opts }) // Set up queues for bull board admin setQueues([new BullAdapter(automationQueue)]) diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index f363bd05ac..7322349b72 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -12,6 +12,8 @@ async function init() { MINIO_SECRET_KEY: "budibase", COUCH_DB_USER: "budibase", COUCH_DB_PASSWORD: "budibase", + REDIS_URL: "localhost:6379", + REDIS_PASSWORD: "budibase", MINIO_URL: "http://localhost:10000/", COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/", }