From e92a31bd459cb3e5524f45945e2892ecb4e509e0 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 10 Oct 2022 08:21:17 +0100 Subject: [PATCH] Add locking framework --- packages/backend-core/src/index.ts | 1 + packages/backend-core/src/pkg/redis.ts | 2 + packages/backend-core/src/redis/init.js | 25 ++++---- packages/backend-core/src/redis/redlock.ts | 74 ++++++++++++++++++++-- packages/backend-core/src/redis/utils.js | 1 + packages/server/src/migrations/index.ts | 47 +++++--------- packages/types/src/sdk/index.ts | 1 + packages/types/src/sdk/locks.ts | 31 +++++++++ packages/worker/src/migrations/index.ts | 47 +++++--------- 9 files changed, 149 insertions(+), 80 deletions(-) create mode 100644 packages/types/src/sdk/locks.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index cb1e6d1c82..275a954a78 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { locks, migrations } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index bae566b42e..0c374dd105 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -7,3 +7,4 @@ export * from "./datasources" export * from "./search" export * from "./koa" export * from "./auth" +export * from "./locks" diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts new file mode 100644 index 0000000000..3aa067bea1 --- /dev/null +++ b/packages/types/src/sdk/locks.ts @@ -0,0 +1,31 @@ +export enum LockType { + /** + * If this lock is already held the attempted operation will not be performed. + * No retries will take place and no error will be thrown. + */ + TRY_ONCE = "try_once", +} + +export enum LockName { + MIGRATIONS = "migrations", + TRIGGER_QUOTA = "trigger_quota", +} + +export interface LockOptions { + /** + * The lock type determines which client to use + */ + type: LockType + /** + * The name for the lock + */ + name: LockName + /** + * The ttl to auto-expire the lock if not unlocked manually + */ + ttl: number + /** + * The suffix to add to the lock name for additional uniqueness + */ + nameSuffix?: string +} diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts index 6900596216..19ef076a52 100644 --- a/packages/worker/src/migrations/index.ts +++ b/packages/worker/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { migrations, locks } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) }