Migration locks and add optional preventRetry option
This commit is contained in:
Rory Powell
parent
081db8423e
commit
86d094dda4
|
|
@ -19,6 +19,7 @@
|
|||
"dotenv": "^16.0.1",
|
||||
"emitter-listener": "^1.1.2",
|
||||
"ioredis": "^4.27.1",
|
||||
"redlock": "^4.0.0",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"koa-passport": "^4.1.4",
|
||||
"lodash": "^4.17.21",
|
||||
|
|
@ -55,6 +56,7 @@
|
|||
"@types/tar-fs": "^2.0.1",
|
||||
"@types/uuid": "^8.3.4",
|
||||
"@types/semver": "^7.0.0",
|
||||
"@types/redlock": "^4.0.0",
|
||||
"ioredis-mock": "^5.5.5",
|
||||
"jest": "^27.0.3",
|
||||
"koa": "2.7.0",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
module.exports = {
|
||||
Client: require("./src/redis"),
|
||||
utils: require("./src/redis/utils"),
|
||||
clients: require("./src/redis/authRedis"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,6 +90,15 @@ exports.runMigration = async (migration, options = {}) => {
|
|||
log(
|
||||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
|
||||
)
|
||||
|
||||
if (migration.preventRetry) {
|
||||
// eagerly set the completion date
|
||||
// so that we never run this migration twice even upon failure
|
||||
doc[migrationName] = Date.now()
|
||||
const response = await db.put(doc)
|
||||
doc._rev = response.rev
|
||||
}
|
||||
|
||||
// run the migration with tenant context
|
||||
if (migrationType === exports.MIGRATION_TYPES.APP) {
|
||||
await context.doInAppContext(db.name, async () => {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,23 @@
|
|||
const Client = require("./index")
|
||||
const utils = require("./utils")
|
||||
const { getRedlock } = require("./redlock")
|
||||
|
||||
let userClient, sessionClient, appClient, cacheClient
|
||||
let migrationsRedlock
|
||||
|
||||
// turn retry off so that only one instance can ever hold the lock
|
||||
const migrationsRedlockConfig = { retryCount: 0 }
|
||||
|
||||
async function init() {
|
||||
userClient = await new Client(utils.Databases.USER_CACHE).init()
|
||||
sessionClient = await new Client(utils.Databases.SESSIONS).init()
|
||||
appClient = await new Client(utils.Databases.APP_METADATA).init()
|
||||
cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
|
||||
// pass the underlying ioredis client to redlock
|
||||
migrationsRedlock = getRedlock(
|
||||
cacheClient.getClient(),
|
||||
migrationsRedlockConfig
|
||||
)
|
||||
}
|
||||
|
||||
process.on("exit", async () => {
|
||||
|
|
@ -42,4 +52,10 @@ module.exports = {
|
|||
}
|
||||
return cacheClient
|
||||
},
|
||||
getMigrationsRedlock: async () => {
|
||||
if (!migrationsRedlock) {
|
||||
await init()
|
||||
}
|
||||
return migrationsRedlock
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -139,6 +139,10 @@ class RedisWrapper {
|
|||
this._db = db
|
||||
}
|
||||
|
||||
getClient() {
|
||||
return CLIENT
|
||||
}
|
||||
|
||||
async init() {
|
||||
CLOSED = false
|
||||
init()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
import Redlock from "redlock"
|
||||
|
||||
export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
|
||||
return new Redlock([redisClient], {
|
||||
// the expected clock drift; for more details
|
||||
// see http://redis.io/topics/distlock
|
||||
driftFactor: 0.01, // multiplied by lock ttl to determine drift time
|
||||
|
||||
// the max number of times Redlock will attempt
|
||||
// to lock a resource before erroring
|
||||
retryCount: opts.retryCount,
|
||||
|
||||
// the time in ms between attempts
|
||||
retryDelay: 200, // time in ms
|
||||
|
||||
// the max time in ms randomly added to retries
|
||||
// to improve performance under high contention
|
||||
// see https://www.awsarchitectureblog.com/2015/03/backoff.html
|
||||
retryJitter: 200, // time in ms
|
||||
})
|
||||
}
|
||||
|
|
@ -829,6 +829,11 @@
|
|||
dependencies:
|
||||
"@babel/types" "^7.3.0"
|
||||
|
||||
"@types/bluebird@*":
|
||||
version "3.5.36"
|
||||
resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.36.tgz#00d9301d4dc35c2f6465a8aec634bb533674c652"
|
||||
integrity sha512-HBNx4lhkxN7bx6P0++W8E289foSu8kO8GCk2unhuVggO+cE7rh9DhZUyPhUxNRG9m+5B5BTKxZQ5ZP92x/mx9Q==
|
||||
|
||||
"@types/body-parser@*":
|
||||
version "1.19.2"
|
||||
resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0"
|
||||
|
|
@ -895,13 +900,6 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
|
||||
integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
|
||||
|
||||
"@types/ioredis@^4.27.1":
|
||||
version "4.28.10"
|
||||
resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
|
||||
integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762"
|
||||
|
|
@ -993,6 +991,21 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
|
||||
integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
|
||||
|
||||
"@types/redis@^2.8.0":
|
||||
version "2.8.32"
|
||||
resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11"
|
||||
integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/redlock@^4.0.0":
|
||||
version "4.0.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.3.tgz#aeab5fe5f0d433a125f6dcf9a884372ac0cddd4b"
|
||||
integrity sha512-mcvvrquwREbAqyZALNBIlf49AL9Aa324BG+J/Dv4TAP8g+nxQMBI4/APNqqS99QEY7VTNT9XvsaczCVGK8uNnQ==
|
||||
dependencies:
|
||||
"@types/bluebird" "*"
|
||||
"@types/redis" "^2.8.0"
|
||||
|
||||
"@types/semver@^7.0.0":
|
||||
version "7.3.9"
|
||||
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
|
||||
|
|
@ -1397,6 +1410,11 @@ bl@^4.0.3:
|
|||
inherits "^2.0.4"
|
||||
readable-stream "^3.4.0"
|
||||
|
||||
bluebird@^3.7.2:
|
||||
version "3.7.2"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
||||
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
|
||||
|
||||
boom@2.x.x:
|
||||
version "2.10.1"
|
||||
resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f"
|
||||
|
|
@ -4835,6 +4853,13 @@ redis-parser@^3.0.0:
|
|||
dependencies:
|
||||
redis-errors "^1.0.0"
|
||||
|
||||
redlock@^4.0.0:
|
||||
version "4.2.0"
|
||||
resolved "https://registry.yarnpkg.com/redlock/-/redlock-4.2.0.tgz#c26590768559afd5fff76aa1133c94b411ff4f5f"
|
||||
integrity sha512-j+oQlG+dOwcetUt2WJWttu4CZVeRzUrcVcISFmEmfyuwCVSJ93rDT7YSgg7H7rnxwoRyk/jU46kycVka5tW7jA==
|
||||
dependencies:
|
||||
bluebird "^3.7.2"
|
||||
|
||||
regenerator-runtime@^0.13.4:
|
||||
version "0.13.9"
|
||||
resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { migrations } from "@budibase/backend-core"
|
||||
import { migrations, redis } from "@budibase/backend-core"
|
||||
|
||||
// migration functions
|
||||
import * as userEmailViewCasing from "./functions/userEmailViewCasing"
|
||||
|
|
@ -15,6 +15,7 @@ export interface Migration {
|
|||
opts?: object
|
||||
fn: Function
|
||||
silent?: boolean
|
||||
preventRetry?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -66,21 +67,63 @@ export const MIGRATIONS: Migration[] = [
|
|||
opts: { all: true },
|
||||
fn: backfill.app.run,
|
||||
silent: !!env.SELF_HOSTED, // reduce noisy logging
|
||||
preventRetry: !!env.SELF_HOSTED, // only ever run once
|
||||
},
|
||||
{
|
||||
type: migrations.MIGRATION_TYPES.GLOBAL,
|
||||
name: "event_global_backfill",
|
||||
fn: backfill.global.run,
|
||||
silent: !!env.SELF_HOSTED, // reduce noisy logging
|
||||
preventRetry: !!env.SELF_HOSTED, // only ever run once
|
||||
},
|
||||
{
|
||||
type: migrations.MIGRATION_TYPES.INSTALLATION,
|
||||
name: "event_installation_backfill",
|
||||
fn: backfill.installation.run,
|
||||
silent: !!env.SELF_HOSTED, // reduce noisy logging
|
||||
preventRetry: !!env.SELF_HOSTED, // only ever run once
|
||||
},
|
||||
]
|
||||
|
||||
export const migrate = async (options?: MigrationOptions) => {
|
||||
await migrations.runMigrations(MIGRATIONS, options)
|
||||
if (env.SELF_HOSTED) {
|
||||
// self host runs migrations on startup
|
||||
// make sure only a single instance runs them
|
||||
await migrateWithLock(options)
|
||||
} else {
|
||||
await migrations.runMigrations(MIGRATIONS, options)
|
||||
}
|
||||
}
|
||||
|
||||
const migrateWithLock = async (options?: MigrationOptions) => {
|
||||
// get a new lock client
|
||||
|
||||
const redlock = await redis.clients.getMigrationsRedlock()
|
||||
// lock for 15 minutes
|
||||
const ttl = 1000 * 60 * 15
|
||||
|
||||
let migrationLock
|
||||
|
||||
// acquire lock
|
||||
try {
|
||||
migrationLock = await redlock.lock("migrations", ttl)
|
||||
} catch (e: any) {
|
||||
if (e.name === "LockError") {
|
||||
return
|
||||
} else {
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// run migrations
|
||||
try {
|
||||
await migrations.runMigrations(MIGRATIONS, options)
|
||||
} finally {
|
||||
// release lock
|
||||
try {
|
||||
await migrationLock.unlock()
|
||||
} catch (e) {
|
||||
console.error("unable to release migration lock")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue