Fix for debugging with webstorm the old way (if desired), updating the builder middleware to be more multi-dev capable, ignoring 409s when attempting to update the last updated at for apps (if multiple devs hit at same time, only use one) also updating writethrough cache to retry once, with the extended TTL on locks, plus the multi-dev collab it can take a minute to update usage quota doc when a lot of updates occur at once.

packages/backend-core/src/cache/writethrough.ts

@@ -44,7 +44,7 @@ async function put(
   if (updateDb) {
     const lockResponse = await locks.doWithLock(
       {
-        type: LockType.TRY_ONCE,
+        type: LockType.TRY_TWICE,
         name: LockName.PERSIST_WRITETHROUGH,
         resource: key,
         ttl: 15000,

packages/backend-core/src/redis/redlockImpl.ts

@@ -4,10 +4,10 @@ import { LockOptions, LockType } from "@budibase/types"
 import * as context from "../context"
 import env from "../environment"
 
-const getClient = async (
+async function getClient(
   type: LockType,
   opts?: Redlock.Options
-): Promise<Redlock> => {
+): Promise<Redlock> {
   if (type === LockType.CUSTOM) {
     return newRedlock(opts)
   }
@@ -18,6 +18,9 @@ const getClient = async (
     case LockType.TRY_ONCE: {
       return newRedlock(OPTIONS.TRY_ONCE)
     }
+    case LockType.TRY_TWICE: {
+      return newRedlock(OPTIONS.TRY_TWICE)
+    }
     case LockType.DEFAULT: {
       return newRedlock(OPTIONS.DEFAULT)
     }
@@ -35,6 +38,9 @@ const OPTIONS = {
     // immediately throws an error if the lock is already held
     retryCount: 0,
   },
+  TRY_TWICE: {
+    retryCount: 1,
+  },
   TEST: {
     // higher retry count in unit tests
     // due to high contention.
@@ -62,7 +68,7 @@ const OPTIONS = {
   },
 }
 
-const newRedlock = async (opts: Redlock.Options = {}) => {
+export async function newRedlock(opts: Redlock.Options = {}) {
   let options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
   const client = redisWrapper.getClient()
@@ -81,22 +87,26 @@ type RedlockExecution<T> =
   | SuccessfulRedlockExecution<T>
   | UnsuccessfulRedlockExecution
 
-export const doWithLock = async <T>(
+function getLockName(opts: LockOptions) {
-  opts: LockOptions,
-  task: () => Promise<T>
-): Promise<RedlockExecution<T>> => {
-  const redlock = await getClient(opts.type, opts.customOptions)
-  let lock
-  try {
   // determine lock name
   // by default use the tenantId for uniqueness, unless using a system lock
   const prefix = opts.systemLock ? "system" : context.getTenantId()
   let name: string = `lock:${prefix}_${opts.name}`
-
   // add additional unique name if required
   if (opts.resource) {
     name = name + `_${opts.resource}`
   }
+  return name
+}
+
+export async function doWithLock<T>(
+  opts: LockOptions,
+  task: () => Promise<T>
+): Promise<RedlockExecution<T>> {
+  const redlock = await getClient(opts.type, opts.customOptions)
+  let lock
+  try {
+    const name = getLockName(opts)
 
     // create the lock
     lock = await redlock.lock(name, opts.ttl)

packages/server/src/middleware/builder.ts

@@ -9,8 +9,8 @@ import {
   checkDebounce,
   setDebounce,
 } from "../utilities/redis"
-import { db as dbCore, cache, permissions } from "@budibase/backend-core"
+import { db as dbCore, cache } from "@budibase/backend-core"
-import { BBContext, Database } from "@budibase/types"
+import { UserCtx, Database } from "@budibase/types"
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -23,7 +23,7 @@ const DEBOUNCE_TIME_SEC = 30
  * through the authorized middleware                *
  ****************************************************/
 
-async function checkDevAppLocks(ctx: BBContext) {
+async function checkDevAppLocks(ctx: UserCtx) {
   const appId = ctx.appId
 
   // if any public usage, don't proceed
@@ -42,7 +42,7 @@ async function checkDevAppLocks(ctx: BBContext) {
   }
 }
 
-async function updateAppUpdatedAt(ctx: BBContext) {
+async function updateAppUpdatedAt(ctx: UserCtx) {
   const appId = ctx.appId
   // if debouncing skip this update
   // get methods also aren't updating
@@ -50,6 +50,7 @@ async function updateAppUpdatedAt(ctx: BBContext) {
     return
   }
   await dbCore.doWithDB(appId, async (db: Database) => {
+    try {
       const metadata = await db.get(DocumentType.APP_METADATA)
       metadata.updatedAt = new Date().toISOString()
 
@@ -60,10 +61,18 @@ async function updateAppUpdatedAt(ctx: BBContext) {
       await cache.app.invalidateAppMetadata(appId, metadata)
       // set a new debounce record with a short TTL
       await setDebounce(appId, DEBOUNCE_TIME_SEC)
+    } catch (err: any) {
+      // if a 409 occurs, then multiple clients connected at the same time - ignore
+      if (err?.status === 409) {
+        return
+      } else {
+        throw err
+      }
+    }
   })
 }
 
-export default async function builder(ctx: BBContext) {
+export default async function builder(ctx: UserCtx) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {

packages/server/src/utilities/fileSystem/app.ts

@@ -35,11 +35,21 @@ export const getComponentLibraryManifest = async (library: string) => {
   const filename = "manifest.json"
 
   if (env.isDev() || env.isTest()) {
-    const path = join(TOP_LEVEL_PATH, "packages/client", filename)
+    const paths = [
+      join(TOP_LEVEL_PATH, "packages/client", filename),
+      join(process.cwd(), "client", filename),
+    ]
+    for (let path of paths) {
+      if (fs.existsSync(path)) {
         // always load from new so that updates are refreshed
         delete require.cache[require.resolve(path)]
         return require(path)
       }
+    }
+    throw new Error(
+      `Unable to find ${filename} in development environment (may need to build).`
+    )
+  }
 
   if (!appId) {
     throw new Error("No app ID found - cannot get component libraries")

packages/types/src/sdk/locks.ts

@@ -6,6 +6,7 @@ export enum LockType {
    * No retries will take place and no error will be thrown.
    */
   TRY_ONCE = "try_once",
+  TRY_TWICE = "try_twice",
   DEFAULT = "default",
   DELAY_500 = "delay_500",
   CUSTOM = "custom",