Merge pull request #12984 from Budibase/fix/backups-with-many-attachments

Fixing an issue with 1000+ attachments being backed up

packages/backend-core/src/objectStore/objectStore.ts

@@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) {
       objects = objects.concat(response.Contents)
     }
     isTruncated = !!response.IsTruncated
-  } while (isTruncated)
+    token = response.NextContinuationToken
+  } while (isTruncated && token)
   return objects
 }
 

packages/backend-core/src/queue/queue.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions } from "bull"
+import BullQueue, { QueueOptions, JobOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
@@ -24,17 +24,24 @@ async function cleanup() {
 
 export function createQueue<T>(
   jobQueue: JobQueue,
-  opts: { removeStalledCb?: StalledFn } = {}
+  opts: {
+    removeStalledCb?: StalledFn
+    maxStalledCount?: number
+    jobOptions?: JobOptions
+  } = {}
 ): BullQueue.Queue<T> {
   const redisOpts = getRedisOptions()
   const queueConfig: QueueOptions = {
     redis: redisOpts,
     settings: {
-      maxStalledCount: 0,
+      maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
       lockDuration: QUEUE_LOCK_MS,
       lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
     },
   }
+  if (opts.jobOptions) {
+    queueConfig.defaultJobOptions = opts.jobOptions
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

yarn.lock

@@ -15867,6 +15867,11 @@ nodemailer@6.7.2:
   resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0"
   integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q==
 
+nodemailer@6.9.9:
+  version "6.9.9"
+  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.9.tgz#4549bfbf710cc6addec5064dd0f19874d24248d9"
+  integrity sha512-dexTll8zqQoVJEZPwQAKzxxtFn0qTnjdQTchoU6Re9BUUGBJiOy3YMn/0ShTW6J5M0dfQ1NeDeRTTl4oIWgQMA==
+
 nodemon@2.0.15:
   version "2.0.15"
   resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.15.tgz#504516ce3b43d9dc9a955ccd9ec57550a31a8d4e"