Fix for #9249 - there was an issue with the updating of attachment URLs when importing an app/backup, this resolves it by utilising pagination in the CouchDB find API, making sure to iterate through all documents when there is a larger number.

packages/server/src/sdk/app/backups/imports.ts

@@ -1,5 +1,5 @@
 import { db as dbCore, objectStore } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
+import { Database, Row } from "@budibase/types"
 import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
@@ -17,6 +17,9 @@ import {
 const uuid = require("uuid/v4")
 const tar = require("tar")
 
+// default limit - seems to work well for performance
+const FIND_LIMIT = 25
+
 type TemplateType = {
   file?: {
     type: string
@@ -25,9 +28,29 @@ type TemplateType = {
   key?: string
 }
 
+function generateAttachmentFindParams(
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      _id: {
+        $regex: `^${TABLE_ROW_PREFIX}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
 async function updateAttachmentColumns(prodAppId: string, db: Database) {
   // iterate through attachment documents and update them
   const tables = await sdk.tables.getAllInternalTables(db)
+  let updatedRows: Row[] = []
   for (let table of tables) {
     const attachmentCols: string[] = []
     for (let [key, column] of Object.entries(table.schema)) {
@@ -39,16 +62,15 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
     if (attachmentCols.length === 0) {
       continue
     }
+    let bookmark: null | string = null,
+      rowsLength = 0
+    do {
+      const params = generateAttachmentFindParams(attachmentCols, bookmark)
       // use the CouchDB Mango query API to lookup rows that have attachments
-    const params: CouchFindOptions = {
+      const resp = await dbCore.directCouchFind(db.name, params)
-      selector: {
+      bookmark = resp.bookmark
-        _id: {
+      rowsLength = resp.rows.length
-          $regex: `^${TABLE_ROW_PREFIX}`,
+      const rows = resp.rows
-        },
-      },
-    }
-    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
-    const { rows } = await dbCore.directCouchFind(db.name, params)
       for (let row of rows) {
         for (let column of attachmentCols) {
           if (!Array.isArray(row[column])) {
@@ -74,9 +96,11 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
           })
         }
       }
-    // write back the updated attachments
+      updatedRows = updatedRows.concat(rows)
-    await db.bulkDocs(rows)
+    } while (rowsLength === FIND_LIMIT)
   }
+  // write back the updated attachments
+  await db.bulkDocs(updatedRows)
 }
 
 async function updateAutomations(prodAppId: string, db: Database) {