Fix for #9249 - there was an issue with the updating of attachment URLs when importing an app/backup, this resolves it by utilising pagination in the CouchDB find API, making sure to iterate through all documents when there is a larger number.

packages/server/src/sdk/app/backups/imports.ts

@@ -1,5 +1,5 @@
 import { db as dbCore, objectStore } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
+import { Database, Row } from "@budibase/types"
 import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
@@ -17,6 +17,9 @@ import {
 const uuid = require("uuid/v4")
 const tar = require("tar")
 
+// default limit - seems to work well for performance
+const FIND_LIMIT = 25
+
 type TemplateType = {
   file?: {
     type: string
@@ -25,9 +28,29 @@ type TemplateType = {
   key?: string
 }
 
+function generateAttachmentFindParams(
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      _id: {
+        $regex: `^${TABLE_ROW_PREFIX}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
 async function updateAttachmentColumns(prodAppId: string, db: Database) {
   // iterate through attachment documents and update them
   const tables = await sdk.tables.getAllInternalTables(db)
+  let updatedRows: Row[] = []
   for (let table of tables) {
     const attachmentCols: string[] = []
     for (let [key, column] of Object.entries(table.schema)) {
@@ -39,44 +62,45 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
     if (attachmentCols.length === 0) {
       continue
     }
-    // use the CouchDB Mango query API to lookup rows that have attachments
+    let bookmark: null | string = null,
-    const params: CouchFindOptions = {
+      rowsLength = 0
-      selector: {
+    do {
-        _id: {
+      const params = generateAttachmentFindParams(attachmentCols, bookmark)
-          $regex: `^${TABLE_ROW_PREFIX}`,
+      // use the CouchDB Mango query API to lookup rows that have attachments
-        },
+      const resp = await dbCore.directCouchFind(db.name, params)
-      },
+      bookmark = resp.bookmark
-    }
+      rowsLength = resp.rows.length
-    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+      const rows = resp.rows
-    const { rows } = await dbCore.directCouchFind(db.name, params)
+      for (let row of rows) {
-    for (let row of rows) {
+        for (let column of attachmentCols) {
-      for (let column of attachmentCols) {
+          if (!Array.isArray(row[column])) {
-        if (!Array.isArray(row[column])) {
+            continue
-          continue
-        }
-        row[column] = row[column].map((attachment: RowAttachment) => {
-          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-          const urlParts = attachment.url.split("/")
-          // drop the first empty element
-          urlParts.shift()
-          // get the prefix
-          const prefix = urlParts.shift()
-          // remove the app ID
-          urlParts.shift()
-          // add new app ID
-          urlParts.unshift(prodAppId)
-          const key = urlParts.join("/")
-          return {
-            ...attachment,
-            key,
-            url: `/${prefix}/${key}`,
           }
-        })
+          row[column] = row[column].map((attachment: RowAttachment) => {
+            // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+            const urlParts = attachment.url.split("/")
+            // drop the first empty element
+            urlParts.shift()
+            // get the prefix
+            const prefix = urlParts.shift()
+            // remove the app ID
+            urlParts.shift()
+            // add new app ID
+            urlParts.unshift(prodAppId)
+            const key = urlParts.join("/")
+            return {
+              ...attachment,
+              key,
+              url: `/${prefix}/${key}`,
+            }
+          })
+        }
       }
-    }
+      updatedRows = updatedRows.concat(rows)
-    // write back the updated attachments
+    } while (rowsLength === FIND_LIMIT)
-    await db.bulkDocs(rows)
   }
+  // write back the updated attachments
+  await db.bulkDocs(updatedRows)
 }
 
 async function updateAutomations(prodAppId: string, db: Database) {