Merge pull request #9251 from Budibase/fix/9249

Fix for attachment URL update on import - large number of attachments

packages/server/src/api/routes/tests/analytics.spec.js

@@ -1,5 +1,5 @@
 const setup = require("./utilities")
-const { events, constants, db } = require("@budibase/backend-core")
+const { events, constants } = require("@budibase/backend-core")
 
 describe("/static", () => {
   let request = setup.getRequest()

packages/server/src/sdk/app/backups/imports.ts

@@ -1,17 +1,16 @@
 import { db as dbCore, objectStore } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
+import { Database, Row } from "@budibase/types"
 import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
+import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
 import {
   Automation,
   AutomationTriggerStepId,
-  CouchFindOptions,
   RowAttachment,
 } from "@budibase/types"
 const uuid = require("uuid/v4")
@@ -25,58 +24,49 @@ type TemplateType = {
   key?: string
 }
 
-async function updateAttachmentColumns(prodAppId: string, db: Database) {
+function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) {
+  // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+  const urlParts = attachment.url.split("/")
+  // drop the first empty element
+  urlParts.shift()
+  // get the prefix
+  const prefix = urlParts.shift()
+  // remove the app ID
+  urlParts.shift()
+  // add new app ID
+  urlParts.unshift(appId)
+  const key = urlParts.join("/")
+  return {
+    ...attachment,
+    key,
+    url: `/${prefix}/${key}`,
+  }
+}
+
+export async function updateAttachmentColumns(prodAppId: string, db: Database) {
   // iterate through attachment documents and update them
   const tables = await sdk.tables.getAllInternalTables(db)
+  let updatedRows: Row[] = []
   for (let table of tables) {
-    const attachmentCols: string[] = []
+    const { rows, columns } = await sdk.rows.getRowsWithAttachments(
-    for (let [key, column] of Object.entries(table.schema)) {
+      db.name,
-      if (column.type === FieldTypes.ATTACHMENT) {
+      table
-        attachmentCols.push(key)
+    )
-      }
+    updatedRows = updatedRows.concat(
-    }
+      rows.map(row => {
-    // no attachment columns, nothing to do
+        for (let column of columns) {
-    if (attachmentCols.length === 0) {
+          if (Array.isArray(row[column])) {
-      continue
+            row[column] = row[column].map((attachment: RowAttachment) =>
-    }
+              rewriteAttachmentUrl(prodAppId, attachment)
-    // use the CouchDB Mango query API to lookup rows that have attachments
+            )
-    const params: CouchFindOptions = {
-      selector: {
-        _id: {
-          $regex: `^${TABLE_ROW_PREFIX}`,
-        },
-      },
-    }
-    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
-    const { rows } = await dbCore.directCouchFind(db.name, params)
-    for (let row of rows) {
-      for (let column of attachmentCols) {
-        if (!Array.isArray(row[column])) {
-          continue
-        }
-        row[column] = row[column].map((attachment: RowAttachment) => {
-          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-          const urlParts = attachment.url.split("/")
-          // drop the first empty element
-          urlParts.shift()
-          // get the prefix
-          const prefix = urlParts.shift()
-          // remove the app ID
-          urlParts.shift()
-          // add new app ID
-          urlParts.unshift(prodAppId)
-          const key = urlParts.join("/")
-          return {
-            ...attachment,
-            key,
-            url: `/${prefix}/${key}`,
           }
-        })
+        }
-      }
+        return row
-    }
+      })
-    // write back the updated attachments
+    )
-    await db.bulkDocs(rows)
   }
+  // write back the updated attachments
+  await db.bulkDocs(updatedRows)
 }
 
 async function updateAutomations(prodAppId: string, db: Database) {

packages/server/src/sdk/app/rows/attachments.ts

@@ -0,0 +1,60 @@
+import { CouchFindOptions, Table, Row } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, SEPARATOR } from "../../../db/utils"
+import { FieldTypes } from "../../../constants"
+
+// default limit - seems to work well for performance
+export const FIND_LIMIT = 25
+
+function generateAttachmentFindParams(
+  tableId: string,
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      _id: {
+        $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
+export async function getRowsWithAttachments(appId: string, table: Table) {
+  // iterate through attachment documents and update them
+  const db = dbCore.getDB(appId)
+  const attachmentCols: string[] = []
+  for (let [key, column] of Object.entries(table.schema)) {
+    if (column.type === FieldTypes.ATTACHMENT) {
+      attachmentCols.push(key)
+    }
+  }
+  // no attachment columns, nothing to do
+  if (attachmentCols.length === 0) {
+    return { rows: [], columns: [] }
+  }
+  let bookmark: null | string = null,
+    rowsLength = 0,
+    rowList: Row[] = []
+  do {
+    const params = generateAttachmentFindParams(
+      table._id!,
+      attachmentCols,
+      bookmark
+    )
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const resp = await dbCore.directCouchFind(db.name, params)
+    bookmark = resp.bookmark
+    rowsLength = resp.rows.length
+    const rows = resp.rows
+    rowList = rowList.concat(rows)
+  } while (rowsLength === FIND_LIMIT)
+  // write back the updated attachments
+  return { rows: rowList, columns: attachmentCols }
+}

packages/server/src/sdk/app/rows/index.ts

@@ -0,0 +1,7 @@
+import * as attachments from "./attachments"
+import * as rows from "./rows"
+
+export default {
+  ...attachments,
+  ...rows,
+}

packages/server/src/sdk/app/rows/rows.ts

@@ -0,0 +1,18 @@
+import { db as dbCore, context } from "@budibase/backend-core"
+import { Database, Row } from "@budibase/types"
+import { getRowParams } from "../../../db/utils"
+
+export async function getAllInternalRows(appId?: string) {
+  let db: Database
+  if (appId) {
+    db = dbCore.getDB(appId)
+  } else {
+    db = context.getAppDB()
+  }
+  const response = await db.allDocs(
+    getRowParams(null, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc) as Row[]
+}

packages/server/src/sdk/index.ts

@@ -2,6 +2,7 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
 const sdk = {
@@ -9,6 +10,7 @@ const sdk = {
   tables,
   automations,
   applications,
+  rows,
   users,
 }
 

packages/server/src/sdk/tests/attachments.spec.ts

@@ -0,0 +1,79 @@
+import newid from "../../db/newid"
+
+const attachment = {
+  size: 73479,
+  name: "2022-12-14 11_11_44-.png",
+  url: "/prod-budi-app-assets/app_bbb/attachments/a.png",
+  extension: "png",
+  key: "app_bbb/attachments/a.png",
+}
+
+const row = {
+  _id: "ro_ta_aaa",
+  photo: [attachment],
+  otherCol: "string",
+}
+
+const table = {
+  _id: "ta_aaa",
+  name: "photos",
+  schema: {
+    photo: {
+      type: "attachment",
+      name: "photo",
+    },
+    otherCol: {
+      type: "string",
+      name: "otherCol",
+    },
+  },
+}
+
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    db: {
+      ...core.db,
+      directCouchFind: jest.fn(),
+    },
+  }
+})
+
+import { db as dbCore } from "@budibase/backend-core"
+import sdk from "../index"
+
+describe("should be able to re-write attachment URLs", () => {
+  it("it should update URLs on a number of rows over the limit", async () => {
+    const db = dbCore.getDB("app_aaa")
+    await db.put(table)
+    const limit = 30
+    let rows = []
+    for (let i = 0; i < limit; i++) {
+      const rowToWrite = {
+        ...row,
+        _id: `${row._id}_${newid()}`,
+      }
+      const { rev } = await db.put(rowToWrite)
+      rows.push({
+        ...rowToWrite,
+        _rev: rev,
+      })
+    }
+
+    dbCore.directCouchFind
+      // @ts-ignore
+      .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" })
+      .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" })
+    await sdk.backups.updateAttachmentColumns(db.name, db)
+    const finalRows = await sdk.rows.getAllInternalRows(db.name)
+    for (let rowToCheck of finalRows) {
+      expect(rowToCheck.otherCol).toBe(row.otherCol)
+      expect(rowToCheck.photo[0].url).not.toBe(row.photo[0].url)
+      expect(rowToCheck.photo[0].url).toBe(
+        `/prod-budi-app-assets/${db.name}/attachments/a.png`
+      )
+      expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`)
+    }
+  })
+})