Moving some stuff around to make more testable.

packages/server/src/sdk/app/backups/imports.ts

@@ -4,22 +4,18 @@ import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
+import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
 import {
   Automation,
   AutomationTriggerStepId,
-  CouchFindOptions,
   RowAttachment,
 } from "@budibase/types"
 const uuid = require("uuid/v4")
 const tar = require("tar")
 
-// default limit - seems to work well for performance
-const FIND_LIMIT = 25
-
 type TemplateType = {
   file?: {
     type: string
@@ -28,23 +24,23 @@ type TemplateType = {
   key?: string
 }
 
-function generateAttachmentFindParams(
+function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) {
-  attachmentCols: string[],
+  // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-  bookmark: null | string
+  const urlParts = attachment.url.split("/")
-) {
+  // drop the first empty element
-  const params: CouchFindOptions = {
+  urlParts.shift()
-    selector: {
+  // get the prefix
-      _id: {
+  const prefix = urlParts.shift()
-        $regex: `^${TABLE_ROW_PREFIX}`,
+  // remove the app ID
-      },
+  urlParts.shift()
-    },
+  // add new app ID
-    limit: FIND_LIMIT,
+  urlParts.unshift(appId)
+  const key = urlParts.join("/")
+  return {
+    ...attachment,
+    key,
+    url: `/${prefix}/${key}`,
   }
-  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
-  if (bookmark) {
-    params.bookmark = bookmark
-  }
-  return params
 }
 
 async function updateAttachmentColumns(prodAppId: string, db: Database) {
@@ -52,52 +48,22 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
   const tables = await sdk.tables.getAllInternalTables(db)
   let updatedRows: Row[] = []
   for (let table of tables) {
-    const attachmentCols: string[] = []
+    const { rows, columns } = await sdk.rows.getRowsWithAttachments(
-    for (let [key, column] of Object.entries(table.schema)) {
+      db.name,
-      if (column.type === FieldTypes.ATTACHMENT) {
+      table
-        attachmentCols.push(key)
+    )
-      }
+    updatedRows = updatedRows.concat(
-    }
+      rows.map(row => {
-    // no attachment columns, nothing to do
+        for (let column of columns) {
-    if (attachmentCols.length === 0) {
+          if (Array.isArray(row[column])) {
-      continue
+            row[column] = row[column].map((attachment: RowAttachment) =>
-    }
+              rewriteAttachmentUrl(prodAppId, attachment)
-    let bookmark: null | string = null,
+            )
-      rowsLength = 0
-    do {
-      const params = generateAttachmentFindParams(attachmentCols, bookmark)
-      // use the CouchDB Mango query API to lookup rows that have attachments
-      const resp = await dbCore.directCouchFind(db.name, params)
-      bookmark = resp.bookmark
-      rowsLength = resp.rows.length
-      const rows = resp.rows
-      for (let row of rows) {
-        for (let column of attachmentCols) {
-          if (!Array.isArray(row[column])) {
-            continue
           }
-          row[column] = row[column].map((attachment: RowAttachment) => {
-            // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-            const urlParts = attachment.url.split("/")
-            // drop the first empty element
-            urlParts.shift()
-            // get the prefix
-            const prefix = urlParts.shift()
-            // remove the app ID
-            urlParts.shift()
-            // add new app ID
-            urlParts.unshift(prodAppId)
-            const key = urlParts.join("/")
-            return {
-              ...attachment,
-              key,
-              url: `/${prefix}/${key}`,
-            }
-          })
         }
-      }
+        return row
-      updatedRows = updatedRows.concat(rows)
+      })
-    } while (rowsLength === FIND_LIMIT)
+    )
   }
   // write back the updated attachments
   await db.bulkDocs(updatedRows)

packages/server/src/sdk/app/rows/attachments.ts

@@ -0,0 +1,60 @@
+import { CouchFindOptions, Table, Row } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, SEPARATOR } from "../../../db/utils"
+import { FieldTypes } from "../../../constants"
+
+// default limit - seems to work well for performance
+export const FIND_LIMIT = 25
+
+function generateAttachmentFindParams(
+  tableId: string,
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      _id: {
+        $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
+export async function getRowsWithAttachments(appId: string, table: Table) {
+  // iterate through attachment documents and update them
+  const db = dbCore.getDB(appId)
+  const attachmentCols: string[] = []
+  for (let [key, column] of Object.entries(table.schema)) {
+    if (column.type === FieldTypes.ATTACHMENT) {
+      attachmentCols.push(key)
+    }
+  }
+  // no attachment columns, nothing to do
+  if (attachmentCols.length === 0) {
+    return { rows: [], columns: [] }
+  }
+  let bookmark: null | string = null,
+    rowsLength = 0,
+    rowList: Row[] = []
+  do {
+    const params = generateAttachmentFindParams(
+      table._id!,
+      attachmentCols,
+      bookmark
+    )
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const resp = await dbCore.directCouchFind(db.name, params)
+    bookmark = resp.bookmark
+    rowsLength = resp.rows.length
+    const rows = resp.rows
+    rowList = rowList.concat(rows)
+  } while (rowsLength === FIND_LIMIT)
+  // write back the updated attachments
+  return { rows: rowList, columns: attachmentCols }
+}

packages/server/src/sdk/app/rows/index.ts

@@ -0,0 +1,5 @@
+import * as attachments from "./attachments"
+
+export default {
+  ...attachments,
+}

packages/server/src/sdk/index.ts

@@ -2,6 +2,7 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
 const sdk = {
@@ -9,6 +10,7 @@ const sdk = {
   tables,
   automations,
   applications,
+  rows,
   users,
 }