diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 4c2a721c2c..8a67122594 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,5 +1,5 @@ import { db as dbCore, objectStore } from "@budibase/backend-core" -import { Database } from "@budibase/types" +import { Database, Row } from "@budibase/types" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" @@ -17,6 +17,9 @@ import { const uuid = require("uuid/v4") const tar = require("tar") +// default limit - seems to work well for performance +const FIND_LIMIT = 25 + type TemplateType = { file?: { type: string @@ -25,9 +28,29 @@ type TemplateType = { key?: string } +function generateAttachmentFindParams( + attachmentCols: string[], + bookmark: null | string +) { + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${TABLE_ROW_PREFIX}`, + }, + }, + limit: FIND_LIMIT, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + if (bookmark) { + params.bookmark = bookmark + } + return params +} + async function updateAttachmentColumns(prodAppId: string, db: Database) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) + let updatedRows: Row[] = [] for (let table of tables) { const attachmentCols: string[] = [] for (let [key, column] of Object.entries(table.schema)) { @@ -39,44 +62,45 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) { if (attachmentCols.length === 0) { continue } - // use the CouchDB Mango query API to lookup rows that have attachments - const params: CouchFindOptions = { - selector: { - _id: { - $regex: `^${TABLE_ROW_PREFIX}`, - }, - }, - } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - const { rows } = await dbCore.directCouchFind(db.name, params) - for (let row of rows) { - for (let column of attachmentCols) { - if (!Array.isArray(row[column])) { - continue - } - row[column] = row[column].map((attachment: RowAttachment) => { - // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv - const urlParts = attachment.url.split("/") - // drop the first empty element - urlParts.shift() - // get the prefix - const prefix = urlParts.shift() - // remove the app ID - urlParts.shift() - // add new app ID - urlParts.unshift(prodAppId) - const key = urlParts.join("/") - return { - ...attachment, - key, - url: `/${prefix}/${key}`, + let bookmark: null | string = null, + rowsLength = 0 + do { + const params = generateAttachmentFindParams(attachmentCols, bookmark) + // use the CouchDB Mango query API to lookup rows that have attachments + const resp = await dbCore.directCouchFind(db.name, params) + bookmark = resp.bookmark + rowsLength = resp.rows.length + const rows = resp.rows + for (let row of rows) { + for (let column of attachmentCols) { + if (!Array.isArray(row[column])) { + continue } - }) + row[column] = row[column].map((attachment: RowAttachment) => { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(prodAppId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, + } + }) + } } - } - // write back the updated attachments - await db.bulkDocs(rows) + updatedRows = updatedRows.concat(rows) + } while (rowsLength === FIND_LIMIT) } + // write back the updated attachments + await db.bulkDocs(updatedRows) } async function updateAutomations(prodAppId: string, db: Database) {