From 0c37c57a4bd704eefc783fd55729512bf166b759 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Jan 2023 18:03:36 +0000 Subject: [PATCH 1/3] Fix for #9249 - there was an issue with the updating of attachment URLs when importing an app/backup, this resolves it by utilising pagination in the CouchDB find API, making sure to iterate through all documents when there is a larger number. --- .../server/src/sdk/app/backups/imports.ts | 96 ++++++++++++------- 1 file changed, 60 insertions(+), 36 deletions(-) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 4c2a721c2c..8a67122594 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,5 +1,5 @@ import { db as dbCore, objectStore } from "@budibase/backend-core" -import { Database } from "@budibase/types" +import { Database, Row } from "@budibase/types" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" @@ -17,6 +17,9 @@ import { const uuid = require("uuid/v4") const tar = require("tar") +// default limit - seems to work well for performance +const FIND_LIMIT = 25 + type TemplateType = { file?: { type: string @@ -25,9 +28,29 @@ type TemplateType = { key?: string } +function generateAttachmentFindParams( + attachmentCols: string[], + bookmark: null | string +) { + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${TABLE_ROW_PREFIX}`, + }, + }, + limit: FIND_LIMIT, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + if (bookmark) { + params.bookmark = bookmark + } + return params +} + async function updateAttachmentColumns(prodAppId: string, db: Database) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) + let updatedRows: Row[] = [] for (let table of tables) { const attachmentCols: string[] = [] for (let [key, column] of Object.entries(table.schema)) { @@ -39,44 +62,45 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) { if (attachmentCols.length === 0) { continue } - // use the CouchDB Mango query API to lookup rows that have attachments - const params: CouchFindOptions = { - selector: { - _id: { - $regex: `^${TABLE_ROW_PREFIX}`, - }, - }, - } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - const { rows } = await dbCore.directCouchFind(db.name, params) - for (let row of rows) { - for (let column of attachmentCols) { - if (!Array.isArray(row[column])) { - continue - } - row[column] = row[column].map((attachment: RowAttachment) => { - // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv - const urlParts = attachment.url.split("/") - // drop the first empty element - urlParts.shift() - // get the prefix - const prefix = urlParts.shift() - // remove the app ID - urlParts.shift() - // add new app ID - urlParts.unshift(prodAppId) - const key = urlParts.join("/") - return { - ...attachment, - key, - url: `/${prefix}/${key}`, + let bookmark: null | string = null, + rowsLength = 0 + do { + const params = generateAttachmentFindParams(attachmentCols, bookmark) + // use the CouchDB Mango query API to lookup rows that have attachments + const resp = await dbCore.directCouchFind(db.name, params) + bookmark = resp.bookmark + rowsLength = resp.rows.length + const rows = resp.rows + for (let row of rows) { + for (let column of attachmentCols) { + if (!Array.isArray(row[column])) { + continue } - }) + row[column] = row[column].map((attachment: RowAttachment) => { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(prodAppId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, + } + }) + } } - } - // write back the updated attachments - await db.bulkDocs(rows) + updatedRows = updatedRows.concat(rows) + } while (rowsLength === FIND_LIMIT) } + // write back the updated attachments + await db.bulkDocs(updatedRows) } async function updateAutomations(prodAppId: string, db: Database) { From cf90282303874c87a3f53f8e5cdf215dfa1d46bb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Jan 2023 12:47:12 +0000 Subject: [PATCH 2/3] Moving some stuff around to make more testable. --- .../server/src/sdk/app/backups/imports.ts | 96 ++++++------------- .../server/src/sdk/app/rows/attachments.ts | 60 ++++++++++++ packages/server/src/sdk/app/rows/index.ts | 5 + packages/server/src/sdk/index.ts | 2 + 4 files changed, 98 insertions(+), 65 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/attachments.ts create mode 100644 packages/server/src/sdk/app/rows/index.ts diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 8a67122594..4893f69588 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -4,22 +4,18 @@ import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" import { downloadTemplate } from "../../../utilities/fileSystem" -import { FieldTypes, ObjectStoreBuckets } from "../../../constants" +import { ObjectStoreBuckets } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" import { Automation, AutomationTriggerStepId, - CouchFindOptions, RowAttachment, } from "@budibase/types" const uuid = require("uuid/v4") const tar = require("tar") -// default limit - seems to work well for performance -const FIND_LIMIT = 25 - type TemplateType = { file?: { type: string @@ -28,23 +24,23 @@ type TemplateType = { key?: string } -function generateAttachmentFindParams( - attachmentCols: string[], - bookmark: null | string -) { - const params: CouchFindOptions = { - selector: { - _id: { - $regex: `^${TABLE_ROW_PREFIX}`, - }, - }, - limit: FIND_LIMIT, +function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(appId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - if (bookmark) { - params.bookmark = bookmark - } - return params } async function updateAttachmentColumns(prodAppId: string, db: Database) { @@ -52,52 +48,22 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) { const tables = await sdk.tables.getAllInternalTables(db) let updatedRows: Row[] = [] for (let table of tables) { - const attachmentCols: string[] = [] - for (let [key, column] of Object.entries(table.schema)) { - if (column.type === FieldTypes.ATTACHMENT) { - attachmentCols.push(key) - } - } - // no attachment columns, nothing to do - if (attachmentCols.length === 0) { - continue - } - let bookmark: null | string = null, - rowsLength = 0 - do { - const params = generateAttachmentFindParams(attachmentCols, bookmark) - // use the CouchDB Mango query API to lookup rows that have attachments - const resp = await dbCore.directCouchFind(db.name, params) - bookmark = resp.bookmark - rowsLength = resp.rows.length - const rows = resp.rows - for (let row of rows) { - for (let column of attachmentCols) { - if (!Array.isArray(row[column])) { - continue + const { rows, columns } = await sdk.rows.getRowsWithAttachments( + db.name, + table + ) + updatedRows = updatedRows.concat( + rows.map(row => { + for (let column of columns) { + if (Array.isArray(row[column])) { + row[column] = row[column].map((attachment: RowAttachment) => + rewriteAttachmentUrl(prodAppId, attachment) + ) } - row[column] = row[column].map((attachment: RowAttachment) => { - // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv - const urlParts = attachment.url.split("/") - // drop the first empty element - urlParts.shift() - // get the prefix - const prefix = urlParts.shift() - // remove the app ID - urlParts.shift() - // add new app ID - urlParts.unshift(prodAppId) - const key = urlParts.join("/") - return { - ...attachment, - key, - url: `/${prefix}/${key}`, - } - }) } - } - updatedRows = updatedRows.concat(rows) - } while (rowsLength === FIND_LIMIT) + return row + }) + ) } // write back the updated attachments await db.bulkDocs(updatedRows) diff --git a/packages/server/src/sdk/app/rows/attachments.ts b/packages/server/src/sdk/app/rows/attachments.ts new file mode 100644 index 0000000000..67f58f8f2c --- /dev/null +++ b/packages/server/src/sdk/app/rows/attachments.ts @@ -0,0 +1,60 @@ +import { CouchFindOptions, Table, Row } from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" +import { DocumentType, SEPARATOR } from "../../../db/utils" +import { FieldTypes } from "../../../constants" + +// default limit - seems to work well for performance +export const FIND_LIMIT = 25 + +function generateAttachmentFindParams( + tableId: string, + attachmentCols: string[], + bookmark: null | string +) { + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`, + }, + }, + limit: FIND_LIMIT, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + if (bookmark) { + params.bookmark = bookmark + } + return params +} + +export async function getRowsWithAttachments(appId: string, table: Table) { + // iterate through attachment documents and update them + const db = dbCore.getDB(appId) + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + return { rows: [], columns: [] } + } + let bookmark: null | string = null, + rowsLength = 0, + rowList: Row[] = [] + do { + const params = generateAttachmentFindParams( + table._id!, + attachmentCols, + bookmark + ) + // use the CouchDB Mango query API to lookup rows that have attachments + const resp = await dbCore.directCouchFind(db.name, params) + bookmark = resp.bookmark + rowsLength = resp.rows.length + const rows = resp.rows + rowList = rowList.concat(rows) + } while (rowsLength === FIND_LIMIT) + // write back the updated attachments + return { rows: rowList, columns: attachmentCols } +} diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts new file mode 100644 index 0000000000..f21928141c --- /dev/null +++ b/packages/server/src/sdk/app/rows/index.ts @@ -0,0 +1,5 @@ +import * as attachments from "./attachments" + +export default { + ...attachments, +} diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 19df8b4388..2ee40992c8 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -2,6 +2,7 @@ import { default as backups } from "./app/backups" import { default as tables } from "./app/tables" import { default as automations } from "./app/automations" import { default as applications } from "./app/applications" +import { default as rows } from "./app/rows" import { default as users } from "./users" const sdk = { @@ -9,6 +10,7 @@ const sdk = { tables, automations, applications, + rows, users, } From 76825c38b9b0232a91be7df49a1a9532ab941fe1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Jan 2023 13:33:22 +0000 Subject: [PATCH 3/3] Adding test cases for the attachment re-write feature, mocking the direct couch find functionality. --- .../src/api/routes/tests/analytics.spec.js | 2 +- .../server/src/sdk/app/backups/imports.ts | 2 +- packages/server/src/sdk/app/rows/index.ts | 2 + packages/server/src/sdk/app/rows/rows.ts | 18 +++++ .../server/src/sdk/tests/attachments.spec.ts | 79 +++++++++++++++++++ 5 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/rows.ts create mode 100644 packages/server/src/sdk/tests/attachments.spec.ts diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js index 51196943e1..f8b3a6763f 100644 --- a/packages/server/src/api/routes/tests/analytics.spec.js +++ b/packages/server/src/api/routes/tests/analytics.spec.js @@ -1,5 +1,5 @@ const setup = require("./utilities") -const { events, constants, db } = require("@budibase/backend-core") +const { events, constants } = require("@budibase/backend-core") describe("/static", () => { let request = setup.getRequest() diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 4893f69588..a5527dcd40 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -43,7 +43,7 @@ function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) { } } -async function updateAttachmentColumns(prodAppId: string, db: Database) { +export async function updateAttachmentColumns(prodAppId: string, db: Database) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) let updatedRows: Row[] = [] diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index f21928141c..12a44ded67 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -1,5 +1,7 @@ import * as attachments from "./attachments" +import * as rows from "./rows" export default { ...attachments, + ...rows, } diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts new file mode 100644 index 0000000000..8709180f0b --- /dev/null +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -0,0 +1,18 @@ +import { db as dbCore, context } from "@budibase/backend-core" +import { Database, Row } from "@budibase/types" +import { getRowParams } from "../../../db/utils" + +export async function getAllInternalRows(appId?: string) { + let db: Database + if (appId) { + db = dbCore.getDB(appId) + } else { + db = context.getAppDB() + } + const response = await db.allDocs( + getRowParams(null, null, { + include_docs: true, + }) + ) + return response.rows.map(row => row.doc) as Row[] +} diff --git a/packages/server/src/sdk/tests/attachments.spec.ts b/packages/server/src/sdk/tests/attachments.spec.ts new file mode 100644 index 0000000000..a340018171 --- /dev/null +++ b/packages/server/src/sdk/tests/attachments.spec.ts @@ -0,0 +1,79 @@ +import newid from "../../db/newid" + +const attachment = { + size: 73479, + name: "2022-12-14 11_11_44-.png", + url: "/prod-budi-app-assets/app_bbb/attachments/a.png", + extension: "png", + key: "app_bbb/attachments/a.png", +} + +const row = { + _id: "ro_ta_aaa", + photo: [attachment], + otherCol: "string", +} + +const table = { + _id: "ta_aaa", + name: "photos", + schema: { + photo: { + type: "attachment", + name: "photo", + }, + otherCol: { + type: "string", + name: "otherCol", + }, + }, +} + +jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + ...core.db, + directCouchFind: jest.fn(), + }, + } +}) + +import { db as dbCore } from "@budibase/backend-core" +import sdk from "../index" + +describe("should be able to re-write attachment URLs", () => { + it("it should update URLs on a number of rows over the limit", async () => { + const db = dbCore.getDB("app_aaa") + await db.put(table) + const limit = 30 + let rows = [] + for (let i = 0; i < limit; i++) { + const rowToWrite = { + ...row, + _id: `${row._id}_${newid()}`, + } + const { rev } = await db.put(rowToWrite) + rows.push({ + ...rowToWrite, + _rev: rev, + }) + } + + dbCore.directCouchFind + // @ts-ignore + .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" }) + .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" }) + await sdk.backups.updateAttachmentColumns(db.name, db) + const finalRows = await sdk.rows.getAllInternalRows(db.name) + for (let rowToCheck of finalRows) { + expect(rowToCheck.otherCol).toBe(row.otherCol) + expect(rowToCheck.photo[0].url).not.toBe(row.photo[0].url) + expect(rowToCheck.photo[0].url).toBe( + `/prod-budi-app-assets/${db.name}/attachments/a.png` + ) + expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`) + } + }) +})