diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js index 51196943e1..f8b3a6763f 100644 --- a/packages/server/src/api/routes/tests/analytics.spec.js +++ b/packages/server/src/api/routes/tests/analytics.spec.js @@ -1,5 +1,5 @@ const setup = require("./utilities") -const { events, constants, db } = require("@budibase/backend-core") +const { events, constants } = require("@budibase/backend-core") describe("/static", () => { let request = setup.getRequest() diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 4c2a721c2c..a5527dcd40 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,17 +1,16 @@ import { db as dbCore, objectStore } from "@budibase/backend-core" -import { Database } from "@budibase/types" +import { Database, Row } from "@budibase/types" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" import { downloadTemplate } from "../../../utilities/fileSystem" -import { FieldTypes, ObjectStoreBuckets } from "../../../constants" +import { ObjectStoreBuckets } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" import { Automation, AutomationTriggerStepId, - CouchFindOptions, RowAttachment, } from "@budibase/types" const uuid = require("uuid/v4") @@ -25,58 +24,49 @@ type TemplateType = { key?: string } -async function updateAttachmentColumns(prodAppId: string, db: Database) { +function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(appId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, + } +} + +export async function updateAttachmentColumns(prodAppId: string, db: Database) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) + let updatedRows: Row[] = [] for (let table of tables) { - const attachmentCols: string[] = [] - for (let [key, column] of Object.entries(table.schema)) { - if (column.type === FieldTypes.ATTACHMENT) { - attachmentCols.push(key) - } - } - // no attachment columns, nothing to do - if (attachmentCols.length === 0) { - continue - } - // use the CouchDB Mango query API to lookup rows that have attachments - const params: CouchFindOptions = { - selector: { - _id: { - $regex: `^${TABLE_ROW_PREFIX}`, - }, - }, - } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - const { rows } = await dbCore.directCouchFind(db.name, params) - for (let row of rows) { - for (let column of attachmentCols) { - if (!Array.isArray(row[column])) { - continue - } - row[column] = row[column].map((attachment: RowAttachment) => { - // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv - const urlParts = attachment.url.split("/") - // drop the first empty element - urlParts.shift() - // get the prefix - const prefix = urlParts.shift() - // remove the app ID - urlParts.shift() - // add new app ID - urlParts.unshift(prodAppId) - const key = urlParts.join("/") - return { - ...attachment, - key, - url: `/${prefix}/${key}`, + const { rows, columns } = await sdk.rows.getRowsWithAttachments( + db.name, + table + ) + updatedRows = updatedRows.concat( + rows.map(row => { + for (let column of columns) { + if (Array.isArray(row[column])) { + row[column] = row[column].map((attachment: RowAttachment) => + rewriteAttachmentUrl(prodAppId, attachment) + ) } - }) - } - } - // write back the updated attachments - await db.bulkDocs(rows) + } + return row + }) + ) } + // write back the updated attachments + await db.bulkDocs(updatedRows) } async function updateAutomations(prodAppId: string, db: Database) { diff --git a/packages/server/src/sdk/app/rows/attachments.ts b/packages/server/src/sdk/app/rows/attachments.ts new file mode 100644 index 0000000000..67f58f8f2c --- /dev/null +++ b/packages/server/src/sdk/app/rows/attachments.ts @@ -0,0 +1,60 @@ +import { CouchFindOptions, Table, Row } from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" +import { DocumentType, SEPARATOR } from "../../../db/utils" +import { FieldTypes } from "../../../constants" + +// default limit - seems to work well for performance +export const FIND_LIMIT = 25 + +function generateAttachmentFindParams( + tableId: string, + attachmentCols: string[], + bookmark: null | string +) { + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`, + }, + }, + limit: FIND_LIMIT, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + if (bookmark) { + params.bookmark = bookmark + } + return params +} + +export async function getRowsWithAttachments(appId: string, table: Table) { + // iterate through attachment documents and update them + const db = dbCore.getDB(appId) + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + return { rows: [], columns: [] } + } + let bookmark: null | string = null, + rowsLength = 0, + rowList: Row[] = [] + do { + const params = generateAttachmentFindParams( + table._id!, + attachmentCols, + bookmark + ) + // use the CouchDB Mango query API to lookup rows that have attachments + const resp = await dbCore.directCouchFind(db.name, params) + bookmark = resp.bookmark + rowsLength = resp.rows.length + const rows = resp.rows + rowList = rowList.concat(rows) + } while (rowsLength === FIND_LIMIT) + // write back the updated attachments + return { rows: rowList, columns: attachmentCols } +} diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts new file mode 100644 index 0000000000..12a44ded67 --- /dev/null +++ b/packages/server/src/sdk/app/rows/index.ts @@ -0,0 +1,7 @@ +import * as attachments from "./attachments" +import * as rows from "./rows" + +export default { + ...attachments, + ...rows, +} diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts new file mode 100644 index 0000000000..8709180f0b --- /dev/null +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -0,0 +1,18 @@ +import { db as dbCore, context } from "@budibase/backend-core" +import { Database, Row } from "@budibase/types" +import { getRowParams } from "../../../db/utils" + +export async function getAllInternalRows(appId?: string) { + let db: Database + if (appId) { + db = dbCore.getDB(appId) + } else { + db = context.getAppDB() + } + const response = await db.allDocs( + getRowParams(null, null, { + include_docs: true, + }) + ) + return response.rows.map(row => row.doc) as Row[] +} diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 19df8b4388..2ee40992c8 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -2,6 +2,7 @@ import { default as backups } from "./app/backups" import { default as tables } from "./app/tables" import { default as automations } from "./app/automations" import { default as applications } from "./app/applications" +import { default as rows } from "./app/rows" import { default as users } from "./users" const sdk = { @@ -9,6 +10,7 @@ const sdk = { tables, automations, applications, + rows, users, } diff --git a/packages/server/src/sdk/tests/attachments.spec.ts b/packages/server/src/sdk/tests/attachments.spec.ts new file mode 100644 index 0000000000..a340018171 --- /dev/null +++ b/packages/server/src/sdk/tests/attachments.spec.ts @@ -0,0 +1,79 @@ +import newid from "../../db/newid" + +const attachment = { + size: 73479, + name: "2022-12-14 11_11_44-.png", + url: "/prod-budi-app-assets/app_bbb/attachments/a.png", + extension: "png", + key: "app_bbb/attachments/a.png", +} + +const row = { + _id: "ro_ta_aaa", + photo: [attachment], + otherCol: "string", +} + +const table = { + _id: "ta_aaa", + name: "photos", + schema: { + photo: { + type: "attachment", + name: "photo", + }, + otherCol: { + type: "string", + name: "otherCol", + }, + }, +} + +jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + ...core.db, + directCouchFind: jest.fn(), + }, + } +}) + +import { db as dbCore } from "@budibase/backend-core" +import sdk from "../index" + +describe("should be able to re-write attachment URLs", () => { + it("it should update URLs on a number of rows over the limit", async () => { + const db = dbCore.getDB("app_aaa") + await db.put(table) + const limit = 30 + let rows = [] + for (let i = 0; i < limit; i++) { + const rowToWrite = { + ...row, + _id: `${row._id}_${newid()}`, + } + const { rev } = await db.put(rowToWrite) + rows.push({ + ...rowToWrite, + _rev: rev, + }) + } + + dbCore.directCouchFind + // @ts-ignore + .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" }) + .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" }) + await sdk.backups.updateAttachmentColumns(db.name, db) + const finalRows = await sdk.rows.getAllInternalRows(db.name) + for (let rowToCheck of finalRows) { + expect(rowToCheck.otherCol).toBe(row.otherCol) + expect(rowToCheck.photo[0].url).not.toBe(row.photo[0].url) + expect(rowToCheck.photo[0].url).toBe( + `/prod-budi-app-assets/${db.name}/attachments/a.png` + ) + expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`) + } + }) +})