From a2d12ef0a6f948e68c9c69c6a6b5bfa4f07498ed Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Oct 2022 17:34:17 +0100 Subject: [PATCH] Adding first pass of attachments updating. --- packages/backend-core/src/db/index.ts | 4 +- .../server/src/sdk/app/backups/constants.ts | 2 +- .../server/src/sdk/app/backups/imports.ts | 84 +++++++++++++------ packages/types/src/documents/app/row.ts | 8 ++ 4 files changed, 71 insertions(+), 27 deletions(-) diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 016155033a..2728b83682 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -32,7 +32,7 @@ const checkInitialised = () => { } } -export async function init(opts: PouchOptions) { +export async function init(opts?: PouchOptions) { PouchDB = pouch.getPouch(opts) initialised = true } @@ -40,7 +40,7 @@ export async function init(opts: PouchOptions) { // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION // this function is prone to leaks, should only be used // in situations that using the function doWithDB does not work -export async function dangerousGetDB(dbName: string, opts: any) { +export async function dangerousGetDB(dbName: string, opts?: any) { checkInitialised() if (env.isTest()) { dbList.add(dbName) diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts index e831172683..f022168846 100644 --- a/packages/server/src/sdk/app/backups/constants.ts +++ b/packages/server/src/sdk/app/backups/constants.ts @@ -1,4 +1,4 @@ -import { ATTACHMENT_DIR as attachmentDir } from "../../constants" +import { ATTACHMENT_DIR as attachmentDir } from "../../../constants" export const DB_EXPORT_FILE = "db.txt" export const ATTACHMENT_DIR = attachmentDir export const GLOBAL_DB_EXPORT_FILE = "global.txt" diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 0ea7eeaa1b..297274e26b 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,5 @@ import { db as dbCore } from "@budibase/backend-core" +import { TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants" import { uploadDirectory } from "../../../utilities/fileSystem/utilities" @@ -6,7 +7,7 @@ import { ObjectStoreBuckets, FieldTypes } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" -import { CouchFindOptions, Row } from "@budibase/types" +import { CouchFindOptions, RowAttachment } from "@budibase/types" const uuid = require("uuid/v4") const tar = require("tar") @@ -18,6 +19,63 @@ type TemplateType = { key?: string } +async function updateAttachmentColumns( + prodAppId: string, + db: PouchDB.Database +) { + // iterate through attachment documents and update them + const tables = await sdk.tables.getAllInternalTables(db) + for (let table of tables) { + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + continue + } + // use the CouchDB Mango query API to lookup rows that have attachments + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${TABLE_ROW_PREFIX}`, + }, + }, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + const { rows } = await dbCore.directCouchFind(db.name, params) + for (let row of rows) { + for (let column of attachmentCols) { + if (!Array.isArray(row[column])) { + continue + } + row[column] = row[column].map((attachment: RowAttachment) => { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(prodAppId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, + } + }) + } + } + // write back the updated attachments + await db.bulkDocs(rows) + } +} + /** * This function manages temporary template files which are stored by Koa. * @param {Object} template The template object retrieved from the Koa context object. @@ -67,28 +125,6 @@ export async function importApp( if (!ok) { throw "Error loading database dump from template." } - // iterate through attachment documents and update them - const tables = await sdk.tables.getAllInternalTables(db) - for (let table of tables) { - const attachmentCols: string[] = [] - for (let [key, column] of Object.entries(table.schema)) { - if (column.type === FieldTypes.ATTACHMENT) { - attachmentCols.push(key) - } - } - // no attachment columns, nothing to do - if (attachmentCols.length === 0) { - continue - } - // use the CouchDB Mango query API to lookup rows that have attachments - const params: CouchFindOptions = { selector: {} } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - const { rows } = await dbCore.directCouchFind(db.name, params) - for (let row of rows) { - // TODO: - } - // write back the updated attachments - await db.bulkDocs(rows) - } + await updateAttachmentColumns(prodAppId, db) return ok } diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index ee5c0231e7..2cac32279b 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -16,6 +16,14 @@ export enum FieldType { INTERNAL = "internal", } +export interface RowAttachment { + size: number + name: string + url: string + extension: string + key: string +} + export interface Row extends Document { type?: string tableId?: string