From eba18a415994d93f005c0f36fa57e5816fca8412 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Mar 2023 19:01:13 +0000 Subject: [PATCH 1/4] Fix for #9739 - there was an issue with the mango syntax, when working with multi attachment columns it was using an AND comparator instead of OR, it should be searching for rows that contain any attachment column, not all attachment columns. --- packages/backend-core/src/migrations/migrations.ts | 6 +++++- packages/server/src/sdk/app/rows/attachments.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 2e3524775f..48d3b14a4a 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -86,7 +86,11 @@ export const runMigration = async ( count++ const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - const db = getDB(dbName) + const db = getDB(dbName, { skip_setup: true }) + // DB doesn't exist - no-op required + if (!(await db.exists())) { + continue + } try { const doc = await getMigrationsDoc(db) diff --git a/packages/server/src/sdk/app/rows/attachments.ts b/packages/server/src/sdk/app/rows/attachments.ts index 67f58f8f2c..bd04f64146 100644 --- a/packages/server/src/sdk/app/rows/attachments.ts +++ b/packages/server/src/sdk/app/rows/attachments.ts @@ -13,13 +13,13 @@ function generateAttachmentFindParams( ) { const params: CouchFindOptions = { selector: { + $or: attachmentCols.map(col => ({ [col]: { $exists: true } })), _id: { $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`, }, }, limit: FIND_LIMIT, } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) if (bookmark) { params.bookmark = bookmark } From 923e9d8ec462726ec9db3b117e8933eec316727d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Mar 2023 19:17:45 +0000 Subject: [PATCH 2/4] Adding in test check. --- packages/backend-core/src/migrations/migrations.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 48d3b14a4a..43d3e7fe72 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -17,6 +17,7 @@ import { MigrationNoOpOptions, App, } from "@budibase/types" +import env from "../environment" export const getMigrationsDoc = async (db: any) => { // get the migrations doc @@ -88,7 +89,8 @@ export const runMigration = async ( const db = getDB(dbName, { skip_setup: true }) // DB doesn't exist - no-op required - if (!(await db.exists())) { + const dbExists = await db.exists() + if (!env.isTest() && !dbExists) { continue } try { From 5b150e774888f4fcd0387c778a685f0f32bd40cb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Mar 2023 19:43:40 +0000 Subject: [PATCH 3/4] Removing migration change and fixing #9738 - making sure that exports with excludeRows enabled don't include attachments. --- .../backend-core/src/migrations/migrations.ts | 7 +--- .../server/src/sdk/app/backups/constants.ts | 1 + .../server/src/sdk/app/backups/exports.ts | 32 ++++++++++++++----- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 43d3e7fe72..6bc16846e1 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -17,7 +17,6 @@ import { MigrationNoOpOptions, App, } from "@budibase/types" -import env from "../environment" export const getMigrationsDoc = async (db: any) => { // get the migrations doc @@ -88,11 +87,7 @@ export const runMigration = async ( const lengthStatement = length > 1 ? `[${count}/${length}]` : "" const db = getDB(dbName, { skip_setup: true }) - // DB doesn't exist - no-op required - const dbExists = await db.exists() - if (!env.isTest() && !dbExists) { - continue - } + try { const doc = await getMigrationsDoc(db) diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts index 2f011ea2de..0584fcb3d0 100644 --- a/packages/server/src/sdk/app/backups/constants.ts +++ b/packages/server/src/sdk/app/backups/constants.ts @@ -1,2 +1,3 @@ export const DB_EXPORT_FILE = "db.txt" export const GLOBAL_DB_EXPORT_FILE = "global.txt" +export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"] diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 25bd1f4795..c10f106451 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -7,10 +7,15 @@ import { TABLE_ROW_PREFIX, USER_METDATA_PREFIX, } from "../../../db/utils" -import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" +import { + DB_EXPORT_FILE, + GLOBAL_DB_EXPORT_FILE, + STATIC_APP_FILES, +} from "./constants" import fs from "fs" import { join } from "path" import env from "../../../environment" + const uuid = require("uuid/v4") const tar = require("tar") const MemoryStream = require("memorystream") @@ -91,14 +96,25 @@ export async function exportApp(appId: string, config?: ExportOpts) { const prodAppId = dbCore.getProdAppID(appId) const appPath = `${prodAppId}/` // export bucket contents - let tmpPath + let tmpPath = createTempFolder(uuid()) if (!env.isTest()) { - tmpPath = await objectStore.retrieveDirectory( - ObjectStoreBuckets.APPS, - appPath - ) - } else { - tmpPath = createTempFolder(uuid()) + // write just the static files + if (config?.excludeRows) { + for (let path of STATIC_APP_FILES) { + const contents = await objectStore.retrieve( + ObjectStoreBuckets.APPS, + join(appPath, path) + ) + fs.writeFileSync(join(tmpPath, path), contents) + } + } + // get all of the files + else { + tmpPath = await objectStore.retrieveDirectory( + ObjectStoreBuckets.APPS, + appPath + ) + } } const downloadedPath = join(tmpPath, appPath) if (fs.existsSync(downloadedPath)) { From b787fafb32a24afe7d3c8eed996aa6f47d219b6c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Mar 2023 19:45:59 +0000 Subject: [PATCH 4/4] Removing skip setup, tests depend on this behaviour - will update in separate PR. --- packages/backend-core/src/migrations/migrations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 6bc16846e1..ab72091d56 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -86,7 +86,7 @@ export const runMigration = async ( count++ const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - const db = getDB(dbName, { skip_setup: true }) + const db = getDB(dbName) try { const doc = await getMigrationsDoc(db)