Some fixes for #8770 - the CLI was very memory intensive when working with large databases, that contained many revisions. We don't need the revisions for app exports/backups, therefore I've updated our export systems to drop the revision history as it compacts the database significantly and speeds up exports/reduces memory usage.

packages/backend-core/package.json

@@ -48,7 +48,7 @@
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
-    "pouchdb-replication-stream": "1.2.9",
+    "@budibase/pouchdb-replication-stream": "1.2.10",
     "redlock": "4.2.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",

packages/backend-core/src/db/couch/pouchDB.ts

@@ -39,7 +39,7 @@ export const getPouch = (opts: PouchOptions = {}) => {
   }
 
   if (opts.replication) {
-    const replicationStream = require("pouchdb-replication-stream")
+    const replicationStream = require("@budibase/pouchdb-replication-stream")
     PouchDB.plugin(replicationStream.plugin)
     // @ts-ignore
     PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)

packages/backend-core/yarn.lock

@@ -310,10 +310,18 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
-"@budibase/types@2.2.12-alpha.62":
-  version "2.2.12-alpha.62"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.62.tgz#385ef000610d5c00b83cb2eafda2bd63c86b7f3f"
-  integrity sha512-idlhB4fSyBCEDWsVvQvdmN9Dg9VAEwxZ8TLE9pGnXIRZPg48MKXPNn5AUT9zv6cDlbQdlU2tFFF8st9b6lyLuw==
+"@budibase/pouchdb-replication-stream@1.2.10":
+  version "1.2.10"
+  resolved "https://registry.yarnpkg.com/@budibase/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.10.tgz#4100df2effd7c823edadddcdbdc380f6827eebf5"
+  integrity sha512-1zeorOwbelZ7HF5vFB+pKE8Mnh31om8k1M6T3AZXVULYTHLsyJrMTozSv5CJ1P8ZfOIJab09HDzCXDh2icFekg==
+  dependencies:
+    argsarray "0.0.1"
+    inherits "^2.0.3"
+    lodash.pick "^4.0.0"
+    ndjson "^1.4.3"
+    pouch-stream "^0.4.0"
+    pouchdb-promise "^6.0.4"
+    through2 "^2.0.0"
 
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"
@@ -4726,19 +4734,6 @@ pouchdb-promise@^6.0.4:
   dependencies:
     lie "3.1.1"
 
-pouchdb-replication-stream@1.2.9:
-  version "1.2.9"
-  resolved "https://registry.yarnpkg.com/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.9.tgz#aa4fa5d8f52df4825392f18e07c7e11acffc650a"
-  integrity sha512-hM8XRBfamTTUwRhKwLS/jSNouBhn9R/4ugdHNRD1EvJzwV8iImh6sDYbCU9PGuznjyOjXz6vpFRzKeI2KYfwnQ==
-  dependencies:
-    argsarray "0.0.1"
-    inherits "^2.0.3"
-    lodash.pick "^4.0.0"
-    ndjson "^1.4.3"
-    pouch-stream "^0.4.0"
-    pouchdb-promise "^6.0.4"
-    through2 "^2.0.0"
-
 pouchdb-selector-core@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.2.2.tgz#264d7436a8c8ac3801f39960e79875ef7f3879a0"

packages/cli/src/backups/utils.js

@@ -74,17 +74,17 @@ exports.getConfig = async (envFile = true) => {
   return config
 }
 
-exports.replication = (from, to) => {
-  return new Promise((resolve, reject) => {
-    from.replicate
-      .to(to)
-      .on("complete", () => {
-        resolve()
-      })
-      .on("error", err => {
-        reject(err)
-      })
+exports.replication = async (from, to) => {
+  const pouch = getPouch()
+  try {
+    await pouch.replicate(from, to, {
+      batch_size: 1000,
+      batch_limit: 5,
+      style: "main_only",
     })
+  } catch (err) {
+    throw new Error(`Replication failed - ${JSON.stringify(err)}`)
+  }
 }
 
 exports.getPouches = config => {

packages/server/src/sdk/app/backups/exports.ts

@@ -45,12 +45,18 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
  * @return {*} either a readable stream or a string
  */
 export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+  const exportOpts = {
+    filter: opts?.filter,
+    batch_size: 1000,
+    batch_limit: 5,
+    style: "main_only",
+  }
   return dbCore.doWithDB(dbName, async (db: any) => {
     // Write the dump to file if required
     if (opts?.exportPath) {
       const path = opts?.exportPath
       const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter: opts?.filter })
+      await db.dump(writeStream, exportOpts)
       return path
     } else {
       // Stringify the dump in memory if required
@@ -59,7 +65,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
       memStream.on("data", (chunk: any) => {
         appString += chunk.toString()
       })
-      await db.dump(memStream, { filter: opts?.filter })
+      await db.dump(memStream, exportOpts)
       return appString
     }
   })