Some updates, making sure databases will always close directly using finally checks around the actual tasks, updating how replication works to have a close statement (to make sure it is controlled correctly) and then updating to PouchDB 7.3.0 for one of the memory leak fixes.

packages/backend-core/package.json

@@ -24,6 +24,7 @@
     "passport-google-oauth": "^2.0.0",
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",
+    "pouchdb": "7.3.0",
     "posthog-node": "^1.3.0",
     "pouchdb-find": "^7.2.2",
     "pouchdb-replication-stream": "^1.2.9",
@@ -40,7 +41,6 @@
   "devDependencies": {
     "ioredis-mock": "^5.5.5",
     "jest": "^26.6.3",
-    "pouchdb": "^7.2.1",
     "pouchdb-adapter-memory": "^7.2.2",
     "pouchdb-all-dbs": "^1.0.2"
   },

packages/backend-core/src/context/index.js

@@ -3,7 +3,7 @@ const { Headers } = require("../../constants")
 const { SEPARATOR, DocumentTypes } = require("../db/constants")
 const { DEFAULT_TENANT_ID } = require("../constants")
 const cls = require("./FunctionContext")
-const { dangerousGetDB } = require("../db")
+const { dangerousGetDB, closeDB } = require("../db")
 const { getProdAppID, getDevelopmentAppID } = require("../db/conversions")
 const { baseGlobalDBName } = require("../tenancy/utils")
 const { isEqual } = require("lodash")
@@ -40,11 +40,7 @@ async function closeAppDBs() {
     if (!db) {
       continue
     }
-    try {
-      await db.close()
-    } catch (err) {
-      // ignore error, its already closed likely
-    }
+    await closeDB(db)
   }
 }
 
@@ -67,12 +63,14 @@ exports.doInTenant = (tenantId, task) => {
       exports.setGlobalDB(tenantId)
     }
 
-    // invoke the task
-    const response = await task()
-    if (!opts.existing) {
-      await exports.getGlobalDB().close()
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      if (!opts.existing) {
+        await closeDB(exports.getGlobalDB())
+      }
     }
-    return response
   }
   if (cls.getFromContext(ContextKeys.TENANT_ID) === tenantId) {
     return internal({ existing: true })
@@ -122,12 +120,14 @@ exports.doInAppContext = (appId, task) => {
     }
     // set the app ID
     cls.setOnContext(ContextKeys.APP_ID, appId)
-    // invoke the task
-    const response = await task()
-    if (!opts.existing) {
-      await closeAppDBs()
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      if (!opts.existing) {
+        await closeAppDBs()
+      }
     }
-    return response
   }
   if (appId === cls.getFromContext(ContextKeys.APP_ID)) {
     return internal({ existing: true })
@@ -144,6 +144,7 @@ exports.updateTenantId = tenantId => {
 
 exports.updateAppId = appId => {
   try {
+    // have to close first, before removing the databases from context
     const promise = closeAppDBs()
     cls.setOnContext(ContextKeys.APP_ID, appId)
     cls.setOnContext(ContextKeys.PROD_DB, null)

packages/backend-core/src/db/Replication.js

@@ -1,4 +1,4 @@
-const { dangerousGetDB } = require(".")
+const { dangerousGetDB, closeDB } = require(".")
 
 class Replication {
   /**
@@ -11,6 +11,10 @@ class Replication {
     this.target = dangerousGetDB(target)
   }
 
+  close() {
+    return Promise.all([closeDB(this.source), closeDB(this.target)])
+  }
+
   promisify(operation, opts = {}) {
     return new Promise(resolve => {
       operation(this.target, opts)

packages/backend-core/src/db/index.js

@@ -33,6 +33,19 @@ exports.dangerousGetDB = (dbName, opts) => {
   return db
 }
 
+// use this function if you have called dangerousGetDB - close
+// the databases you've opened once finished
+exports.closeDB = async db => {
+  if (!db) {
+    return
+  }
+  try {
+    return db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}
+
 // we have to use a callback for this so that we can close
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
@@ -41,11 +54,7 @@ exports.doWithDB = async (dbName, cb, opts) => {
   // need this to be async so that we can correctly close DB after all
   // async operations have been completed
   const resp = await cb(db)
-  try {
-    await db.close()
-  } catch (err) {
-    // ignore error - it may have not opened database/is closed already
-  }
+  await exports.closeDB(db)
   return resp
 }
 

packages/backend-core/yarn.lock

@@ -1070,7 +1070,7 @@ buffer-from@1.1.1:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
-buffer-from@^1.0.0:
+buffer-from@1.1.2, buffer-from@^1.0.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
   integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
@@ -1814,6 +1814,13 @@ fetch-cookie@0.10.1:
   dependencies:
     tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
 
+fetch-cookie@0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.11.0.tgz#e046d2abadd0ded5804ce7e2cae06d4331c15407"
+  integrity sha512-BQm7iZLFhMWFy5CZ/162sAGjBfdNWb7a8LEqqnzsHFhxT/X/SVj/z2t2nu3aJvjlbQkrAlTUApplPRjWyH4mhA==
+  dependencies:
+    tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
+
 fill-range@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7"
@@ -3490,7 +3497,7 @@ node-fetch@2.6.0:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
 
-node-fetch@^2.6.1:
+node-fetch@2.6.7, node-fetch@^2.6.1:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
@@ -4056,17 +4063,17 @@ pouchdb-utils@7.2.2:
     pouchdb-md5 "7.2.2"
     uuid "8.1.0"
 
-pouchdb@^7.2.1:
-  version "7.2.2"
-  resolved "https://registry.yarnpkg.com/pouchdb/-/pouchdb-7.2.2.tgz#fcae82862db527e4cf7576ed8549d1384961f364"
-  integrity sha512-5gf5nw5XH/2H/DJj8b0YkvG9fhA/4Jt6kL0Y8QjtztVjb1y4J19Rg4rG+fUbXu96gsUrlyIvZ3XfM0b4mogGmw==
+pouchdb@^7.3.0:
+  version "7.3.0"
+  resolved "https://registry.yarnpkg.com/pouchdb/-/pouchdb-7.3.0.tgz#440fbef12dfd8f9002320802528665e883a3b7f8"
+  integrity sha512-OwsIQGXsfx3TrU1pLruj6PGSwFH+h5k4hGNxFkZ76Um7/ZI8F5TzUHFrpldVVIhfXYi2vP31q0q7ot1FSLFYOw==
   dependencies:
     abort-controller "3.0.0"
     argsarray "0.0.1"
-    buffer-from "1.1.1"
+    buffer-from "1.1.2"
     clone-buffer "1.0.0"
     double-ended-queue "2.1.0-0"
-    fetch-cookie "0.10.1"
+    fetch-cookie "0.11.0"
     immediate "3.3.0"
     inherits "2.0.4"
     level "6.0.1"
@@ -4075,11 +4082,11 @@ pouchdb@^7.2.1:
     leveldown "5.6.0"
     levelup "4.4.0"
     ltgt "2.2.1"
-    node-fetch "2.6.0"
+    node-fetch "2.6.7"
     readable-stream "1.1.14"
-    spark-md5 "3.0.1"
+    spark-md5 "3.0.2"
     through2 "3.0.2"
-    uuid "8.1.0"
+    uuid "8.3.2"
     vuvuzela "1.0.3"
 
 prelude-ls@~1.1.2:
@@ -4628,6 +4635,11 @@ spark-md5@3.0.1:
   resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.1.tgz#83a0e255734f2ab4e5c466e5a2cfc9ba2aa2124d"
   integrity sha512-0tF3AGSD1ppQeuffsLDIOWlKUd3lS92tFxcsrh5Pe3ZphhnoK+oXIBTzOAThZCiuINZLvpiLH/1VS1/ANEJVig==
 
+spark-md5@3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc"
+  integrity sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==
+
 spdx-correct@^3.0.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9"
@@ -5102,6 +5114,11 @@ uuid@8.1.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d"
   integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg==
 
+uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2:
+  version "8.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
+  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
+
 uuid@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
@@ -5112,11 +5129,6 @@ uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
-uuid@^8.3.0, uuid@^8.3.2:
-  version "8.3.2"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
-  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
-
 v8-to-istanbul@^7.0.0:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-7.1.2.tgz#30898d1a7fa0c84d225a2c1434fb958f290883c1"

packages/server/package.json

@@ -117,7 +117,7 @@
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "posthog-node": "^1.1.4",
-    "pouchdb": "7.2.1",
+    "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "^7.2.1",
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "^7.2.2",

packages/server/scripts/replicateApp.js

@@ -1,49 +0,0 @@
-/**
- * Script to replicate your PouchDb (in your home directory) to a remote CouchDB
- * USAGE...
- * node scripts/replicateApp <app_name> <remote_url>
- * e.g. node scripts/replicateApp Mike http://admin:password@127.0.0.1:5984
- */
-
-require("../src/db").init()
-const { DocumentTypes } = require("../src/db/utils")
-const { getAllDbs, dangerousGetDB } = require("@budibase/backend-core/db")
-const appName = process.argv[2].toLowerCase()
-const remoteUrl = process.argv[3]
-
-console.log(`Replicating from ${appName} to ${remoteUrl}/${appName}`)
-
-const run = async () => {
-  const dbs = await getAllDbs()
-  const appDbNames = dbs.filter(dbName => dbName.startsWith("inst_app"))
-  let apps = []
-  for (let dbName of appDbNames) {
-    const db = dangerousGetDB(dbName)
-    apps.push(db.get(DocumentTypes.APP_METADATA))
-  }
-  apps = await Promise.all(apps)
-  const app = apps.find(
-    a => a.name === appName || a.name.toLowerCase() === appName
-  )
-
-  if (!app) {
-    console.log(
-      `Could not find app... apps: ${apps.map(app => app.name).join(", ")}`
-    )
-    return
-  }
-
-  const instanceDb = dangerousGetDB(app.appId)
-  const remoteDb = dangerousGetDB(`${remoteUrl}/${appName}`)
-
-  instanceDb.replicate
-    .to(remoteDb)
-    .on("complete", function () {
-      console.log("SUCCESS!")
-    })
-    .on("error", function (err) {
-      console.log(`FAILED: ${err}`)
-    })
-}
-
-run()

packages/server/src/api/controllers/application.js

@@ -404,6 +404,8 @@ exports.sync = async (ctx, next) => {
     })
   } catch (err) {
     error = err
+  } finally {
+    await replication.close()
   }
 
   // sync the users

packages/server/src/api/controllers/deploy/index.js

@@ -94,18 +94,16 @@ async function initDeployedApp(prodAppId) {
 }
 
 async function deployApp(deployment) {
+  const appId = getAppId()
+  const devAppId = getDevelopmentAppID(appId)
+  const productionAppId = getProdAppID(appId)
+  const replication = new Replication({
+    source: devAppId,
+    target: productionAppId,
+  })
+
   try {
-    const appId = getAppId()
-    const devAppId = getDevelopmentAppID(appId)
-    const productionAppId = getProdAppID(appId)
-
-    const replication = new Replication({
-      source: devAppId,
-      target: productionAppId,
-    })
-
     console.log("Replication object created")
-
     await replication.replicate()
     console.log("replication complete.. replacing app meta doc")
     const db = getProdAppDB()
@@ -126,6 +124,8 @@ async function deployApp(deployment) {
       ...err,
       message: `Deployment Failed: ${err.message}`,
     }
+  } finally {
+    await replication.close()
   }
 }
 

packages/server/src/api/controllers/dev.js

@@ -83,7 +83,9 @@ exports.revert = async ctx => {
   try {
     const db = getProdAppDB({ skip_setup: true })
     const info = await db.info()
-    if (info.error) throw info.error
+    if (info.error) {
+      throw info.error
+    }
     const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
     if (
       !deploymentDoc.history ||
@@ -95,12 +97,11 @@ exports.revert = async ctx => {
     return ctx.throw(400, "App has not yet been deployed")
   }
 
+  const replication = new Replication({
+    source: productionAppId,
+    target: appId,
+  })
   try {
-    const replication = new Replication({
-      source: productionAppId,
-      target: appId,
-    })
-
     await replication.rollback()
     // update appID in reverted app to be dev version again
     const db = getAppDB()
@@ -114,6 +115,8 @@ exports.revert = async ctx => {
     }
   } catch (err) {
     ctx.throw(400, `Unable to revert. ${err}`)
+  } finally {
+    await replication.close()
   }
 }
 

packages/server/src/db/inMemoryView.js

@@ -2,7 +2,7 @@ const newid = require("./newid")
 
 // bypass the main application db config
 // use in memory pouchdb directly
-const { getPouch } = require("@budibase/backend-core/db")
+const { getPouch, closeDB } = require("@budibase/backend-core/db")
 const Pouch = getPouch({ inMemory: true })
 
 exports.runView = async (view, calculation, group, data) => {
@@ -10,37 +10,40 @@ exports.runView = async (view, calculation, group, data) => {
   // are always unique for each query, don't want overlap
   // which could cause 409s
   const db = new Pouch(newid())
-  // write all the docs to the in memory Pouch (remove revs)
-  await db.bulkDocs(
-    data.map(row => ({
-      ...row,
-      _rev: undefined,
-    }))
-  )
-  let fn = (doc, emit) => emit(doc._id)
-  eval("fn = " + view.map.replace("function (doc)", "function (doc, emit)"))
-  const queryFns = {
-    meta: view.meta,
-    map: fn,
-  }
-  if (view.reduce) {
-    queryFns.reduce = view.reduce
-  }
-  const response = await db.query(queryFns, {
-    include_docs: !calculation,
-    group: !!group,
-  })
-  // need to fix the revs to be totally accurate
-  for (let row of response.rows) {
-    if (!row._rev || !row._id) {
-      continue
+  try {
+    // write all the docs to the in memory Pouch (remove revs)
+    await db.bulkDocs(
+      data.map(row => ({
+        ...row,
+        _rev: undefined,
+      }))
+    )
+    let fn = (doc, emit) => emit(doc._id)
+    eval("fn = " + view.map.replace("function (doc)", "function (doc, emit)"))
+    const queryFns = {
+      meta: view.meta,
+      map: fn,
     }
-    const found = data.find(possible => possible._id === row._id)
-    if (found) {
-      row._rev = found._rev
+    if (view.reduce) {
+      queryFns.reduce = view.reduce
     }
+    const response = await db.query(queryFns, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+    // need to fix the revs to be totally accurate
+    for (let row of response.rows) {
+      if (!row._rev || !row._id) {
+        continue
+      }
+      const found = data.find(possible => possible._id === row._id)
+      if (found) {
+        row._rev = found._rev
+      }
+    }
+    return response
+  } finally {
+    await db.destroy()
+    await closeDB(db)
   }
-  await db.destroy()
-  await db.close()
-  return response
 }

packages/worker/package.json

@@ -55,7 +55,7 @@
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",
     "pino-pretty": "^4.0.0",
-    "pouchdb": "^7.2.1",
+    "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "^1.0.2",
     "server-destroy": "^1.0.1"
   },