Merge pull request #3760 from Budibase/fix/all-db-perf

Large number of Couch DBs perf fix

packages/auth/src/db/utils.js

@@ -3,10 +3,15 @@ const Replication = require("./Replication")
 const { DEFAULT_TENANT_ID, Configs } = require("../constants")
 const env = require("../environment")
 const { StaticDatabases, SEPARATOR, DocumentTypes } = require("./constants")
-const { getTenantId, getTenantIDFromAppID } = require("../tenancy")
+const {
+  getTenantId,
+  getTenantIDFromAppID,
+  getGlobalDBName,
+} = require("../tenancy")
 const fetch = require("node-fetch")
 const { getCouch } = require("./index")
 const { getAppMetadata } = require("../cache/appMetadata")
+const { checkSlashesInUrl } = require("../helpers")
 
 const NO_APP_ERROR = "No app provided"
 
@@ -194,6 +199,11 @@ exports.getCouchUrl = () => {
   return `${protocol}://${env.COUCH_DB_USERNAME}:${env.COUCH_DB_PASSWORD}@${rest}`
 }
 
+exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
+  const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
+  return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
+}
+
 /**
  * if in production this will use the CouchDB _all_dbs call to retrieve a list of databases. If testing
  * when using Pouch it will use the pouchdb-all-dbs package.
@@ -203,12 +213,34 @@ exports.getAllDbs = async () => {
   if (env.isTest()) {
     return getCouch().allDbs()
   }
-  const response = await fetch(`${exports.getCouchUrl()}/_all_dbs`)
+  let dbs = []
-  if (response.status === 200) {
+  async function addDbs(url) {
-    return response.json()
+    const response = await fetch(checkSlashesInUrl(encodeURI(url)))
-  } else {
+    if (response.status === 200) {
-    throw "Cannot connect to CouchDB instance"
+      let json = await response.json()
+      dbs = dbs.concat(json)
+    } else {
+      throw "Cannot connect to CouchDB instance"
+    }
   }
+  let couchUrl = `${exports.getCouchUrl()}_all_dbs`
+  if (env.MULTI_TENANCY) {
+    let tenantId = getTenantId()
+    // get prod apps
+    await addDbs(
+      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)
+    )
+    // get dev apps
+    await addDbs(
+      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId)
+    )
+    // add global db name
+    dbs.push(getGlobalDBName(tenantId))
+  } else {
+    // just get all DBs in self host
+    await addDbs(couchUrl)
+  }
+  return dbs
 }
 
 /**
@@ -389,7 +421,7 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
 }
 
 const getPlatformUrl = async settings => {
-  let platformUrl = env.PLATFORM_URL
+  let platformUrl = env.PLATFORM_URL || "http://localhost:10000"
 
   if (!env.SELF_HOSTED && env.MULTI_TENANCY) {
     // cloud and multi tenant - add the tenant to the default platform url
@@ -404,7 +436,7 @@ const getPlatformUrl = async settings => {
     }
   }
 
-  return platformUrl ? platformUrl : "http://localhost:10000"
+  return platformUrl
 }
 
 async function getScopedConfig(db, params) {

packages/auth/src/helpers.js

@@ -0,0 +1,9 @@
+/**
+ * Makes sure that a URL has the correct number of slashes, while maintaining the
+ * http(s):// double slashes.
+ * @param {string} url The URL to test and remove any extra double slashes.
+ * @return {string} The updated url.
+ */
+exports.checkSlashesInUrl = url => {
+  return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
+}