merge

.github/workflows/deploy-cloud.yaml

@@ -66,7 +66,7 @@ jobs:
           config-files: values.production.yaml
           chart-path: charts/budibase
           namespace: budibase
-          values: globals.appVersion=v${{ env.RELEASE_VERSION }}
+          values: globals.appVersion=v${{ env.RELEASE_VERSION }},services.couchdb.url=${{ secrets.PRODUCTION_COUCHDB_URL }},services.couchdb.password=${{ secrets.PRODUCTION_COUCHDB_PASSWORD }}
           name: budibase-prod
 
       - name: Discord Webhook Action

.github/workflows/release.yml

@@ -53,8 +53,8 @@ jobs:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
-          git config user.name "Budibase Release Bot"
+          git config --global user.name "Budibase Release Bot"
-          git config user.email "<>"
+          git config --global user.email "<>"
           echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
           yarn release
 

charts/budibase/templates/minio-data-persistentvolumeclaim.yaml

@@ -12,5 +12,8 @@ spec:
   resources:
     requests:
       storage: {{ .Values.services.objectStore.storage }}
+  {{ if .Values.services.objectStore.storageClass }}
+  storageClassName: {{ .Values.services.objectStore.storageClass }}
+  {{- end }}
 status: {}
 {{- end }}

charts/budibase/templates/redis-data-persistentvolumeclaim.yaml

@@ -12,5 +12,8 @@ spec:
   resources:
     requests:
       storage: {{ .Values.services.redis.storage }}
+  {{ if .Values.services.redis.storageClass }}
+  storageClassName: {{ .Values.services.redis.storageClass }}
+  {{ end }}
 status: {}
 {{- end }}

charts/budibase/values.yaml

@@ -47,6 +47,8 @@ ingress:
   className: ""
   annotations:
     kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/client-max-body-size: 150M
+    nginx.ingress.kubernetes.io/proxy-body-size: 50m
   hosts:
     - host: # change if using custom domain
       paths:
@@ -149,6 +151,11 @@ services:
     url: "" # only change if pointing to existing redis cluster and enabled: false
     password: "budibase" # recommended to override if using built-in redis
     storage: 100Mi
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.
+    storageClass: ""
 
   objectStore:
     minio: true
@@ -160,6 +167,11 @@ services:
     region: "" # AWS_REGION if using S3 or existing minio secret
     url: "http://minio-service:9000" # only change if pointing to existing minio cluster or S3 and minio: false
     storage: 100Mi
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.
+    storageClass: ""
 
 # Override values in couchDB subchart
 couchdb:

hosting/nginx.prod.conf.hbs

@@ -48,7 +48,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com";
+    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com";
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -74,6 +74,7 @@
     "mode:account": "yarn mode:cloud && yarn env:account:enable",
     "security:audit": "node scripts/audit.js",
     "postinstall": "husky install",
-    "install:pro": "bash scripts/pro/install.sh"
+    "install:pro": "bash scripts/pro/install.sh",
+    "dep:clean": "yarn clean && yarn bootstrap"
   }
 }

packages/backend-core/db.js

@@ -3,4 +3,5 @@ module.exports = {
   ...require("./src/db/constants"),
   ...require("./src/db"),
   ...require("./src/db/views"),
+  ...require("./src/db/pouch"),
 }

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",
@@ -24,6 +24,10 @@
     "passport-google-oauth": "^2.0.0",
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",
+    "posthog-node": "^1.3.0",
+    "pouchdb": "7.3.0",
+    "pouchdb-find": "^7.2.2",
+    "pouchdb-replication-stream": "^1.2.9",
     "sanitize-s3-objectkey": "^0.0.1",
     "tar-fs": "^2.1.1",
     "uuid": "^8.3.2",
@@ -37,7 +41,6 @@
   "devDependencies": {
     "ioredis-mock": "^5.5.5",
     "jest": "^26.6.3",
-    "pouchdb": "^7.2.1",
     "pouchdb-adapter-memory": "^7.2.2",
     "pouchdb-all-dbs": "^1.0.2"
   },

packages/backend-core/src/cache/appMetadata.js

@@ -1,5 +1,5 @@
 const redis = require("../redis/authRedis")
-const { getCouch } = require("../db")
+const { doWithDB } = require("../db")
 const { DocumentTypes } = require("../db/constants")
 
 const AppState = {
@@ -10,12 +10,14 @@ const EXPIRY_SECONDS = 3600
 /**
  * The default populate app metadata function
  */
-const populateFromDB = async (appId, CouchDB = null) => {
+const populateFromDB = async appId => {
-  if (!CouchDB) {
+  return doWithDB(
-    CouchDB = getCouch()
+    appId,
-  }
+    db => {
-  const db = new CouchDB(appId, { skip_setup: true })
       return db.get(DocumentTypes.APP_METADATA)
+    },
+    { skip_setup: true }
+  )
 }
 
 const isInvalid = metadata => {
@@ -27,17 +29,16 @@ const isInvalid = metadata => {
  * Use redis cache to first read the app metadata.
  * If not present fallback to loading the app metadata directly and re-caching.
  * @param {string} appId the id of the app to get metadata from.
- * @param {object} CouchDB the database being passed
  * @returns {object} the app metadata.
  */
-exports.getAppMetadata = async (appId, CouchDB = null) => {
+exports.getAppMetadata = async appId => {
   const client = await redis.getAppClient()
   // try cache
   let metadata = await client.get(appId)
   if (!metadata) {
     let expiry = EXPIRY_SECONDS
     try {
-      metadata = await populateFromDB(appId, CouchDB)
+      metadata = await populateFromDB(appId)
     } catch (err) {
       // app DB left around, but no metadata, it is invalid
       if (err && err.status === 404) {

packages/backend-core/src/cache/user.js

@@ -1,5 +1,5 @@
 const redis = require("../redis/authRedis")
-const { getTenantId, lookupTenantId, getGlobalDB } = require("../tenancy")
+const { getTenantId, lookupTenantId, doWithGlobalDB } = require("../tenancy")
 const env = require("../environment")
 const accounts = require("../cloud/accounts")
 
@@ -9,9 +9,8 @@ const EXPIRY_SECONDS = 3600
  * The default populate user function
  */
 const populateFromDB = async (userId, tenantId) => {
-  const user = await getGlobalDB(tenantId).get(userId)
+  const user = await doWithGlobalDB(tenantId, db => db.get(userId))
   user.budibaseAccess = true
-
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
     if (account) {

packages/backend-core/src/cloud/api.js

@@ -29,9 +29,7 @@ class API {
         credentials: "include",
       }
 
-      const resp = await fetch(`${this.host}${url}`, requestOptions)
+      return await fetch(`${this.host}${url}`, requestOptions)
-
-      return resp
     }
 
   post = this.apiCall("POST")

packages/backend-core/src/context/FunctionContext.js

@@ -4,7 +4,11 @@ const { newid } = require("../hashing")
 const REQUEST_ID_KEY = "requestId"
 
 class FunctionContext {
-  static getMiddleware(updateCtxFn = null, contextName = "session") {
+  static getMiddleware(
+    updateCtxFn = null,
+    destroyFn = null,
+    contextName = "session"
+  ) {
     const namespace = this.createNamespace(contextName)
 
     return async function (ctx, next) {
@@ -18,7 +22,14 @@ class FunctionContext {
           if (updateCtxFn) {
             updateCtxFn(ctx)
           }
-          next().then(resolve).catch(reject)
+          next()
+            .then(resolve)
+            .catch(reject)
+            .finally(() => {
+              if (destroyFn) {
+                return destroyFn(ctx)
+              }
+            })
         })
       )
     }

packages/backend-core/src/context/deprovision.js

@@ -1,6 +1,6 @@
 const { getGlobalUserParams, getAllApps } = require("../db/utils")
-const { getDB } = require("../db")
+const { doWithDB } = require("../db")
-const { getGlobalDB } = require("../tenancy")
+const { doWithGlobalDB } = require("../tenancy")
 const { StaticDatabases } = require("../db/constants")
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
@@ -8,11 +8,12 @@ const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
 const removeTenantFromInfoDB = async tenantId => {
   try {
-    const infoDb = getDB(PLATFORM_INFO_DB)
+    await doWithDB(PLATFORM_INFO_DB, async infoDb => {
       let tenants = await infoDb.get(TENANT_DOC)
       tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
 
       await infoDb.put(tenants)
+    })
   } catch (err) {
     console.error(`Error removing tenant ${tenantId} from info db`, err)
     throw err
@@ -20,7 +21,7 @@ const removeTenantFromInfoDB = async tenantId => {
 }
 
 exports.removeUserFromInfoDB = async dbUser => {
-  const infoDb = getDB(PLATFORM_INFO_DB)
+  await doWithDB(PLATFORM_INFO_DB, async infoDb => {
     const keys = [dbUser._id, dbUser.email]
     const userDocs = await infoDb.allDocs({
       keys,
@@ -33,17 +34,18 @@ exports.removeUserFromInfoDB = async dbUser => {
       }
     })
     await infoDb.bulkDocs(toDelete)
+  })
 }
 
 const removeUsersFromInfoDB = async tenantId => {
+  return doWithGlobalDB(tenantId, async db => {
     try {
-    const globalDb = getGlobalDB(tenantId)
+      const allUsers = await db.allDocs(
-    const infoDb = getDB(PLATFORM_INFO_DB)
-    const allUsers = await globalDb.allDocs(
         getGlobalUserParams(null, {
           include_docs: true,
         })
       )
+      await doWithDB(PLATFORM_INFO_DB, async infoDb => {
         const allEmails = allUsers.rows.map(row => row.doc.email)
         // get the id docs
         let keys = allUsers.rows.map(row => row.id)
@@ -61,26 +63,31 @@ const removeUsersFromInfoDB = async tenantId => {
           }
         })
         await infoDb.bulkDocs(toDelete)
+      })
     } catch (err) {
       console.error(`Error removing tenant ${tenantId} users from info db`, err)
       throw err
     }
+  })
 }
 
 const removeGlobalDB = async tenantId => {
+  return doWithGlobalDB(tenantId, async db => {
     try {
-    const globalDb = getGlobalDB(tenantId)
+      await db.destroy()
-    await globalDb.destroy()
     } catch (err) {
       console.error(`Error removing tenant ${tenantId} users from info db`, err)
       throw err
     }
+  })
 }
 
 const removeTenantApps = async tenantId => {
   try {
     const apps = await getAllApps({ all: true })
-    const destroyPromises = apps.map(app => getDB(app.appId).destroy())
+    const destroyPromises = apps.map(app =>
+      doWithDB(app.appId, db => db.destroy())
+    )
     await Promise.allSettled(destroyPromises)
   } catch (err) {
     console.error(`Error removing tenant ${tenantId} apps`, err)

packages/backend-core/src/context/index.js

@@ -1,9 +1,11 @@
 const env = require("../environment")
 const { Headers } = require("../../constants")
 const { SEPARATOR, DocumentTypes } = require("../db/constants")
+const { DEFAULT_TENANT_ID } = require("../constants")
 const cls = require("./FunctionContext")
-const { getCouch } = require("../db")
+const { dangerousGetDB, closeDB } = require("../db")
 const { getProdAppID, getDevelopmentAppID } = require("../db/conversions")
+const { baseGlobalDBName } = require("../tenancy/utils")
 const { isEqual } = require("lodash")
 
 // some test cases call functions directly, need to
@@ -12,6 +14,7 @@ let TEST_APP_ID = null
 
 const ContextKeys = {
   TENANT_ID: "tenantId",
+  GLOBAL_DB: "globalDb",
   APP_ID: "appId",
   // whatever the request app DB was
   CURRENT_DB: "currentDb",
@@ -20,9 +23,37 @@ const ContextKeys = {
   // get the dev app DB from the request
   DEV_DB: "devDb",
   DB_OPTS: "dbOpts",
+  // check if something else is using the context, don't close DB
+  IN_USE: "inUse",
 }
 
-exports.DEFAULT_TENANT_ID = "default"
+exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID
+
+// this function makes sure the PouchDB objects are closed and
+// fully deleted when finished - this protects against memory leaks
+async function closeAppDBs() {
+  const dbKeys = [
+    ContextKeys.CURRENT_DB,
+    ContextKeys.PROD_DB,
+    ContextKeys.DEV_DB,
+  ]
+  for (let dbKey of dbKeys) {
+    const db = cls.getFromContext(dbKey)
+    if (!db) {
+      continue
+    }
+    await closeDB(db)
+    // clear the DB from context, incase someone tries to use it again
+    cls.setOnContext(dbKey, null)
+  }
+  // clear the app ID now that the databases are closed
+  if (cls.getFromContext(ContextKeys.APP_ID)) {
+    cls.setOnContext(ContextKeys.APP_ID, null)
+  }
+  if (cls.getFromContext(ContextKeys.DB_OPTS)) {
+    cls.setOnContext(ContextKeys.DB_OPTS, null)
+  }
+}
 
 exports.isDefaultTenant = () => {
   return exports.getTenantId() === exports.DEFAULT_TENANT_ID
@@ -34,14 +65,45 @@ exports.isMultiTenant = () => {
 
 // used for automations, API endpoints should always be in context already
 exports.doInTenant = (tenantId, task) => {
-  return cls.run(() => {
+  // the internal function is so that we can re-use an existing
+  // context - don't want to close DB on a parent context
+  async function internal(opts = { existing: false }) {
     // set the tenant id
+    if (!opts.existing) {
       cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+      if (env.USE_COUCH) {
+        exports.setGlobalDB(tenantId)
+      }
+    }
 
+    try {
       // invoke the task
-    return task()
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        if (env.USE_COUCH) {
+          await closeDB(exports.getGlobalDB())
+        }
+        // clear from context now that database is closed/task is finished
+        cls.setOnContext(ContextKeys.TENANT_ID, null)
+        cls.setOnContext(ContextKeys.GLOBAL_DB, null)
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (using && cls.getFromContext(ContextKeys.TENANT_ID) === tenantId) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal()
     })
   }
+}
 
 /**
  * Given an app ID this will attempt to retrieve the tenant ID from it.
@@ -64,37 +126,59 @@ exports.getTenantIDFromAppID = appId => {
 }
 
 const setAppTenantId = appId => {
-  const appTenantId = this.getTenantIDFromAppID(appId) || this.DEFAULT_TENANT_ID
+  const appTenantId =
-  this.updateTenantId(appTenantId)
+    exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID
+  exports.updateTenantId(appTenantId)
 }
 
 exports.doInAppContext = (appId, task) => {
   if (!appId) {
     throw new Error("appId is required")
   }
-  return cls.run(() => {
-    // set the app tenant id
-    setAppTenantId(appId)
 
+  // the internal function is so that we can re-use an existing
+  // context - don't want to close DB on a parent context
+  async function internal(opts = { existing: false }) {
+    // set the app tenant id
+    if (!opts.existing) {
+      setAppTenantId(appId)
+    }
     // set the app ID
     cls.setOnContext(ContextKeys.APP_ID, appId)
-
+    try {
       // invoke the task
-    return task()
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        await closeAppDBs()
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (using && cls.getFromContext(ContextKeys.APP_ID) === appId) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal()
     })
   }
+}
 
 exports.updateTenantId = tenantId => {
   cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+  exports.setGlobalDB(tenantId)
 }
 
-exports.updateAppId = appId => {
+exports.updateAppId = async appId => {
   try {
+    // have to close first, before removing the databases from context
+    await closeAppDBs()
     cls.setOnContext(ContextKeys.APP_ID, appId)
-    cls.setOnContext(ContextKeys.PROD_DB, null)
-    cls.setOnContext(ContextKeys.DEV_DB, null)
-    cls.setOnContext(ContextKeys.CURRENT_DB, null)
-    cls.setOnContext(ContextKeys.DB_OPTS, null)
   } catch (err) {
     if (env.isTest()) {
       TEST_APP_ID = appId
@@ -111,8 +195,8 @@ exports.setTenantId = (
   let tenantId
   // exit early if not multi-tenant
   if (!exports.isMultiTenant()) {
-    cls.setOnContext(ContextKeys.TENANT_ID, this.DEFAULT_TENANT_ID)
+    cls.setOnContext(ContextKeys.TENANT_ID, exports.DEFAULT_TENANT_ID)
-    return
+    return exports.DEFAULT_TENANT_ID
   }
 
   const allowQs = opts && opts.allowQs
@@ -140,6 +224,22 @@ exports.setTenantId = (
   if (tenantId) {
     cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
   }
+  return tenantId
+}
+
+exports.setGlobalDB = tenantId => {
+  const dbName = baseGlobalDBName(tenantId)
+  const db = dangerousGetDB(dbName)
+  cls.setOnContext(ContextKeys.GLOBAL_DB, db)
+  return db
+}
+
+exports.getGlobalDB = () => {
+  const db = cls.getFromContext(ContextKeys.GLOBAL_DB)
+  if (!db) {
+    throw new Error("Global DB not found")
+  }
+  return db
 }
 
 exports.isTenantIdSet = () => {
@@ -167,16 +267,17 @@ exports.getAppId = () => {
   }
 }
 
-function getDB(key, opts) {
+function getContextDB(key, opts) {
   const dbOptsKey = `${key}${ContextKeys.DB_OPTS}`
   let storedOpts = cls.getFromContext(dbOptsKey)
   let db = cls.getFromContext(key)
   if (db && isEqual(opts, storedOpts)) {
     return db
   }
+
   const appId = exports.getAppId()
-  const CouchDB = getCouch()
   let toUseAppId
+
   switch (key) {
     case ContextKeys.CURRENT_DB:
       toUseAppId = appId
@@ -188,7 +289,7 @@ function getDB(key, opts) {
       toUseAppId = getDevelopmentAppID(appId)
       break
   }
-  db = new CouchDB(toUseAppId, opts)
+  db = dangerousGetDB(toUseAppId, opts)
   try {
     cls.setOnContext(key, db)
     if (opts) {
@@ -207,7 +308,7 @@ function getDB(key, opts) {
  * contained, dev or prod.
  */
 exports.getAppDB = opts => {
-  return getDB(ContextKeys.CURRENT_DB, opts)
+  return getContextDB(ContextKeys.CURRENT_DB, opts)
 }
 
 /**
@@ -215,7 +316,7 @@ exports.getAppDB = opts => {
  * contained a development app ID, this will open the prod one.
  */
 exports.getProdAppDB = opts => {
-  return getDB(ContextKeys.PROD_DB, opts)
+  return getContextDB(ContextKeys.PROD_DB, opts)
 }
 
 /**
@@ -223,5 +324,5 @@ exports.getProdAppDB = opts => {
  * contained a prod app ID, this will open the dev one.
  */
 exports.getDevAppDB = opts => {
-  return getDB(ContextKeys.DEV_DB, opts)
+  return getContextDB(ContextKeys.DEV_DB, opts)
 }

packages/backend-core/src/db/Replication.js

@@ -1,4 +1,4 @@
-const { getDB } = require(".")
+const { dangerousGetDB, closeDB } = require(".")
 
 class Replication {
   /**
@@ -7,8 +7,12 @@ class Replication {
    * @param {String} target - the DB you want to replicate to, or rollback from
    */
   constructor({ source, target }) {
-    this.source = getDB(source)
+    this.source = dangerousGetDB(source)
-    this.target = getDB(target)
+    this.target = dangerousGetDB(target)
+  }
+
+  close() {
+    return Promise.all([closeDB(this.source), closeDB(this.target)])
   }
 
   promisify(operation, opts = {}) {
@@ -51,7 +55,7 @@ class Replication {
   async rollback() {
     await this.target.destroy()
     // Recreate the DB again
-    this.target = getDB(this.target.name)
+    this.target = dangerousGetDB(this.target.name)
     await this.replicate()
   }
 

packages/backend-core/src/db/index.js

@@ -1,13 +1,68 @@
-let Pouch
+const pouch = require("./pouch")
+const env = require("../environment")
 
-module.exports.setDB = pouch => {
+let PouchDB
-  Pouch = pouch
+let initialised = false
+
+const put =
+  dbPut =>
+  async (doc, options = {}) => {
+    const response = await dbPut(doc, options)
+    // TODO: add created / updated
+    return response
   }
 
-module.exports.getDB = dbName => {
+const checkInitialised = () => {
-  return new Pouch(dbName)
+  if (!initialised) {
+    throw new Error("init has not been called")
+  }
 }
 
-module.exports.getCouch = () => {
+exports.init = opts => {
-  return Pouch
+  PouchDB = pouch.getPouch(opts)
+  initialised = true
+}
+
+// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
+// this function is prone to leaks, should only be used
+// in situations that using the function doWithDB does not work
+exports.dangerousGetDB = (dbName, opts) => {
+  checkInitialised()
+  const db = new PouchDB(dbName, opts)
+  const dbPut = db.put
+  db.put = put(dbPut)
+  return db
+}
+
+// use this function if you have called dangerousGetDB - close
+// the databases you've opened once finished
+exports.closeDB = async db => {
+  if (!db || env.isTest()) {
+    return
+  }
+  try {
+    // specifically await so that if there is an error, it can be ignored
+    return await db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}
+
+// we have to use a callback for this so that we can close
+// the DB when we're done, without this manual requests would
+// need to close the database when done with it to avoid memory leaks
+exports.doWithDB = async (dbName, cb, opts) => {
+  const db = exports.dangerousGetDB(dbName, opts)
+  // need this to be async so that we can correctly close DB after all
+  // async operations have been completed
+  try {
+    return await cb(db)
+  } finally {
+    await exports.closeDB(db)
+  }
+}
+
+exports.allDbs = () => {
+  checkInitialised()
+  return PouchDB.allDbs()
 }

packages/backend-core/src/db/pouch.js

@@ -0,0 +1,102 @@
+const PouchDB = require("pouchdb")
+const env = require("../environment")
+
+function getUrlInfo() {
+  let url = env.COUCH_DB_URL
+  let username, password, host
+  const [protocol, rest] = url.split("://")
+  if (url.includes("@")) {
+    const hostParts = rest.split("@")
+    host = hostParts[1]
+    const authParts = hostParts[0].split(":")
+    username = authParts[0]
+    password = authParts[1]
+  } else {
+    host = rest
+  }
+  return {
+    url: `${protocol}://${host}`,
+    auth: {
+      username,
+      password,
+    },
+  }
+}
+
+exports.getCouchInfo = () => {
+  const urlInfo = getUrlInfo()
+  let username
+  let password
+  if (env.COUCH_DB_USERNAME) {
+    // set from env
+    username = env.COUCH_DB_USERNAME
+  } else if (urlInfo.auth.username) {
+    // set from url
+    username = urlInfo.auth.username
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB username not set")
+  }
+  if (env.COUCH_DB_PASSWORD) {
+    // set from env
+    password = env.COUCH_DB_PASSWORD
+  } else if (urlInfo.auth.password) {
+    // set from url
+    password = urlInfo.auth.password
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB password not set")
+  }
+  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
+  return {
+    url: urlInfo.url,
+    auth: {
+      username: username,
+      password: password,
+    },
+    cookie: `Basic ${authCookie}`,
+  }
+}
+
+/**
+ * Return a constructor for PouchDB.
+ * This should be rarely used outside of the main application config.
+ * Exposed for exceptional cases such as in-memory views.
+ */
+exports.getPouch = (opts = {}) => {
+  let { url, cookie } = exports.getCouchInfo()
+  let POUCH_DB_DEFAULTS = {
+    prefix: url,
+    fetch: (url, opts) => {
+      // use a specific authorization cookie - be very explicit about how we authenticate
+      opts.headers.set("Authorization", cookie)
+      return PouchDB.fetch(url, opts)
+    },
+  }
+
+  if (opts.inMemory) {
+    const inMemory = require("pouchdb-adapter-memory")
+    PouchDB.plugin(inMemory)
+    POUCH_DB_DEFAULTS = {
+      prefix: undefined,
+      adapter: "memory",
+    }
+  }
+
+  if (opts.replication) {
+    const replicationStream = require("pouchdb-replication-stream")
+    PouchDB.plugin(replicationStream.plugin)
+    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+  }
+
+  if (opts.find) {
+    const find = require("pouchdb-find")
+    PouchDB.plugin(find)
+  }
+
+  const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS)
+  if (opts.allDbs) {
+    const allDbs = require("pouchdb-all-dbs")
+    allDbs(Pouch)
+  }
+
+  return Pouch
+}

packages/backend-core/src/db/utils.js

@@ -11,7 +11,8 @@ const {
 } = require("./constants")
 const { getTenantId, getGlobalDBName } = require("../tenancy")
 const fetch = require("node-fetch")
-const { getCouch } = require("./index")
+const { doWithDB, allDbs } = require("./index")
+const { getCouchInfo } = require("./pouch")
 const { getAppMetadata } = require("../cache/appMetadata")
 const { checkSlashesInUrl } = require("../helpers")
 const {
@@ -150,25 +151,6 @@ exports.getRoleParams = (roleId = null, otherProps = {}) => {
   return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
 }
 
-exports.getCouchUrl = () => {
-  if (!env.COUCH_DB_URL) return
-
-  // username and password already exist in URL
-  if (env.COUCH_DB_URL.includes("@")) {
-    return env.COUCH_DB_URL
-  }
-
-  const [protocol, ...rest] = env.COUCH_DB_URL.split("://")
-
-  if (!env.COUCH_DB_USERNAME || !env.COUCH_DB_PASSWORD) {
-    throw new Error(
-      "CouchDB configuration invalid. You must provide a fully qualified CouchDB url, or the COUCH_DB_USER and COUCH_DB_PASSWORD environment variables."
-    )
-  }
-
-  return `${protocol}://${env.COUCH_DB_USERNAME}:${env.COUCH_DB_PASSWORD}@${rest}`
-}
-
 exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
   const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
   return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
@@ -184,11 +166,17 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
   const efficient = opts && opts.efficient
   // specifically for testing we use the pouch package for this
   if (env.isTest()) {
-    return getCouch().allDbs()
+    return allDbs()
   }
   let dbs = []
-  async function addDbs(url) {
+  let { url, cookie } = getCouchInfo()
-    const response = await fetch(checkSlashesInUrl(encodeURI(url)))
+  async function addDbs(couchUrl) {
+    const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), {
+      method: "GET",
+      headers: {
+        Authorization: cookie,
+      },
+    })
     if (response.status === 200) {
       let json = await response.json()
       dbs = dbs.concat(json)
@@ -196,7 +184,7 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
       throw "Cannot connect to CouchDB instance"
     }
   }
-  let couchUrl = `${exports.getCouchUrl()}/_all_dbs`
+  let couchUrl = `${url}/_all_dbs`
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
     // just get all DBs when:
@@ -227,7 +215,6 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
  * @return {Promise<object[]>} returns the app information document stored in each app database.
  */
 exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
-  const CouchDB = getCouch()
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY && !tenantId) {
     tenantId = DEFAULT_TENANT_ID
@@ -255,7 +242,7 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
   }
   const appPromises = appDbNames.map(app =>
     // skip setup otherwise databases could be re-created
-    getAppMetadata(app, CouchDB)
+    getAppMetadata(app)
   )
   if (appPromises.length === 0) {
     return []
@@ -299,10 +286,11 @@ exports.getDevAppIDs = async () => {
 }
 
 exports.dbExists = async dbName => {
-  const CouchDB = getCouch()
   let exists = false
+  return doWithDB(
+    dbName,
+    async db => {
       try {
-    const db = CouchDB(dbName, { skip_setup: true })
         // check if database exists
         const info = await db.info()
         if (info && !info.error) {
@@ -312,6 +300,9 @@ exports.dbExists = async dbName => {
         exists = false
       }
       return exists
+    },
+    { skip_setup: true }
+  )
 }
 
 /**
@@ -436,3 +427,4 @@ exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
 exports.getScopedFullConfig = getScopedFullConfig
 exports.generateDevInfoID = generateDevInfoID
+exports.getPlatformUrl = getPlatformUrl

packages/backend-core/src/environment.js

@@ -8,7 +8,7 @@ function isTest() {
 
 module.exports = {
   JWT_SECRET: process.env.JWT_SECRET,
-  COUCH_DB_URL: process.env.COUCH_DB_URL,
+  COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
@@ -36,9 +36,18 @@ module.exports = {
   GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global",
   GLOBAL_CLOUD_BUCKET_NAME:
     process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads",
+  USE_COUCH: process.env.USE_COUCH || true,
   isTest,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value
   },
 }
+
+// clean up any environment variable edge cases
+for (let [key, value] of Object.entries(module.exports)) {
+  // handle the edge case of "0" to disable an environment variable
+  if (value === "0") {
+    module.exports[key] = 0
+  }
+}

packages/backend-core/src/index.js

@@ -1,8 +1,8 @@
-const { setDB } = require("./db")
+const db = require("./db")
 
 module.exports = {
-  init(pouch) {
+  init(opts = {}) {
-    setDB(pouch)
+    db.init(opts.db)
   },
   // some default exports from the library, however these ideally shouldn't
   // be used, instead the syntax require("@budibase/backend-core/db") should be used

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,8 +1,8 @@
 const google = require("../google")
 const { Cookies, Configs } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
-const { getDB } = require("../../../db")
+const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
-const { getScopedConfig } = require("../../../db/utils")
+const { doWithDB } = require("../../../db")
 const environment = require("../../../environment")
 const { getGlobalDB } = require("../../../tenancy")
 
@@ -13,18 +13,28 @@ async function fetchGoogleCreds() {
     type: Configs.GOOGLE,
   })
   // or fall back to env variables
-  const config = googleConfig || {
+  return (
+    googleConfig || {
       clientID: environment.GOOGLE_CLIENT_ID,
       clientSecret: environment.GOOGLE_CLIENT_SECRET,
     }
+  )
+}
 
-  return config
+async function platformUrl() {
+  const db = getGlobalDB()
+  const publicConfig = await getScopedConfig(db, {
+    type: Configs.SETTINGS,
+  })
+  return getPlatformUrl(publicConfig)
 }
 
 async function preAuth(passport, ctx, next) {
   // get the relevant config
   const googleConfig = await fetchGoogleCreds()
-  let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback`
+  const platUrl = await platformUrl()
+
+  let callbackUrl = `${platUrl}/api/global/auth/datasource/google/callback`
   const strategy = await google.strategyFactory(googleConfig, callbackUrl)
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
@@ -41,14 +51,15 @@ async function preAuth(passport, ctx, next) {
 async function postAuth(passport, ctx, next) {
   // get the relevant config
   const config = await fetchGoogleCreds()
+  const platUrl = await platformUrl()
 
-  let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback`
+  let callbackUrl = `${platUrl}/api/global/auth/datasource/google/callback`
   const strategy = await google.strategyFactory(
     config,
     callbackUrl,
     (accessToken, refreshToken, profile, done) => {
       clearCookie(ctx, Cookies.DatasourceAuth)
-      done(null, { accessToken, refreshToken })
+      done(null, { refreshToken })
     }
   )
 
@@ -59,7 +70,7 @@ async function postAuth(passport, ctx, next) {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err, tokens) => {
       // update the DB for the datasource with all the user info
-      const db = getDB(authStateCookie.appId)
+      await doWithDB(authStateCookie.appId, async db => {
         const datasource = await db.get(authStateCookie.datasourceId)
         if (!datasource.config) {
           datasource.config = {}
@@ -69,6 +80,7 @@ async function postAuth(passport, ctx, next) {
         ctx.redirect(
           `/builder/app/${authStateCookie.appId}/data/datasource/${authStateCookie.datasourceId}`
         )
+      })
     }
   )(ctx, next)
 }

packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js

@@ -2,17 +2,13 @@
 
 require("../../../tests/utilities/dbConfig")
 
-const database = require("../../../db")
 const { authenticateThirdParty } = require("../third-party-common")
 const { data } = require("./utilities/mock-data")
+const { DEFAULT_TENANT_ID } = require("../../../constants")
 
-const {
+const { generateGlobalUserID } = require("../../../db/utils")
-  StaticDatabases,
-  generateGlobalUserID
-} = require("../../../db/utils")
 const { newid } = require("../../../hashing")
-
+const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
-let db
 
 const done = jest.fn()
 
@@ -21,7 +17,15 @@ const getErrorMessage = () => {
 }
 
 const saveUser = async (user) => {
+  return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
     return await db.put(user)
+  })
+}
+
+function authenticate(user, requireLocal, saveFn) {
+  return doInTenant(DEFAULT_TENANT_ID, () => {
+    return authenticateThirdParty(user, requireLocal, done, saveFn)
+  })
 }
 
 describe("third party common", () => {
@@ -29,35 +33,36 @@ describe("third party common", () => {
     let thirdPartyUser
 
     beforeEach(() => {
-      db = database.getDB(StaticDatabases.GLOBAL.name)
       thirdPartyUser = data.buildThirdPartyUser()
     })
 
     afterEach(async () => {
+      return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
         jest.clearAllMocks()
         await db.destroy()
       })
+    })
 
     describe("validation", () => {
       const testValidation = async (message) => {
-        await authenticateThirdParty(thirdPartyUser, false, done, saveUser)
+        await authenticate(thirdPartyUser, false, saveUser)
         expect(done.mock.calls.length).toBe(1)
         expect(getErrorMessage()).toContain(message)
       }
 
       it("provider fails", async () => {
         delete thirdPartyUser.provider
-        testValidation("third party user provider required")
+        await testValidation("third party user provider required")
       })
 
       it("user id fails", async () => {
         delete thirdPartyUser.userId
-        testValidation("third party user id required")
+        await testValidation("third party user id required")
       })
 
       it("email fails", async () => {
         delete thirdPartyUser.email
-        testValidation("third party user email required")
+        await testValidation("third party user email required")
       })
     })
 
@@ -81,7 +86,7 @@ describe("third party common", () => {
     describe("when the user doesn't exist", () => {
       describe("when a local account is required", () => {
         it("returns an error message", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
+          await authenticate(thirdPartyUser, true, saveUser)
           expect(done.mock.calls.length).toBe(1)
           expect(getErrorMessage()).toContain("Email does not yet exist. You must set up your local budibase account first.")
         })
@@ -89,7 +94,7 @@ describe("third party common", () => {
 
       describe("when a local account isn't required", () => {
         it("creates and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, false, done, saveUser)
+          await authenticate(thirdPartyUser, false, saveUser)
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)
           expect(user.roles).toStrictEqual({})
@@ -103,12 +108,15 @@ describe("third party common", () => {
       let email
 
       const createUser = async () => {
+        return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
           dbUser = {
             _id: id,
             email: email,
           }
           const response = await db.put(dbUser)
           dbUser._rev = response.rev
+          return dbUser
+        })
       }
 
       const expectUserIsUpdated = (user) => {
@@ -126,7 +134,7 @@ describe("third party common", () => {
         })
 
         it("syncs and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
+          await authenticate(thirdPartyUser, true, saveUser)
 
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)
@@ -142,7 +150,7 @@ describe("third party common", () => {
         })
 
         it("syncs and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
+          await authenticate(thirdPartyUser, true, saveUser)
 
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)
@@ -160,7 +168,7 @@ describe("third party common", () => {
         })
 
         it("syncs and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
+          await authenticate(thirdPartyUser, true, saveUser)
 
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)

packages/backend-core/src/middleware/tenancy.js

@@ -1,4 +1,5 @@
-const { setTenantId } = require("../tenancy")
+const { setTenantId, setGlobalDB, getGlobalDB } = require("../tenancy")
+const { closeDB } = require("../db")
 const ContextFactory = require("../context/FunctionContext")
 const { buildMatcherRegex, matches } = require("./matchers")
 
@@ -10,10 +11,17 @@ module.exports = (
   const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
   const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
 
-  return ContextFactory.getMiddleware(ctx => {
+  const updateCtxFn = ctx => {
     const allowNoTenant =
       opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
     const allowQs = !!matches(ctx, allowQsOptions)
-    setTenantId(ctx, { allowQs, allowNoTenant })
+    const tenantId = setTenantId(ctx, { allowQs, allowNoTenant })
-  })
+    setGlobalDB(tenantId)
+  }
+  const destroyFn = async () => {
+    const db = getGlobalDB()
+    await closeDB(db)
+  }
+
+  return ContextFactory.getMiddleware(updateCtxFn, destroyFn)
 }

packages/backend-core/src/migrations/index.js

@@ -1,4 +1,5 @@
 const { DEFAULT_TENANT_ID } = require("../constants")
+const { doWithDB } = require("../db")
 const { DocumentTypes } = require("../db/constants")
 const { getAllApps } = require("../db/utils")
 const environment = require("../environment")
@@ -26,7 +27,7 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-const runMigration = async (CouchDB, migration, options = {}) => {
+const runMigration = async (migration, options = {}) => {
   const tenantId = getTenantId()
   const migrationType = migration.type
   const migrationName = migration.name
@@ -46,7 +47,7 @@ const runMigration = async (CouchDB, migration, options = {}) => {
 
   // run the migration against each db
   for (const dbName of dbNames) {
-    const db = new CouchDB(dbName)
+    await doWithDB(dbName, async db => {
       try {
         const doc = await exports.getMigrationsDoc(db)
 
@@ -62,7 +63,7 @@ const runMigration = async (CouchDB, migration, options = {}) => {
             )
           } else {
             // the migration has already been performed
-          continue
+            return
           }
         }
 
@@ -85,10 +86,11 @@ const runMigration = async (CouchDB, migration, options = {}) => {
         )
         throw err
       }
+    })
   }
 }
 
-exports.runMigrations = async (CouchDB, migrations, options = {}) => {
+exports.runMigrations = async (migrations, options = {}) => {
   console.log("Running migrations")
   let tenantIds
   if (environment.MULTI_TENANCY) {
@@ -108,9 +110,7 @@ exports.runMigrations = async (CouchDB, migrations, options = {}) => {
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await doInTenant(tenantId, () =>
+      await doInTenant(tenantId, () => runMigration(migration, options))
-        runMigration(CouchDB, migration, options)
-      )
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,7 +1,7 @@
 require("../../tests/utilities/dbConfig")
 
 const { runMigrations, getMigrationsDoc } = require("../index")
-const CouchDB = require("../../db").getCouch()
+const { dangerousGetDB } = require("../../db")
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -20,7 +20,7 @@ describe("migrations", () => {
   }]
 
   beforeEach(() => {
-    db = new CouchDB(StaticDatabases.GLOBAL.name)
+    db = dangerousGetDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {
@@ -29,7 +29,7 @@ describe("migrations", () => {
   })
 
   const migrate = () => {
-    return runMigrations(CouchDB, MIGRATIONS)
+    return runMigrations(MIGRATIONS)
   }
 
   it("should run a new migration", async () => {

packages/backend-core/src/security/roles.js

@@ -7,7 +7,7 @@ const {
   SEPARATOR,
 } = require("../db/utils")
 const { getAppDB } = require("../context")
-const { getDB } = require("../db")
+const { doWithDB } = require("../db")
 
 const BUILTIN_IDS = {
   ADMIN: "ADMIN",
@@ -199,7 +199,12 @@ exports.checkForRoleResourceArray = (rolePerms, resourceId) => {
  * @return {Promise<object[]>} An array of the role objects that were found.
  */
 exports.getAllRoles = async appId => {
-  const db = appId ? getDB(appId) : getAppDB()
+  if (appId) {
+    return doWithDB(appId, internal)
+  } else {
+    return internal(getAppDB())
+  }
+  async function internal(db) {
     const body = await db.allDocs(
       getRoleParams(null, {
         include_docs: true,
@@ -237,6 +242,7 @@ exports.getAllRoles = async appId => {
     }
     return roles
   }
+}
 
 /**
  * This retrieves the required role for a resource

packages/backend-core/src/tenancy/tenancy.js

@@ -1,5 +1,6 @@
-const { getDB } = require("../db")
+const { doWithDB } = require("../db")
-const { SEPARATOR, StaticDatabases } = require("../db/constants")
+const { StaticDatabases } = require("../db/constants")
+const { baseGlobalDBName } = require("./utils")
 const {
   getTenantId,
   DEFAULT_TENANT_ID,
@@ -23,7 +24,7 @@ exports.addTenantToUrl = url => {
 }
 
 exports.doesTenantExist = async tenantId => {
-  const db = getDB(PLATFORM_INFO_DB)
+  return doWithDB(PLATFORM_INFO_DB, async db => {
     let tenants
     try {
       tenants = await db.get(TENANT_DOC)
@@ -36,10 +37,11 @@ exports.doesTenantExist = async tenantId => {
       Array.isArray(tenants.tenantIds) &&
       tenants.tenantIds.indexOf(tenantId) !== -1
     )
+  })
 }
 
 exports.tryAddTenant = async (tenantId, userId, email) => {
-  const db = getDB(PLATFORM_INFO_DB)
+  return doWithDB(PLATFORM_INFO_DB, async db => {
     const getDoc = async id => {
       if (!id) {
         return null
@@ -76,6 +78,7 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
       promises.push(db.put(tenants))
     }
     await Promise.all(promises)
+  })
 }
 
 exports.getGlobalDBName = (tenantId = null) => {
@@ -84,23 +87,15 @@ exports.getGlobalDBName = (tenantId = null) => {
   if (!tenantId) {
     tenantId = getTenantId()
   }
-
+  return baseGlobalDBName(tenantId)
-  let dbName
-  if (tenantId === DEFAULT_TENANT_ID) {
-    dbName = StaticDatabases.GLOBAL.name
-  } else {
-    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
-  }
-  return dbName
 }
 
-exports.getGlobalDB = (tenantId = null) => {
+exports.doWithGlobalDB = (tenantId, cb) => {
-  const dbName = exports.getGlobalDBName(tenantId)
+  return doWithDB(exports.getGlobalDBName(tenantId), cb)
-  return getDB(dbName)
 }
 
 exports.lookupTenantId = async userId => {
-  const db = getDB(StaticDatabases.PLATFORM_INFO.name)
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
     let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
     try {
       const doc = await db.get(userId)
@@ -111,16 +106,18 @@ exports.lookupTenantId = async userId => {
       // just return the default
     }
     return tenantId
+  })
 }
 
 // lookup, could be email or userId, either will return a doc
 exports.getTenantUser = async identifier => {
-  const db = getDB(PLATFORM_INFO_DB)
+  return doWithDB(PLATFORM_INFO_DB, async db => {
     try {
       return await db.get(identifier)
     } catch (err) {
       return null
     }
+  })
 }
 
 exports.isUserInAppTenant = (appId, user = null) => {
@@ -135,7 +132,7 @@ exports.isUserInAppTenant = (appId, user = null) => {
 }
 
 exports.getTenantIds = async () => {
-  const db = getDB(PLATFORM_INFO_DB)
+  return doWithDB(PLATFORM_INFO_DB, async db => {
     let tenants
     try {
       tenants = await db.get(TENANT_DOC)
@@ -144,4 +141,5 @@ exports.getTenantIds = async () => {
       return []
     }
     return (tenants && tenants.tenantIds) || []
+  })
 }

packages/backend-core/src/tenancy/utils.js

@@ -0,0 +1,12 @@
+const { DEFAULT_TENANT_ID } = require("../constants")
+const { StaticDatabases, SEPARATOR } = require("../db/constants")
+
+exports.baseGlobalDBName = tenantId => {
+  let dbName
+  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    dbName = StaticDatabases.GLOBAL.name
+  } else {
+    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+  }
+  return dbName
+}

packages/backend-core/src/tests/utilities/db.js

@@ -1,17 +0,0 @@
-const PouchDB = require("pouchdb")
-const env = require("../../environment")
-
-let POUCH_DB_DEFAULTS
-
-// should always be test but good to do the sanity check
-if (env.isTest()) {
-  PouchDB.plugin(require("pouchdb-adapter-memory"))
-  POUCH_DB_DEFAULTS = {
-    prefix: undefined,
-    adapter: "memory",
-  }
-}
-
-const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS)
-
-module.exports = Pouch

packages/backend-core/src/tests/utilities/dbConfig.js

@@ -1,3 +1,5 @@
-const packageConfiguration = require("../../index")
+const core = require("../../index")
-const CouchDB = require("./db")
+const dbConfig = {
-packageConfiguration.init(CouchDB)
+  inMemory: true,
+}
+core.init({ db: dbConfig })

packages/backend-core/src/utils.js

@@ -10,7 +10,7 @@ const { options } = require("./middleware/passport/jwt")
 const { queryGlobalView } = require("./db/views")
 const { Headers, UserStatus, Cookies, MAX_VALID_DATE } = require("./constants")
 const {
-  getGlobalDB,
+  doWithGlobalDB,
   updateTenantId,
   getTenantUser,
   tryAddTenant,
@@ -209,7 +209,7 @@ exports.saveUser = async (
   // need to set the context for this request, as specified
   updateTenantId(tenantId)
   // specify the tenancy incase we're making a new admin user (public)
-  const db = getGlobalDB(tenantId)
+  return doWithGlobalDB(tenantId, async db => {
     let { email, password, _id } = user
     // make sure another user isn't using the same email
     let dbUser
@@ -285,6 +285,7 @@ exports.saveUser = async (
         throw err
       }
     }
+  })
 }
 
 /**

packages/backend-core/yarn.lock

@@ -258,6 +258,13 @@
   dependencies:
     "@babel/helper-plugin-utils" "^7.14.5"
 
+"@babel/runtime@^7.15.4":
+  version "7.17.9"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.17.9.tgz#d19fbf802d01a8cb6cf053a64e472d42c434ba72"
+  integrity sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==
+  dependencies:
+    regenerator-runtime "^0.13.4"
+
 "@babel/template@^7.16.0", "@babel/template@^7.3.3":
   version "7.16.0"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.16.0.tgz#d16a35ebf4cd74e202083356fab21dd89363ddd6"
@@ -857,6 +864,21 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
   integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
 
+axios-retry@^3.1.9:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.2.4.tgz#f447a53c3456f5bfeca18f20c3a3272207d082ae"
+  integrity sha512-Co3UXiv4npi6lM963mfnuH90/YFLKWWDmoBYfxkHT5xtkSSWNqK9zdG3fw5/CP/dsoKB5aMMJCsgab+tp1OxLQ==
+  dependencies:
+    "@babel/runtime" "^7.15.4"
+    is-retry-allowed "^2.2.0"
+
+axios@0.24.0:
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
+  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
+  dependencies:
+    follow-redirects "^1.14.4"
+
 babel-jest@^26.6.3:
   version "26.6.3"
   resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-26.6.3.tgz#d87d25cb0037577a0c89f82e5755c5d293c01056"
@@ -1048,7 +1070,7 @@ buffer-from@1.1.1:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
-buffer-from@^1.0.0:
+buffer-from@1.1.2, buffer-from@^1.0.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
   integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
@@ -1139,6 +1161,11 @@ char-regex@^1.0.2:
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
   integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
 
+charenc@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
+  integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=
+
 chownr@^1.1.1:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
@@ -1273,6 +1300,11 @@ component-emitter@^1.2.1:
   resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0"
   integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==
 
+component-type@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9"
+  integrity sha1-ikeQFwAjjk/DIml3EjAibyS0Fak=
+
 concat-map@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -1315,6 +1347,11 @@ cross-spawn@^7.0.0:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+crypt@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
+  integrity sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=
+
 cryptiles@2.x.x:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8"
@@ -1777,6 +1814,13 @@ fetch-cookie@0.10.1:
   dependencies:
     tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
 
+fetch-cookie@0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.11.0.tgz#e046d2abadd0ded5804ce7e2cae06d4331c15407"
+  integrity sha512-BQm7iZLFhMWFy5CZ/162sAGjBfdNWb7a8LEqqnzsHFhxT/X/SVj/z2t2nu3aJvjlbQkrAlTUApplPRjWyH4mhA==
+  dependencies:
+    tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
+
 fill-range@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7"
@@ -1802,6 +1846,11 @@ find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
+follow-redirects@^1.14.4:
+  version "1.14.9"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7"
+  integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==
+
 for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -2175,7 +2224,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -2226,7 +2275,7 @@ is-arrayish@^0.2.1:
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
   integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=
 
-is-buffer@^1.1.5:
+is-buffer@^1.1.5, is-buffer@~1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
   integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
@@ -2328,6 +2377,11 @@ is-potential-custom-element-name@^1.0.1:
   resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5"
   integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==
 
+is-retry-allowed@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz#88f34cbd236e043e71b6932d09b0c65fb7b4d71d"
+  integrity sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==
+
 is-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
@@ -2360,7 +2414,7 @@ isarray@0.0.1:
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
   integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=
 
-isarray@1.0.0, isarray@^1.0.0:
+isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
   integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
@@ -2824,6 +2878,11 @@ jodid25519@^1.0.0:
   dependencies:
     jsbn "~0.1.0"
 
+join-component@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5"
+  integrity sha1-uEF7dQZho5K+4sJTfGiyqdSXfNU=
+
 js-tokens@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
@@ -2902,7 +2961,7 @@ json-stable-stringify@^1.0.1:
   dependencies:
     jsonify "~0.0.0"
 
-json-stringify-safe@~5.0.1:
+json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
@@ -3204,6 +3263,11 @@ lodash.once@^4.0.0:
   resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
   integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=
 
+lodash.pick@^4.0.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
+  integrity sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=
+
 lodash@^4.14.0:
   version "4.17.4"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
@@ -3252,6 +3316,15 @@ map-visit@^1.0.0:
   dependencies:
     object-visit "^1.0.0"
 
+md5@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f"
+  integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==
+  dependencies:
+    charenc "0.0.2"
+    crypt "0.0.2"
+    is-buffer "~1.1.6"
+
 memdown@1.4.1:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/memdown/-/memdown-1.4.1.tgz#b4e4e192174664ffbae41361aa500f3119efe215"
@@ -3372,6 +3445,11 @@ ms@2.1.2, ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
+ms@^2.1.3:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
+  integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
+
 nanomatch@^1.2.9:
   version "1.2.13"
   resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119"
@@ -3399,6 +3477,16 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
+ndjson@^1.4.3:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8"
+  integrity sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg=
+  dependencies:
+    json-stringify-safe "^5.0.1"
+    minimist "^1.2.0"
+    split2 "^2.1.0"
+    through2 "^2.0.3"
+
 nice-try@^1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
@@ -3409,7 +3497,7 @@ node-fetch@2.6.0:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
 
-node-fetch@^2.6.1:
+node-fetch@2.6.7, node-fetch@^2.6.1:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
@@ -3769,6 +3857,42 @@ posix-character-classes@^0.1.0:
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
   integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=
 
+posthog-node@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-1.3.0.tgz#804ed2f213a2f05253f798bf9569d55a9cad94f7"
+  integrity sha512-2+VhqiY/rKIqKIXyvemBFHbeijHE25sP7eKltnqcFqAssUE6+sX6vusN9A4luzToOqHQkUZexiCKxvuGagh7JA==
+  dependencies:
+    axios "0.24.0"
+    axios-retry "^3.1.9"
+    component-type "^1.2.1"
+    join-component "^1.1.0"
+    md5 "^2.3.0"
+    ms "^2.1.3"
+    remove-trailing-slash "^0.1.1"
+    uuid "^8.3.2"
+
+pouch-stream@^0.4.0:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/pouch-stream/-/pouch-stream-0.4.1.tgz#0c6d8475c9307677627991a2f079b301c3b89bdd"
+  integrity sha1-DG2EdckwdndieZGi8HmzAcO4m90=
+  dependencies:
+    inherits "^2.0.1"
+    readable-stream "^1.0.27-1"
+
+pouchdb-abstract-mapreduce@7.2.2:
+  version "7.2.2"
+  resolved "https://registry.yarnpkg.com/pouchdb-abstract-mapreduce/-/pouchdb-abstract-mapreduce-7.2.2.tgz#dd1b10a83f8d24361dce9aaaab054614b39f766f"
+  integrity sha512-7HWN/2yV2JkwMnGnlp84lGvFtnm0Q55NiBUdbBcaT810+clCGKvhssBCrXnmwShD1SXTwT83aszsgiSfW+SnBA==
+  dependencies:
+    pouchdb-binary-utils "7.2.2"
+    pouchdb-collate "7.2.2"
+    pouchdb-collections "7.2.2"
+    pouchdb-errors "7.2.2"
+    pouchdb-fetch "7.2.2"
+    pouchdb-mapreduce-utils "7.2.2"
+    pouchdb-md5 "7.2.2"
+    pouchdb-utils "7.2.2"
+
 pouchdb-adapter-leveldb-core@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-adapter-leveldb-core/-/pouchdb-adapter-leveldb-core-7.2.2.tgz#e0aa6a476e2607d7ae89f4a803c9fba6e6d05a8a"
@@ -3828,6 +3952,11 @@ pouchdb-binary-utils@7.2.2:
   dependencies:
     buffer-from "1.1.1"
 
+pouchdb-collate@7.2.2:
+  version "7.2.2"
+  resolved "https://registry.yarnpkg.com/pouchdb-collate/-/pouchdb-collate-7.2.2.tgz#fc261f5ef837c437e3445fb0abc3f125d982c37c"
+  integrity sha512-/SMY9GGasslknivWlCVwXMRMnQ8myKHs4WryQ5535nq1Wj/ehpqWloMwxEQGvZE1Sda3LOm7/5HwLTcB8Our+w==
+
 pouchdb-collections@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-collections/-/pouchdb-collections-7.2.2.tgz#aeed77f33322429e3f59d59ea233b48ff0e68572"
@@ -3840,6 +3969,28 @@ pouchdb-errors@7.2.2:
   dependencies:
     inherits "2.0.4"
 
+pouchdb-fetch@7.2.2:
+  version "7.2.2"
+  resolved "https://registry.yarnpkg.com/pouchdb-fetch/-/pouchdb-fetch-7.2.2.tgz#492791236d60c899d7e9973f9aca0d7b9cc02230"
+  integrity sha512-lUHmaG6U3zjdMkh8Vob9GvEiRGwJfXKE02aZfjiVQgew+9SLkuOxNw3y2q4d1B6mBd273y1k2Lm0IAziRNxQnA==
+  dependencies:
+    abort-controller "3.0.0"
+    fetch-cookie "0.10.1"
+    node-fetch "2.6.0"
+
+pouchdb-find@^7.2.2:
+  version "7.2.2"
+  resolved "https://registry.yarnpkg.com/pouchdb-find/-/pouchdb-find-7.2.2.tgz#1227afdd761812d508fe0794b3e904518a721089"
+  integrity sha512-BmFeFVQ0kHmDehvJxNZl9OmIztCjPlZlVSdpijuFbk/Fi1EFPU1BAv3kLC+6DhZuOqU/BCoaUBY9sn66pPY2ag==
+  dependencies:
+    pouchdb-abstract-mapreduce "7.2.2"
+    pouchdb-collate "7.2.2"
+    pouchdb-errors "7.2.2"
+    pouchdb-fetch "7.2.2"
+    pouchdb-md5 "7.2.2"
+    pouchdb-selector-core "7.2.2"
+    pouchdb-utils "7.2.2"
+
 pouchdb-json@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-json/-/pouchdb-json-7.2.2.tgz#b939be24b91a7322e9a24b8880a6e21514ec5e1f"
@@ -3847,6 +3998,16 @@ pouchdb-json@7.2.2:
   dependencies:
     vuvuzela "1.0.3"
 
+pouchdb-mapreduce-utils@7.2.2:
+  version "7.2.2"
+  resolved "https://registry.yarnpkg.com/pouchdb-mapreduce-utils/-/pouchdb-mapreduce-utils-7.2.2.tgz#13a46a3cc2a3f3b8e24861da26966904f2963146"
+  integrity sha512-rAllb73hIkU8rU2LJNbzlcj91KuulpwQu804/F6xF3fhZKC/4JQMClahk+N/+VATkpmLxp1zWmvmgdlwVU4HtQ==
+  dependencies:
+    argsarray "0.0.1"
+    inherits "2.0.4"
+    pouchdb-collections "7.2.2"
+    pouchdb-utils "7.2.2"
+
 pouchdb-md5@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-md5/-/pouchdb-md5-7.2.2.tgz#415401acc5a844112d765bd1fb4e5d9f38fb0838"
@@ -3860,13 +4021,34 @@ pouchdb-merge@7.2.2:
   resolved "https://registry.yarnpkg.com/pouchdb-merge/-/pouchdb-merge-7.2.2.tgz#940d85a2b532d6a93a6cab4b250f5648511bcc16"
   integrity sha512-6yzKJfjIchBaS7Tusuk8280WJdESzFfQ0sb4jeMUNnrqs4Cx3b0DIEOYTRRD9EJDM+je7D3AZZ4AT0tFw8gb4A==
 
-pouchdb-promise@6.4.3:
+pouchdb-promise@6.4.3, pouchdb-promise@^6.0.4:
   version "6.4.3"
   resolved "https://registry.yarnpkg.com/pouchdb-promise/-/pouchdb-promise-6.4.3.tgz#74516f4acf74957b54debd0fb2c0e5b5a68ca7b3"
   integrity sha512-ruJaSFXwzsxRHQfwNHjQfsj58LBOY1RzGzde4PM5CWINZwFjCQAhZwfMrch2o/0oZT6d+Xtt0HTWhq35p3b0qw==
   dependencies:
     lie "3.1.1"
 
+pouchdb-replication-stream@^1.2.9:
+  version "1.2.9"
+  resolved "https://registry.yarnpkg.com/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.9.tgz#aa4fa5d8f52df4825392f18e07c7e11acffc650a"
+  integrity sha1-qk+l2PUt9IJTkvGOB8fhGs/8ZQo=
+  dependencies:
+    argsarray "0.0.1"
+    inherits "^2.0.3"
+    lodash.pick "^4.0.0"
+    ndjson "^1.4.3"
+    pouch-stream "^0.4.0"
+    pouchdb-promise "^6.0.4"
+    through2 "^2.0.0"
+
+pouchdb-selector-core@7.2.2:
+  version "7.2.2"
+  resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.2.2.tgz#264d7436a8c8ac3801f39960e79875ef7f3879a0"
+  integrity sha512-XYKCNv9oiNmSXV5+CgR9pkEkTFqxQGWplnVhO3W9P154H08lU0ZoNH02+uf+NjZ2kjse7Q1fxV4r401LEcGMMg==
+  dependencies:
+    pouchdb-collate "7.2.2"
+    pouchdb-utils "7.2.2"
+
 pouchdb-utils@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-utils/-/pouchdb-utils-7.2.2.tgz#c17c4788f1d052b0daf4ef8797bbc4aaa3945aa4"
@@ -3881,17 +4063,17 @@ pouchdb-utils@7.2.2:
     pouchdb-md5 "7.2.2"
     uuid "8.1.0"
 
-pouchdb@^7.2.1:
+pouchdb@7.3.0:
-  version "7.2.2"
+  version "7.3.0"
-  resolved "https://registry.yarnpkg.com/pouchdb/-/pouchdb-7.2.2.tgz#fcae82862db527e4cf7576ed8549d1384961f364"
+  resolved "https://registry.yarnpkg.com/pouchdb/-/pouchdb-7.3.0.tgz#440fbef12dfd8f9002320802528665e883a3b7f8"
-  integrity sha512-5gf5nw5XH/2H/DJj8b0YkvG9fhA/4Jt6kL0Y8QjtztVjb1y4J19Rg4rG+fUbXu96gsUrlyIvZ3XfM0b4mogGmw==
+  integrity sha512-OwsIQGXsfx3TrU1pLruj6PGSwFH+h5k4hGNxFkZ76Um7/ZI8F5TzUHFrpldVVIhfXYi2vP31q0q7ot1FSLFYOw==
   dependencies:
     abort-controller "3.0.0"
     argsarray "0.0.1"
-    buffer-from "1.1.1"
+    buffer-from "1.1.2"
     clone-buffer "1.0.0"
     double-ended-queue "2.1.0-0"
-    fetch-cookie "0.10.1"
+    fetch-cookie "0.11.0"
     immediate "3.3.0"
     inherits "2.0.4"
     level "6.0.1"
@@ -3900,11 +4082,11 @@ pouchdb@^7.2.1:
     leveldown "5.6.0"
     levelup "4.4.0"
     ltgt "2.2.1"
-    node-fetch "2.6.0"
+    node-fetch "2.6.7"
     readable-stream "1.1.14"
-    spark-md5 "3.0.1"
+    spark-md5 "3.0.2"
     through2 "3.0.2"
-    uuid "8.1.0"
+    uuid "8.3.2"
     vuvuzela "1.0.3"
 
 prelude-ls@~1.1.2:
@@ -3927,6 +4109,11 @@ private@^0.1.6, private@~0.1.5:
   resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
   integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==
 
+process-nextick-args@~2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
+  integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
+
 prompts@^2.0.1:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"
@@ -4012,7 +4199,7 @@ read-pkg@^5.2.0:
     parse-json "^5.0.0"
     type-fest "^0.6.0"
 
-readable-stream@1.1.14:
+readable-stream@1.1.14, readable-stream@^1.0.27-1:
   version "1.1.14"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
   integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk=
@@ -4036,6 +4223,19 @@ readable-stream@~0.0.2:
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-0.0.4.tgz#f32d76e3fb863344a548d79923007173665b3b8d"
   integrity sha1-8y124/uGM0SlSNeZIwBxc2ZbO40=
 
+readable-stream@~2.3.6:
+  version "2.3.7"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
+  integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~2.0.0"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.1.1"
+    util-deprecate "~1.0.1"
+
 readline-sync@^1.4.9:
   version "1.4.10"
   resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b"
@@ -4068,6 +4268,11 @@ redis-parser@^3.0.0:
   dependencies:
     redis-errors "^1.0.0"
 
+regenerator-runtime@^0.13.4:
+  version "0.13.9"
+  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52"
+  integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==
+
 regex-not@^1.0.0, regex-not@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c"
@@ -4081,6 +4286,11 @@ remove-trailing-separator@^1.0.1:
   resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef"
   integrity sha1-wkvOKig62tW8P1jg1IJJuSN52O8=
 
+remove-trailing-slash@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz#be2285a59f39c74d1bce4f825950061915e3780d"
+  integrity sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA==
+
 repeat-element@^1.1.2:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.4.tgz#be681520847ab58c7568ac75fbfad28ed42d39e9"
@@ -4202,7 +4412,7 @@ safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
-safe-buffer@~5.1.1:
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
@@ -4425,6 +4635,11 @@ spark-md5@3.0.1:
   resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.1.tgz#83a0e255734f2ab4e5c466e5a2cfc9ba2aa2124d"
   integrity sha512-0tF3AGSD1ppQeuffsLDIOWlKUd3lS92tFxcsrh5Pe3ZphhnoK+oXIBTzOAThZCiuINZLvpiLH/1VS1/ANEJVig==
 
+spark-md5@3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc"
+  integrity sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==
+
 spdx-correct@^3.0.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9"
@@ -4458,6 +4673,13 @@ split-string@^3.0.1, split-string@^3.0.2:
   dependencies:
     extend-shallow "^3.0.0"
 
+split2@^2.1.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-2.2.0.tgz#186b2575bcf83e85b7d18465756238ee4ee42493"
+  integrity sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==
+  dependencies:
+    through2 "^2.0.2"
+
 sprintf-js@^1.1.1:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673"
@@ -4548,6 +4770,13 @@ string_decoder@~0.10.x:
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
   integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
 
+string_decoder@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+  dependencies:
+    safe-buffer "~5.1.0"
+
 stringstream@~0.0.4:
   version "0.0.6"
   resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.6.tgz#7880225b0d4ad10e30927d167a1d6f2fd3b33a72"
@@ -4663,6 +4892,14 @@ through2@3.0.2:
     inherits "^2.0.4"
     readable-stream "2 || 3"
 
+through2@^2.0.0, through2@^2.0.2, through2@^2.0.3:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
+  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
+  dependencies:
+    readable-stream "~2.3.6"
+    xtend "~4.0.1"
+
 through@~2.3.4:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@@ -4857,7 +5094,7 @@ use@^3.1.0:
   resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
   integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==
 
-util-deprecate@^1.0.1:
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
@@ -4877,6 +5114,11 @@ uuid@8.1.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d"
   integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg==
 
+uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2:
+  version "8.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
+  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
+
 uuid@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
@@ -4887,11 +5129,6 @@ uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
-uuid@^8.3.0, uuid@^8.3.2:
-  version "8.3.2"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
-  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
-
 v8-to-istanbul@^7.0.0:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-7.1.2.tgz#30898d1a7fa0c84d225a2c1434fb958f290883c1"
@@ -5084,7 +5321,7 @@ xmlchars@^2.2.0:
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
   integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
 
-xtend@^4.0.2, xtend@~4.0.0:
+xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.105-alpha.42",
+    "@budibase/string-templates": "^1.0.130-alpha.0",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/bbui/src/ActionMenu/ActionMenu.svelte

@@ -6,6 +6,7 @@
   export let disabled = false
   export let align = "left"
   export let portalTarget
+  export let dataCy
 
   let anchor
   let dropdown
@@ -36,7 +37,7 @@
 <div use:getAnchor on:click={openMenu}>
   <slot name="control" />
 </div>
-<Popover bind:this={dropdown} {anchor} {align} {portalTarget}>
+<Popover bind:this={dropdown} {anchor} {align} {portalTarget} {dataCy}>
   <Menu>
     <slot />
   </Menu>

packages/bbui/src/Icon/Icon.svelte

@@ -3,6 +3,9 @@
 </script>
 
 <script>
+  import Tooltip from "../Tooltip/Tooltip.svelte"
+  import { fade } from "svelte/transition"
+
   export let direction = "n"
   export let name = "Add"
   export let hidden = false
@@ -10,14 +13,24 @@
   export let hoverable = false
   export let disabled = false
   export let color
+  export let tooltip
 
   $: rotation = getRotation(direction)
 
+  let showTooltip = false
+
   const getRotation = direction => {
     return directions.indexOf(direction) * 45
   }
 </script>
 
+<div
+  class="icon"
+  on:mouseover={() => (showTooltip = true)}
+  on:focus={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
+  on:click={() => (showTooltip = false)}
+>
   <svg
     on:click
     class:hoverable
@@ -30,10 +43,22 @@
       color ? `color: ${color};` : ""
     }`}
   >
-  <use xlink:href="#spectrum-icon-18-{name}" />
+    <use style="pointer-events: none;" xlink:href="#spectrum-icon-18-{name}" />
   </svg>
+  {#if tooltip && showTooltip}
+    <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
+      <Tooltip textWrapping direction={"bottom"} text={tooltip} />
+    </div>
+  {/if}
+</div>
 
 <style>
+  .icon {
+    position: relative;
+    display: grid;
+    place-items: center;
+  }
+
   svg.hoverable {
     pointer-events: all;
     transition: color var(--spectrum-global-animation-duration-100, 130ms);
@@ -47,4 +72,15 @@
     color: var(--spectrum-global-color-gray-500) !important;
     pointer-events: none !important;
   }
+
+  .tooltip {
+    position: absolute;
+    pointer-events: none;
+    left: 50%;
+    top: calc(100% + 4px);
+    width: 100vw;
+    max-width: 150px;
+    transform: translateX(-50%);
+    text-align: center;
+  }
 </style>

packages/bbui/src/Modal/ModalContent.svelte

@@ -23,6 +23,7 @@
   export let secondaryButtonText = undefined
   export let secondaryAction = undefined
   export let secondaryButtonWarning = false
+  export let dataCy = null
 
   const { hide, cancel } = getContext(Context.Modal)
   let loading = false
@@ -63,21 +64,26 @@
   role="dialog"
   tabindex="-1"
   aria-modal="true"
+  data-cy={dataCy}
 >
   <div class="spectrum-Dialog-grid">
-    {#if title}
+    {#if title || $$slots.header}
       <h1
         class="spectrum-Dialog-heading spectrum-Dialog-heading--noHeader"
         class:noDivider={!showDivider}
         class:header-spacing={$$slots.header}
       >
+        {#if title}
           {title}
+        {:else if $$slots.header}
           <slot name="header" />
+        {/if}
       </h1>
       {#if showDivider}
         <Divider size="M" />
       {/if}
     {/if}
+
     <!-- TODO: Remove content-grid class once Layout components are in bbui -->
     <section class="spectrum-Dialog-content content-grid">
       <slot />

packages/bbui/src/Popover/Popover.svelte

@@ -10,6 +10,17 @@
   export let anchor
   export let align = "right"
   export let portalTarget
+  export let dataCy
+
+  export let direction = "bottom"
+  export let showTip = false
+
+  let tipSvg =
+    '<svg xmlns="http://www.w3.org/svg/2000" width="23" height="12" class="spectrum-Popover-tip" > <path class="spectrum-Popover-tip-triangle" d="M 0.7071067811865476 0 L 11.414213562373096 10.707106781186548 L 22.121320343559645 0" /> </svg>'
+
+  $: tooltipClasses = showTip
+    ? `spectrum-Popover--withTip spectrum-Popover--${direction}`
+    : ""
 
   export const show = () => {
     dispatch("open")
@@ -37,9 +48,14 @@
       use:positionDropdown={{ anchor, align }}
       use:clickOutside={hide}
       on:keydown={handleEscape}
-      class="spectrum-Popover is-open"
+      class={"spectrum-Popover is-open " + (tooltipClasses || "")}
       role="presentation"
+      data-cy={dataCy}
     >
+      {#if showTip}
+        {@html tipSvg}
+      {/if}
+
       <slot />
     </div>
   </Portal>
@@ -49,4 +65,13 @@
   .spectrum-Popover {
     min-width: var(--spectrum-global-dimension-size-2000) !important;
   }
+  .spectrum-Popover.is-open.spectrum-Popover--withTip {
+    margin-top: var(--spacing-xs);
+    margin-left: var(--spacing-xl);
+  }
+  :global(.spectrum-Popover--bottom .spectrum-Popover-tip),
+  :global(.spectrum-Popover--top .spectrum-Popover-tip) {
+    left: 90%;
+    margin-left: calc(var(--spectrum-global-dimension-size-150) * -1);
+  }
 </style>

packages/builder/cypress/integration/appPublishWorkflow.spec.js

@@ -0,0 +1,112 @@
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+  context("Publish Application Workflow", () => {
+    before(() => {
+      cy.login()
+      cy.createTestApp()
+    })
+
+    it("Should reflect the unpublished status correctly", () => {
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.wait(1000)
+
+      cy.get(".appTable .app-status").eq(0)
+      .within(() => {
+        cy.contains("Unpublished")
+        cy.get("svg[aria-label='GlobeStrike']").should("exist")
+      })
+
+      cy.get(".appTable .app-row-actions").eq(0)
+      .within(() => {
+        cy.get(".spectrum-Button").contains("Preview")
+        cy.get(".spectrum-Button").contains("Edit").click({ force: true })
+      })
+      
+      cy.get(".deployment-top-nav svg[aria-label='GlobeStrike']").should("exist")
+      cy.get(".deployment-top-nav svg[aria-label='Globe']").should("not.exist")
+    })
+
+    it("Should publish an application and correctly reflect that", () => {
+      //Assuming the previous test was run and the unpublished app is open in edit mode.
+      cy.get(".toprightnav button.spectrum-Button").contains("Publish").click({ force : true })
+
+      cy.get(".spectrum-Modal [data-cy='deploy-app-modal']").should("be.visible")
+      .within(() => {
+        cy.get(".spectrum-Button").contains("Publish").click({ force : true })
+        cy.wait(1000)
+      });
+
+      //Verify that the app url is presented correctly to the user
+      cy.get(".spectrum-Modal [data-cy='deploy-app-success-modal']")
+      .should("be.visible")
+      .within(() => {
+        let appUrl = Cypress.config().baseUrl + '/app/cypress-tests'
+        cy.get("[data-cy='deployed-app-url'] input").should('have.value', appUrl)
+        cy.get(".spectrum-Button").contains("Done").click({ force: true })
+      })
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.wait(1000)
+
+      cy.get(".appTable .app-status").eq(0)
+      .within(() => {
+        cy.contains("Published")
+        cy.get("svg[aria-label='Globe']").should("exist")
+      })
+
+      cy.get(".appTable .app-row-actions").eq(0)
+      .within(() => {
+        cy.get(".spectrum-Button").contains("View app")
+        cy.get(".spectrum-Button").contains("Edit").click({ force: true })
+      })
+
+      cy.get(".deployment-top-nav svg[aria-label='Globe']").should("exist").click({ force: true })
+      
+      cy.get("[data-cy='publish-popover-menu']").should("be.visible")
+      .within(() => {
+        cy.get("[data-cy='publish-popover-action']").should("exist")
+        cy.get("button").contains("View app").should("exist")
+        cy.get(".publish-popover-message").should("have.text", "Last published a few seconds ago")
+      })
+    })
+
+    it("Should unpublish an application from the top navigation and reflect the status change", () => {
+      //Assuming the previous test app exists and is published
+      
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+
+      cy.get(".appTable .app-status").eq(0)
+      .within(() => {
+        cy.contains("Published")
+        cy.get("svg[aria-label='Globe']").should("exist")
+      })
+
+      cy.get(".appTable .app-row-actions").eq(0)
+      .within(() => {
+        cy.get(".spectrum-Button").contains("View app")
+        cy.get(".spectrum-Button").contains("Edit").click({ force: true })
+      })
+
+      //The published status 
+      cy.get(".deployment-top-nav svg[aria-label='Globe']").should("exist")
+      .click({ force: true })
+
+      cy.get("[data-cy='publish-popover-menu']").should("be.visible")
+      cy.get("[data-cy='publish-popover-menu'] [data-cy='publish-popover-action']")
+      .click({ force : true })
+
+      cy.get("[data-cy='unpublish-modal']").should("be.visible")
+      .within(() => {
+        cy.get(".confirm-wrap button").click({ force: true }
+      )})
+
+      cy.get(".deployment-top-nav svg[aria-label='GlobeStrike']").should("exist")
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+
+      cy.get(".appTable .app-status").eq(0).contains("Unpublished")
+
+    })
+  })
+})

packages/builder/cypress/integration/changeAppIconAndColour.spec.js

@@ -11,7 +11,7 @@ filterTests(['all'], () => {
           cy.applicationInAppTable("Cypress Tests")
           cy.get(".appTable")
             .within(() => {
-              cy.get(".spectrum-Icon").eq(1).click()
+              cy.get(".app-row-actions-icon").eq(0).click()
             })
           cy.get(".spectrum-Menu").contains("Edit icon").click()
           // Select random icon
@@ -38,6 +38,7 @@ filterTests(['all'], () => {
               cy.get(".title").children().children()
               .should('have.attr', 'style').and('contains', 'color')
             })
+          cy.deleteAllApps()
       })
   })
 })

packages/builder/cypress/integration/createAutomation.spec.js

@@ -11,7 +11,7 @@ filterTests(['smoke', 'all'], () => {
       cy.createTestTableWithData()
       cy.wait(2000)
       cy.contains("Automate").click()
-      cy.get("[data-cy='new-screen'] > .spectrum-Icon").click()
+      cy.get(".add-button .spectrum-Icon").click()
       cy.get(".modal-inner-wrapper").within(() => {
         cy.get("input").type("Add Row")
         cy.contains("Row Created").click({ force: true })

packages/builder/cypress/integration/createView.spec.js

@@ -125,7 +125,7 @@ filterTests(['smoke', 'all'], () => {
 
     it("renames a view", () => {
       cy.contains(".nav-item", "Test View")
-        .find(".actions .icon")
+        .find(".actions .icon.open-popover")
         .click({ force: true })
       cy.get(".spectrum-Menu-itemLabel").contains("Edit").click()
       cy.get(".modal-inner-wrapper").within(() => {
@@ -138,7 +138,7 @@ filterTests(['smoke', 'all'], () => {
 
     it("deletes a view", () => {
       cy.contains(".nav-item", "Test View Updated")
-        .find(".actions .icon")
+        .find(".actions .icon.open-popover")
         .click({ force: true })
       cy.contains("Delete").click()
       cy.contains("Delete View").click()

packages/builder/cypress/integration/renameAnApplication.spec.js

@@ -99,7 +99,7 @@ filterTests(['all'], () => {
       cy.searchForApplication(originalName)
       cy.get(".appTable")
         .within(() => {
-          cy.get(".spectrum-Icon").eq(1).click()
+          cy.get("[aria-label='More']").eq(0).click()
         })
       // Check for when an app is published
       if (published == true) {
@@ -109,7 +109,9 @@ filterTests(['all'], () => {
         cy.get(".spectrum-Dialog-grid").contains("Unpublish app").click()
         cy.get(".appTable > :nth-child(5) > :nth-child(2) > .spectrum-Icon").click()
       }
-        cy.contains("Edit").click()
+      cy.get("[data-cy='app-row-actions-menu-popover']").eq(0).within(() => {
+        cy.get(".spectrum-Menu-item").contains("Edit").click({ force: true })
+      })
       cy.get(".spectrum-Modal")
         .within(() => {
           if (noName == true) {

packages/builder/cypress/integration/revertApp.spec.js

@@ -10,9 +10,9 @@ filterTests(['smoke', 'all'], () => {
         it("should try to revert an unpublished app", () => {
             // Click revert icon
             cy.get(".toprightnav").within(() => {
-                cy.get(".spectrum-Icon").eq(1).click()
+                cy.get("[aria-label='Revert']").click({ force: true })
             })
-            cy.get(".spectrum-Dialog-grid").within(() => {
+            cy.get(".spectrum-Modal").within(() => {
                 // Enter app name before revert
                 cy.get("input").type("Cypress Tests")
                 cy.intercept('**/revert').as('revertApp')
@@ -33,11 +33,15 @@ filterTests(['smoke', 'all'], () => {
             cy.get(".spectrum-ButtonGroup").within(() => {
                 cy.get(".spectrum-Button").contains("Publish").click({ force: true })
             })
+            cy.wait(1000)
+            cy.get(".spectrum-ButtonGroup").within(() => {
+                cy.get(".spectrum-Button").contains("Done").click({ force: true })
+            })
             // Add second component - Button
             cy.addComponent("Elements", "Button")
             // Click Revert
             cy.get(".toprightnav").within(() => {
-                cy.get(".spectrum-Icon").eq(1).click()
+                cy.get("[aria-label='Revert']").click({ force: true })
             })
             cy.get(".spectrum-Dialog-grid").within(() => {
                 // Click Revert
@@ -54,7 +58,7 @@ filterTests(['smoke', 'all'], () => {
         it("should enter incorrect app name when reverting", () => {
             // Click Revert
             cy.get(".toprightnav").within(() => {
-                cy.get(".spectrum-Icon").eq(1).click({ force: true })
+                cy.get("[aria-label='Revert']").click({ force: true })
             })
             // Enter incorrect app name
             cy.get(".spectrum-Dialog-grid").within(() => {

packages/builder/cypress/support/commands.js

@@ -292,7 +292,7 @@ Cypress.Commands.add("createScreen", (route, accessLevelLabel) => {
   cy.contains("Design").click()
   cy.get("[aria-label=AddCircle]").click()
   cy.get(".spectrum-Modal").within(() => {
-    cy.get(".item").contains("Blank screen").click()
+    cy.get("[data-cy='blank-screen']").click()
     cy.get(".spectrum-Button").contains("Continue").click({ force: true })
     cy.wait(500)
   })
@@ -473,6 +473,7 @@ Cypress.Commands.add("selectExternalDatasource", datasourceName => {
     cy.get(".add-button").click()
   })
   // Clicks specified datasource & continue
+  cy.wait(1000)
   cy.get(".item-list").contains(datasourceName).click()
   cy.get(".spectrum-Dialog-grid").within(() => {
     cy.get(".spectrum-Button").contains("Continue").click({ force: true })
@@ -495,7 +496,7 @@ Cypress.Commands.add("addDatasourceConfig", (datasource, skipFetch) => {
           } else {
             cy.get("input")
               .clear({ force: true })
-              .type(Cypress.env("mysql").HOST, { force: true })
+              .type(Cypress.env("HOST_IP"), { force: true })
           }
         })
       })

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.105-alpha.42",
+    "@budibase/bbui": "^1.0.130-alpha.0",
-    "@budibase/client": "^1.0.105-alpha.42",
+    "@budibase/client": "^1.0.130-alpha.0",
-    "@budibase/frontend-core": "^1.0.105-alpha.42",
+    "@budibase/frontend-core": "^1.0.130-alpha.0",
-    "@budibase/string-templates": "^1.0.105-alpha.42",
+    "@budibase/string-templates": "^1.0.130-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/analytics/constants.js

@@ -36,6 +36,7 @@ export const Events = {
     CREATED: "budibase:app_created",
     PUBLISHED: "budibase:app_published",
     UNPUBLISHED: "budibase:app_unpublished",
+    VIEW_PUBLISHED: "budibase:view_published_app",
   },
   ANALYTICS: {
     OPT_IN: "budibase:analytics_opt_in",
@@ -51,3 +52,9 @@ export const Events = {
     SAVED: "budibase:sso_saved",
   },
 }
+
+export const EventSource = {
+  PORTAL: "portal",
+  URL: "url",
+  NOTIFICATION: "notification",
+}

packages/builder/src/analytics/index.js

@@ -2,7 +2,7 @@ import { API } from "api"
 import PosthogClient from "./PosthogClient"
 import IntercomClient from "./IntercomClient"
 import SentryClient from "./SentryClient"
-import { Events } from "./constants"
+import { Events, EventSource } from "./constants"
 
 const posthog = new PosthogClient(
   process.env.POSTHOG_TOKEN,
@@ -57,5 +57,5 @@ class AnalyticsHub {
 
 const analytics = new AnalyticsHub()
 
-export { Events }
+export { Events, EventSource }
 export default analytics

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ResultsModal.svelte

@@ -10,11 +10,11 @@
 <ModalContent
   showCloseIcon={false}
   showConfirmButton={false}
-  title="Test Results"
   cancelText="Close"
 >
-  <div slot="header">
+  <div slot="header" class="result-modal-header">
-    <div style="float: right;">
+    <span>Test Results</span>
+    <div>
       {#if isTrigger || testResult[0].outputs.success}
         <div class="iconSuccess">
           <Icon size="S" name="CheckmarkCircle" />
@@ -100,6 +100,14 @@
 </ModalContent>
 
 <style>
+  .result-modal-header {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: space-between;
+    width: 100%;
+  }
+
   .iconSuccess {
     color: var(--spectrum-global-color-green-600);
   }

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -16,7 +16,7 @@
       </Modal>
     </Tab>
   </Tabs>
-  <div class="add-button" data-cy="new-screen">
+  <div class="add-button">
     <Icon hoverable name="AddCircle" on:click={modal.show} />
   </div>
 </div>

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -165,7 +165,7 @@
           <TableFilterButton
             {schema}
             on:change={onFilter}
-            disabled={!hasCols || !hasRows}
+            disabled={!hasCols}
           />
         {/key}
       </div>

packages/builder/src/components/backend/DataTable/formula.js

@@ -60,6 +60,7 @@ export function getBindings({
     )
 
     const label = path == null ? column : `${path}.0.${column}`
+    const binding = path == null ? `[${column}]` : `${path}.0.[${column}]`
     // only supply a description for relationship paths
     const description =
       path == null
@@ -73,8 +74,8 @@ export function getBindings({
       description,
       // don't include path, it messes things up, relationship path
       // will be replaced by the main array binding
-      readableBinding: column,
+      readableBinding: label,
-      runtimeBinding: `[${column}]`,
+      runtimeBinding: binding,
     })
   }
   return bindings

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -15,7 +15,6 @@
   import ArrayRenderer from "components/common/renderers/ArrayRenderer.svelte"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { goto } from "@roxi/routify"
-  import GoogleButton from "../_components/GoogleButton.svelte"
 
   export let datasource
   export let save
@@ -161,11 +160,6 @@
       Fetch tables
     </Button>
     <Button cta icon="Add" on:click={createNewTable}>New table</Button>
-    {#if integration.auth}
-      {#if integration.auth.type === "google"}
-        <GoogleButton {datasource} />
-      {/if}
-    {/if}
   </div>
 </div>
 <Body>

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -40,7 +40,7 @@
 </script>
 
 <ActionMenu>
-  <div slot="control" class="icon">
+  <div slot="control" class="icon open-popover">
     <Icon s hoverable name="MoreSmallList" />
   </div>
   <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>

packages/builder/src/components/common/ConfirmDialog.svelte

@@ -9,6 +9,7 @@
   export let onCancel = undefined
   export let warning = true
   export let disabled
+  export let dataCy = null
 
   let modal
 
@@ -28,6 +29,7 @@
     {cancelText}
     {warning}
     {disabled}
+    {dataCy}
   >
     <Body size="S">
       {body}

packages/builder/src/components/common/inputs/CopyInput.svelte

@@ -4,6 +4,7 @@
   export let label = null
   export let value
   export let copyValue
+  export let dataCy = null
 
   const copyToClipboard = val => {
     const dummy = document.createElement("textarea")
@@ -16,7 +17,7 @@
   }
 </script>
 
-<div>
+<div data-cy={dataCy}>
   <Input readonly {value} {label} />
   <div class="icon" on:click={() => copyToClipboard(value || copyValue)}>
     <Icon size="S" name="Copy" />

packages/builder/src/components/deploy/DeployModal.svelte

@@ -1,24 +1,61 @@
 <script>
-  import { Button, Modal, notifications, ModalContent } from "@budibase/bbui"
+  import {
+    Button,
+    Modal,
+    notifications,
+    ModalContent,
+    Layout,
+  } from "@budibase/bbui"
   import { API } from "api"
-  import analytics, { Events } from "analytics"
+  import analytics, { Events, EventSource } from "analytics"
   import { store } from "builderStore"
+  import { ProgressCircle } from "@budibase/bbui"
+  import CopyInput from "components/common/inputs/CopyInput.svelte"
 
   let feedbackModal
   let publishModal
+  let asyncModal
+  let publishCompleteModal
+
+  let published
+
+  $: publishedUrl = published ? `${window.origin}/app${published.appUrl}` : ""
+
+  export let onOk
 
   async function deployApp() {
     try {
-      await API.deployAppChanges()
+      //In Progress
+      asyncModal.show()
+      publishModal.hide()
+
+      published = await API.deployAppChanges()
+
       analytics.captureEvent(Events.APP.PUBLISHED, {
         appId: $store.appId,
       })
-      notifications.success("Application published successfully")
+      if (typeof onOk === "function") {
+        await onOk()
+      }
+
+      //Request completed
+      asyncModal.hide()
+      publishCompleteModal.show()
     } catch (error) {
       analytics.captureException(error)
       notifications.error("Error publishing app")
     }
   }
+
+  const viewApp = () => {
+    if (published) {
+      analytics.captureEvent(Events.APP.VIEW_PUBLISHED, {
+        appId: $store.appId,
+        eventSource: EventSource.PORTAL,
+      })
+      window.open(publishedUrl, "_blank")
+    }
+  }
 </script>
 
 <Button secondary on:click={publishModal.show}>Publish</Button>
@@ -30,11 +67,13 @@
     showCancelButton={false}
   />
 </Modal>
+
 <Modal bind:this={publishModal}>
   <ModalContent
     title="Publish to Production"
     confirmText="Publish"
     onConfirm={deployApp}
+    dataCy={"deploy-app-modal"}
   >
     <span
       >The changes you have made will be published to the production version of
@@ -42,3 +81,57 @@
     >
   </ModalContent>
 </Modal>
+
+<!-- Publish in progress -->
+<Modal bind:this={asyncModal}>
+  <ModalContent
+    showCancelButton={false}
+    showConfirmButton={false}
+    showCloseIcon={false}
+  >
+    <Layout justifyItems="center">
+      <ProgressCircle size="XL" />
+    </Layout>
+  </ModalContent>
+</Modal>
+
+<!-- Publish complete -->
+<Modal bind:this={publishCompleteModal}>
+  <ModalContent
+    confirmText="Done"
+    cancelText="View App"
+    onCancel={viewApp}
+    dataCy={"deploy-app-success-modal"}
+  >
+    <div slot="header" class="app-published-header">
+      <svg
+        width="26px"
+        height="26px"
+        class="spectrum-Icon success-icon"
+        focusable="false"
+      >
+        <use xlink:href="#spectrum-icon-18-GlobeCheck" />
+      </svg>
+      <span class="app-published-header-text">App Published!</span>
+    </div>
+    <CopyInput
+      value={publishedUrl}
+      label="You can view your app at:"
+      dataCy="deployed-app-url"
+    />
+  </ModalContent>
+</Modal>
+
+<style>
+  .app-published-header {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+  }
+  .success-icon {
+    color: var(--spectrum-global-color-green-600);
+  }
+  .app-published-header .app-published-header-text {
+    padding-left: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/deploy/DeployNavigation.svelte

@@ -0,0 +1,189 @@
+<script>
+  import {
+    notifications,
+    Popover,
+    Layout,
+    Heading,
+    Body,
+    Button,
+    Icon,
+  } from "@budibase/bbui"
+  import { processStringSync } from "@budibase/string-templates"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import analytics, { Events, EventSource } from "analytics"
+  import { checkIncomingDeploymentStatus } from "components/deploy/utils"
+  import { API } from "api"
+  import { onMount } from "svelte"
+  import DeployModal from "components/deploy/DeployModal.svelte"
+  import { apps } from "stores/portal"
+
+  export let application
+
+  let publishPopover
+  let publishPopoverAnchor
+  let unpublishModal
+
+  $: filteredApps = $apps.filter(
+    app => app.devId === application && app.status === "published"
+  )
+  $: selectedApp = filteredApps?.length ? filteredApps[0] : null
+
+  $: deployments = []
+  $: latestDeployments = deployments
+    .filter(deployment => deployment.status === "SUCCESS")
+    .sort((a, b) => a.updatedAt > b.updatedAt)
+
+  $: isPublished = selectedApp && latestDeployments?.length > 0
+
+  const reviewPendingDeployments = (deployments, newDeployments) => {
+    if (deployments.length > 0) {
+      const pending = checkIncomingDeploymentStatus(deployments, newDeployments)
+      if (pending.length) {
+        notifications.warning(
+          "Deployment has been queued and will be processed shortly"
+        )
+      }
+    }
+  }
+
+  async function fetchDeployments() {
+    try {
+      const newDeployments = await API.getAppDeployments()
+      reviewPendingDeployments(deployments, newDeployments)
+      return newDeployments
+    } catch (err) {
+      notifications.error("Error fetching deployment history")
+    }
+  }
+
+  const viewApp = () => {
+    analytics.captureEvent(Events.APP.VIEW_PUBLISHED, {
+      appId: selectedApp.appId,
+      eventSource: EventSource.PORTAL,
+    })
+    if (selectedApp.url) {
+      window.open(`/app${selectedApp.url}`)
+    } else {
+      window.open(`/${selectedApp.prodId}`)
+    }
+  }
+
+  const unpublishApp = () => {
+    publishPopover.hide()
+    unpublishModal.show()
+  }
+
+  const confirmUnpublishApp = async () => {
+    if (!application || !isPublished) {
+      //confirm the app has loaded.
+      return
+    }
+    try {
+      analytics.captureEvent(Events.APP.UNPUBLISHED, {
+        appId: selectedApp.appId,
+      })
+      await API.unpublishApp(selectedApp.prodId)
+      await apps.load()
+      notifications.success("App unpublished successfully")
+    } catch (err) {
+      notifications.error("Error unpublishing app")
+    }
+  }
+
+  const completePublish = async () => {
+    try {
+      await apps.load()
+      deployments = await fetchDeployments()
+    } catch (err) {
+      notifications.error("Error refreshing app")
+    }
+  }
+
+  onMount(async () => {
+    if (!$apps.length) {
+      await apps.load()
+    }
+    deployments = await fetchDeployments()
+  })
+</script>
+
+<div class="deployment-top-nav">
+  {#if isPublished}
+    <div class="publish-popover">
+      <div bind:this={publishPopoverAnchor}>
+        <Icon
+          size="M"
+          hoverable
+          name="Globe"
+          tooltip="Your published app"
+          on:click={publishPopover.show()}
+        />
+      </div>
+      <Popover
+        bind:this={publishPopover}
+        align="right"
+        disabled={!isPublished}
+        dataCy="publish-popover-menu"
+        showTip={true}
+        anchor={publishPopoverAnchor}
+      >
+        <Layout gap="M">
+          <Heading size="XS">Your published app</Heading>
+          <Body size="S">
+            <span class="publish-popover-message">
+              {processStringSync(
+                "Last published {{ duration time 'millisecond' }} ago",
+                {
+                  time:
+                    new Date().getTime() -
+                    new Date(latestDeployments[0].updatedAt).getTime(),
+                }
+              )}
+            </span>
+          </Body>
+          <div class="publish-popover-actions">
+            <Button
+              warning={true}
+              icon="GlobeStrike"
+              disabled={!isPublished}
+              on:click={unpublishApp}
+              dataCy="publish-popover-action"
+            >
+              Unpublish
+            </Button>
+            <Button cta on:click={viewApp}>View app</Button>
+          </div>
+        </Layout>
+      </Popover>
+    </div>
+  {/if}
+
+  {#if !isPublished}
+    <Icon
+      size="M"
+      name="GlobeStrike"
+      disabled
+      tooltip="Your app has not been published yet"
+    />
+  {/if}
+</div>
+<ConfirmDialog
+  bind:this={unpublishModal}
+  title="Confirm unpublish"
+  okText="Unpublish app"
+  onOk={confirmUnpublishApp}
+  dataCy={"unpublish-modal"}
+>
+  Are you sure you want to unpublish the app <b>{selectedApp?.name}</b>?
+</ConfirmDialog>
+
+<DeployModal onOk={completePublish} />
+
+<style>
+  .publish-popover-actions :global([data-cy="publish-popover-action"]) {
+    margin-right: var(--spacing-s);
+  }
+  :global([data-cy="publish-popover-menu"]) {
+    padding: 10px;
+  }
+</style>

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -7,12 +7,10 @@
   import { notifications } from "@budibase/bbui"
   import CreateWebhookDeploymentModal from "./CreateWebhookDeploymentModal.svelte"
   import { store } from "builderStore"
-
+  import {
-  const DeploymentStatus = {
+    checkIncomingDeploymentStatus,
-    SUCCESS: "SUCCESS",
+    DeploymentStatus,
-    PENDING: "PENDING",
+  } from "components/deploy/utils"
-    FAILURE: "FAILURE",
-  }
 
   const DATE_OPTIONS = {
     fullDate: {
@@ -42,30 +40,17 @@
   const formatDate = (date, format) =>
     Intl.DateTimeFormat("en-GB", DATE_OPTIONS[format]).format(date)
 
-  // Required to check any updated deployment statuses between polls
-  function checkIncomingDeploymentStatus(current, incoming) {
-    for (let incomingDeployment of incoming) {
-      if (incomingDeployment.status === DeploymentStatus.FAILURE) {
-        const currentDeployment = current.find(
-          deployment => deployment._id === incomingDeployment._id
-        )
-
-        // We have just been notified of an ongoing deployments failure
-        if (
-          !currentDeployment ||
-          currentDeployment.status === DeploymentStatus.PENDING
-        ) {
-          showErrorReasonModal(incomingDeployment.err)
-        }
-      }
-    }
-  }
-
   async function fetchDeployments() {
     try {
       const newDeployments = await API.getAppDeployments()
       if (deployments.length > 0) {
-        checkIncomingDeploymentStatus(deployments, newDeployments)
+        const pendingDeployments = checkIncomingDeploymentStatus(
+          deployments,
+          newDeployments
+        )
+        if (pendingDeployments.length) {
+          showErrorReasonModal(pendingDeployments[0].err)
+        }
       }
       deployments = newDeployments
     } catch (err) {

packages/builder/src/components/deploy/utils.js

@@ -0,0 +1,25 @@
+export const DeploymentStatus = {
+  SUCCESS: "SUCCESS",
+  PENDING: "PENDING",
+  FAILURE: "FAILURE",
+}
+
+// Required to check any updated deployment statuses between polls
+export function checkIncomingDeploymentStatus(current, incoming) {
+  return incoming.reduce((acc, incomingDeployment) => {
+    if (incomingDeployment.status === DeploymentStatus.FAILURE) {
+      const currentDeployment = current.find(
+        deployment => deployment._id === incomingDeployment._id
+      )
+
+      //We have just been notified of an ongoing deployments failure
+      if (
+        !currentDeployment ||
+        currentDeployment.status === DeploymentStatus.PENDING
+      ) {
+        acc.push(incomingDeployment)
+      }
+    }
+    return acc
+  }, [])
+}

packages/builder/src/components/start/AppRow.svelte

@@ -15,6 +15,7 @@
   export let editApp
   export let updateApp
   export let deleteApp
+  export let previewApp
   export let unpublishApp
   export let releaseLock
   export let editIcon
@@ -22,7 +23,7 @@
 
 <div class="title">
   <div style="display: flex;">
-    <div style="color: {app.icon?.color || ''}">
+    <div class="app-icon" style="color: {app.icon?.color || ''}">
       <Icon size="XL" name={app.icon?.name || "Apps"} />
     </div>
     <div class="name" on:click={() => editApp(app)}>
@@ -57,26 +58,40 @@
   </StatusLight>
 </div>
 <div class="desktop">
-  <StatusLight active={app.deployed} neutral={!app.deployed}>
+  <div class="app-status">
-    {#if app.deployed}Published{:else}Unpublished{/if}
+    {#if app.deployed}
-  </StatusLight>
+      <Icon name="Globe" disabled={false} />
+      Published
+    {:else}
+      <Icon name="GlobeStrike" disabled={true} />
+      <span class="disabled"> Unpublished </span>
+    {/if}
+  </div>
 </div>
 <div data-cy={`row_actions_${app.appId}`}>
+  <div class="app-row-actions">
+    {#if app.deployed}
+      <Button size="S" secondary quiet on:click={() => viewApp(app)}
+        >View app
+      </Button>
+    {:else}
+      <Button size="S" secondary quiet on:click={() => previewApp(app)}
+        >Preview
+      </Button>
+    {/if}
     <Button
       size="S"
+      cta
       disabled={app.lockedOther}
       on:click={() => editApp(app)}
-    secondary
     >
-    Open
+      Edit
     </Button>
-  <ActionMenu align="right">
+  </div>
-    <Icon hoverable slot="control" name="More" />
+  <ActionMenu align="right" dataCy="app-row-actions-menu-popover">
-    {#if app.deployed}
+    <span slot="control" class="app-row-actions-icon">
-      <MenuItem on:click={() => viewApp(app)} icon="GlobeOutline">
+      <Icon hoverable name="More" />
-        View published app
+    </span>
-      </MenuItem>
-    {/if}
     {#if app.lockedYou}
       <MenuItem on:click={() => releaseLock(app)} icon="LockOpen">
         Release lock
@@ -97,6 +112,18 @@
 </div>
 
 <style>
+  .app-row-actions {
+    grid-gap: var(--spacing-s);
+    display: grid;
+    grid-template-columns: 75px 75px;
+  }
+  .app-status {
+    display: grid;
+    grid-template-columns: 24px 100px;
+  }
+  .app-status span.disabled {
+    opacity: 0.3;
+  }
   .name {
     text-decoration: none;
     overflow: hidden;

packages/builder/src/constants/backend/index.js

@@ -144,7 +144,11 @@ export const RelationshipTypes = {
   MANY_TO_ONE: "many-to-one",
 }
 
-export const ALLOWABLE_STRING_OPTIONS = [FIELDS.STRING, FIELDS.OPTIONS]
+export const ALLOWABLE_STRING_OPTIONS = [
+  FIELDS.STRING,
+  FIELDS.OPTIONS,
+  FIELDS.LONGFORM,
+]
 export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
   opt => opt.type
 )

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -2,9 +2,9 @@
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
   import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
-  import DeployModal from "components/deploy/DeployModal.svelte"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
+  import DeployNavigation from "components/deploy/DeployNavigation.svelte"
   import { API } from "api"
   import { auth, admin } from "stores/portal"
   import { isActive, goto, layout, redirect } from "@roxi/routify"
@@ -24,7 +24,7 @@
     $layout.children.find(layout => $isActive(layout.path))?.title ?? "data"
   )
 
-  function previewApp() {
+  const previewApp = () => {
     window.open(`/${application}`)
   }
 
@@ -112,7 +112,7 @@
         <VersionModal />
         <RevertModal />
         <Icon name="Play" hoverable on:click={previewApp} />
-        <DeployModal />
+        <DeployNavigation {application} />
       </div>
     </div>
     <slot />
@@ -128,7 +128,6 @@
     width: 100%;
     background: var(--background);
   }
-
   .root {
     min-height: 100%;
     height: 100%;

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte

@@ -136,7 +136,7 @@
         notifications.success("Request sent successfully")
       }
     } catch (error) {
-      notifications.error("Error running query")
+      notifications.error(`Query Error: ${error.message}`)
     }
   }
 

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -28,7 +28,7 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import AppRow from "components/start/AppRow.svelte"
   import { AppStatus } from "constants"
-  import analytics, { Events } from "analytics"
+  import analytics, { Events, EventSource } from "analytics"
   import Logo from "assets/bb-space-man.svg"
 
   let sortBy = "name"
@@ -167,6 +167,10 @@
   }
 
   const viewApp = app => {
+    analytics.captureEvent(Events.APP.VIEW_PUBLISHED, {
+      appId: app.appId,
+      eventSource: EventSource.PORTAL,
+    })
     if (app.url) {
       window.open(`/app${app.url}`)
     } else {
@@ -174,6 +178,10 @@
     }
   }
 
+  const previewApp = app => {
+    window.open(`/${app.devId}`)
+  }
+
   const editApp = app => {
     if (app.lockedOther) {
       notifications.error(
@@ -205,6 +213,9 @@
       return
     }
     try {
+      analytics.captureEvent(Events.APP.UNPUBLISHED, {
+        appId: selectedApp.appId,
+      })
       await API.unpublishApp(selectedApp.prodId)
       await apps.load()
       notifications.success("App unpublished successfully")
@@ -392,6 +403,7 @@
                 {exportApp}
                 {deleteApp}
                 {updateApp}
+                {previewApp}
               />
             {/each}
           </div>
@@ -444,6 +456,7 @@
   title="Confirm unpublish"
   okText="Unpublish app"
   onOk={confirmUnpublishApp}
+  dataCy={"unpublish-modal"}
 >
   Are you sure you want to unpublish the app <b>{selectedApp?.name}</b>?
 </ConfirmDialog>

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.105-alpha.42",
+    "@budibase/bbui": "^1.0.130-alpha.0",
-    "@budibase/frontend-core": "^1.0.105-alpha.42",
+    "@budibase/frontend-core": "^1.0.130-alpha.0",
-    "@budibase/string-templates": "^1.0.105-alpha.42",
+    "@budibase/string-templates": "^1.0.130-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/api/api.js

@@ -34,6 +34,7 @@ export const API = createAPIClient({
   // Or we could check error.status and redirect to login on a 403 etc.
   onError: error => {
     const { status, method, url, message, handled } = error || {}
+    const ignoreErrorUrls = ["bbtel", "/api/global/self"]
 
     // Log any errors that we haven't manually handled
     if (!handled) {
@@ -45,7 +46,14 @@ export const API = createAPIClient({
     if (message) {
       // Don't notify if the URL contains the word analytics as it may be
       // blocked by browser extensions
-      if (!url?.includes("analytics")) {
+      let ignore = false
+      for (let ignoreUrl of ignoreErrorUrls) {
+        if (url?.includes(ignoreUrl)) {
+          ignore = true
+          break
+        }
+      }
+      if (!ignore) {
         notificationStore.actions.error(message)
       }
     }

packages/client/src/components/app/Icon.svelte

@@ -36,8 +36,13 @@
   div {
     font-style: italic;
   }
+  @media (hover: hover) {
     .hoverable:hover {
       color: var(--spectrum-alias-icon-color-selected-hover) !important;
       cursor: pointer;
     }
+  }
+  .hoverable:active {
+    color: var(--spectrum-alias-icon-color-selected-hover) !important;
+  }
 </style>

packages/client/yarn.lock

@@ -775,9 +775,9 @@ minimatch@^3.0.2, minimatch@^3.0.4:
     brace-expansion "^1.1.7"
 
 minimist@^1.2.0:
-  version "1.2.5"
+  version "1.2.6"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
-  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
 nanoid@^2.1.0:
   version "2.1.11"
@@ -790,9 +790,9 @@ nanoid@^3.1.30, nanoid@^3.1.32:
   integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==
 
 nanoid@^3.3.1:
-  version "3.3.2"
+  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.2.tgz#c89622fafb4381cd221421c69ec58547a1eec557"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.1.tgz#6347a18cac88af88f58af0b3594b723d5e99bb35"
-  integrity sha512-CuHBogktKwpm5g2sRgv83jEy2ijFzBwMoYA60orPDR7ynsLijJDqgsi4RDGj3OJpy3Ieb+LYwiRmIOGyytgITA==
+  integrity sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==
 
 node-releases@^2.0.1:
   version "2.0.1"

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.105-alpha.42",
+    "@budibase/bbui": "^1.0.130-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.105-alpha.42",
+  "version": "1.0.130-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,10 +68,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.105-alpha.42",
+    "@budibase/backend-core": "^1.0.130-alpha.0",
-    "@budibase/client": "^1.0.105-alpha.42",
+    "@budibase/client": "^1.0.130-alpha.0",
-    "@budibase/pro": "1.0.105-alpha.42",
+    "@budibase/pro": "1.0.130-alpha.0",
-    "@budibase/string-templates": "^1.0.105-alpha.42",
+    "@budibase/string-templates": "^1.0.130-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -121,7 +121,7 @@
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "posthog-node": "^1.1.4",
-    "pouchdb": "7.2.1",
+    "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "^7.2.1",
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "^7.2.2",

packages/server/scripts/exportAppTemplate.js

@@ -2,7 +2,8 @@
 const yargs = require("yargs")
 const fs = require("fs")
 const { join } = require("path")
-const CouchDB = require("../src/db")
+require("../src/db").init()
+const { doWithDB } = require("@budibase/backend-core/db")
 // load environment
 const env = require("../src/environment")
 const {
@@ -47,14 +48,15 @@ yargs
       const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
       // perform couch dump
 
-      const instanceDb = new CouchDB(appId)
+      await doWithDB(appId, async db => {
-      await instanceDb.dump(writeStream, {
+        return db.dump(writeStream, {
           filter: doc =>
             !(
               doc._id.includes(USER_METDATA_PREFIX) ||
               doc.includes(LINK_USER_METADATA_PREFIX)
             ),
         })
+      })
       console.log(`Template ${name} exported to ${exportPath}`)
     }
   )

packages/server/scripts/integrations/mysql/init.sql

@@ -26,6 +26,7 @@ CREATE TABLE Products (
     updated time
 );
 INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
+INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Dave', 'Johnson', 29, '124 Fake Street', 'Belfast', '2022-04-01 00:11:11');
 INSERT INTO Tasks (PersonID, TaskName, CreatedAt) VALUES (1, 'assembling', '2020-01-01');
 INSERT INTO Tasks (PersonID, TaskName, CreatedAt) VALUES (2, 'processing', '2019-12-31');
 INSERT INTO Products (name, updated) VALUES ('Meat', '11:00:22'), ('Fruit', '10:00:00');

packages/server/scripts/replicateApp.js

@@ -1,50 +0,0 @@
-/**
- * Script to replicate your PouchDb (in your home directory) to a remote CouchDB
- * USAGE...
- * node scripts/replicateApp <app_name> <remote_url>
- * e.g. node scripts/replicateApp Mike http://admin:password@127.0.0.1:5984
- */
-
-const CouchDB = require("../src/db")
-const { DocumentTypes } = require("../src/db/utils")
-const { getAllDbs } = require("@budibase/backend-core/db")
-
-const appName = process.argv[2].toLowerCase()
-const remoteUrl = process.argv[3]
-
-console.log(`Replicating from ${appName} to ${remoteUrl}/${appName}`)
-
-const run = async () => {
-  const dbs = await getAllDbs()
-  const appDbNames = dbs.filter(dbName => dbName.startsWith("inst_app"))
-  let apps = []
-  for (let dbName of appDbNames) {
-    const db = new CouchDB(dbName)
-    apps.push(db.get(DocumentTypes.APP_METADATA))
-  }
-  apps = await Promise.all(apps)
-  const app = apps.find(
-    a => a.name === appName || a.name.toLowerCase() === appName
-  )
-
-  if (!app) {
-    console.log(
-      `Could not find app... apps: ${apps.map(app => app.name).join(", ")}`
-    )
-    return
-  }
-
-  const instanceDb = new CouchDB(app.appId)
-  const remoteDb = new CouchDB(`${remoteUrl}/${appName}`)
-
-  instanceDb.replicate
-    .to(remoteDb)
-    .on("complete", function () {
-      console.log("SUCCESS!")
-    })
-    .on("error", function (err) {
-      console.log(`FAILED: ${err}`)
-    })
-}
-
-run()

packages/server/src/api/controllers/application.ts

@@ -131,7 +131,7 @@ async function createInstance(template: any) {
   const tenantId = isMultiTenant() ? getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
-  updateAppId(appId)
+  await updateAppId(appId)
 
   const db = getAppDB()
   await db.put({
@@ -386,24 +386,35 @@ export const revertClient = async (ctx: any) => {
 }
 
 const destroyApp = async (ctx: any) => {
-  const db = getAppDB()
+  let appId = ctx.params.appId
+  let isUnpublish = ctx.query && ctx.query.unpublish
 
+  if (isUnpublish) {
+    appId = getProdAppID(appId)
+  }
+
+  const db = isUnpublish ? getProdAppDB() : getAppDB()
   const result = await db.destroy()
-  if (ctx.query?.unpublish) {
+
+  if (isUnpublish) {
     await quotas.removePublishedApp()
   } else {
     await quotas.removeApp()
   }
+
   /* istanbul ignore next */
-  if (!env.isTest() && !ctx.query.unpublish) {
+  if (!env.isTest() && !isUnpublish) {
-    await deleteApp(ctx.params.appId)
+    await deleteApp(appId)
   }
-  if (ctx.query && ctx.query.unpublish) {
+  // automations only in production
-    await cleanupAutomations(ctx.params.appId)
+  if (isUnpublish) {
+    await cleanupAutomations(appId)
   }
-  // make sure the app/role doesn't stick around after the app has been deleted
+  // remove app role when the dev app is deleted (no trace of app anymore)
-  await removeAppFromUserRoles(ctx, ctx.params.appId)
+  else {
-  await appCache.invalidateAppMetadata(ctx.params.appId)
+    await removeAppFromUserRoles(ctx, appId)
+  }
+  await appCache.invalidateAppMetadata(appId)
   return result
 }
 
@@ -463,6 +474,8 @@ export const sync = async (ctx: any, next: any) => {
     })
   } catch (err) {
     error = err
+  } finally {
+    await replication.close()
   }
 
   // sync the users

packages/server/src/api/controllers/deploy/index.ts

@@ -93,6 +93,7 @@ async function initDeployedApp(prodAppId: any) {
 }
 
 async function deployApp(deployment: any) {
+  let replication
   try {
     const appId = getAppId()
     const devAppId = getDevelopmentAppID(appId)
@@ -102,14 +103,16 @@ async function deployApp(deployment: any) {
       source: devAppId,
       target: productionAppId,
     }
-    const replication = new Replication(config)
+    replication = new Replication(config)
 
     console.log("Replication object created")
-
     await replication.replicate()
     console.log("replication complete.. replacing app meta doc")
     const db = getProdAppDB()
     const appDoc = await db.get(DocumentTypes.APP_METADATA)
+
+    deployment.appUrl = appDoc.url
+
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
@@ -126,6 +129,10 @@ async function deployApp(deployment: any) {
       ...err,
       message: `Deployment Failed: ${err.message}`,
     }
+  } finally {
+    if (replication) {
+      await replication.close()
+    }
   }
 }
 

packages/server/src/api/controllers/dev.js

@@ -83,7 +83,9 @@ exports.revert = async ctx => {
   try {
     const db = getProdAppDB({ skip_setup: true })
     const info = await db.info()
-    if (info.error) throw info.error
+    if (info.error) {
+      throw info.error
+    }
     const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
     if (
       !deploymentDoc.history ||
@@ -95,12 +97,11 @@ exports.revert = async ctx => {
     return ctx.throw(400, "App has not yet been deployed")
   }
 
-  try {
   const replication = new Replication({
     source: productionAppId,
     target: appId,
   })
-
+  try {
     await replication.rollback()
     // update appID in reverted app to be dev version again
     const db = getAppDB()
@@ -114,6 +115,8 @@ exports.revert = async ctx => {
     }
   } catch (err) {
     ctx.throw(400, `Unable to revert. ${err}`)
+  } finally {
+    await replication.close()
   }
 }
 

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -1,12 +1,4 @@
-
+const TestConfig = require("../../../../../tests/utilities/TestConfiguration")
-const bulkDocs = jest.fn()
-const db = jest.fn(() => {
-  return {
-    bulkDocs
-  }
-})
-jest.mock("../../../../../db", () => db)
-require("@budibase/backend-core").init(require("../../../../../db"))
 
 const { RestImporter } = require("../index")
 
@@ -48,6 +40,12 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
+  const config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
   let restImporter 
 
   const init = async (data) => {
@@ -105,11 +103,9 @@ describe("Rest Importer", () => {
 
   const testImportQueries = async (key, data, assertions) => {
     await init(data)
-    bulkDocs.mockReturnValue([])
     const importResult = await restImporter.importQueries("datasourceId")
     expect(importResult.errorQueries.length).toBe(0)
     expect(importResult.queries.length).toBe(assertions[key].count)
-    expect(bulkDocs).toHaveBeenCalledTimes(1)
     jest.clearAllMocks()
   }
 

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -324,6 +324,28 @@ module External {
       return { row: newRow, manyRelationships }
     }
 
+    squashRelationshipColumns(
+      table: Table,
+      row: Row,
+      relationships: RelationshipsJson[]
+    ): Row {
+      for (let relationship of relationships) {
+        const linkedTable = this.tables[relationship.tableName]
+        if (!linkedTable || !row[relationship.column]) {
+          continue
+        }
+        const display = linkedTable.primaryDisplay
+        for (let key of Object.keys(row[relationship.column])) {
+          const related: Row = row[relationship.column][key]
+          row[relationship.column][key] = {
+            primaryDisplay: display ? related[display] : undefined,
+            _id: related._id,
+          }
+        }
+      }
+      return row
+    }
+
     /**
      * This iterates through the returned rows and works out what elements of the rows
      * actually match up to another row (based on primary keys) - this is pretty specific
@@ -355,12 +377,6 @@ module External {
         if (!linked._id) {
           continue
         }
-        // if not returning full docs then get the minimal links out
-        const display = linkedTable.primaryDisplay
-        linked = {
-          primaryDisplay: display ? linked[display] : undefined,
-          _id: linked._id,
-        }
         columns[relationship.column] = linked
       }
       for (let [column, related] of Object.entries(columns)) {
@@ -418,7 +434,9 @@ module External {
           relationships
         )
       }
-      return processFormulas(table, Object.values(finalRows))
+      return processFormulas(table, Object.values(finalRows)).map((row: Row) =>
+        this.squashRelationshipColumns(table, row, relationships)
+      )
     }
 
     /**

packages/server/src/api/controllers/row/internal.js

@@ -6,6 +6,7 @@ const {
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
+const { dangerousGetDB } = require("@budibase/backend-core/db")
 const userController = require("../user")
 const {
   inputProcessing,
@@ -250,7 +251,7 @@ exports.fetch = async ctx => {
 }
 
 exports.find = async ctx => {
-  const db = getAppDB()
+  const db = dangerousGetDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
   row = await outputProcessing(table, row)

packages/server/src/api/controllers/row/internalSearch.js

@@ -1,6 +1,6 @@
 const { SearchIndexes } = require("../../../db/utils")
 const fetch = require("node-fetch")
-const { getCouchUrl } = require("@budibase/backend-core/db")
+const { getCouchInfo } = require("@budibase/backend-core/db")
 const { getAppId } = require("@budibase/backend-core/context")
 
 /**
@@ -242,11 +242,10 @@ class QueryBuilder {
 
   async run() {
     const appId = getAppId()
-    const url = `${getCouchUrl()}/${appId}/_design/database/_search/${
+    const { url, cookie } = getCouchInfo()
-      SearchIndexes.ROWS
+    const fullPath = `${url}/${appId}/_design/database/_search/${SearchIndexes.ROWS}`
-    }`
     const body = this.buildSearchBody()
-    return await runQuery(url, body)
+    return await runQuery(fullPath, body, cookie)
   }
 }
 
@@ -254,12 +253,16 @@ class QueryBuilder {
  * Executes a lucene search query.
  * @param url The query URL
  * @param body The request body defining search criteria
+ * @param cookie The auth cookie for CouchDB
  * @returns {Promise<{rows: []}>}
  */
-const runQuery = async (url, body) => {
+const runQuery = async (url, body, cookie) => {
   const response = await fetch(url, {
     body: JSON.stringify(body),
     method: "POST",
+    headers: {
+      Authorization: cookie,
+    },
   })
   const json = await response.json()
 

packages/server/src/api/controllers/table/internal.ts

@@ -9,9 +9,31 @@ import {
 } from "./utils"
 const { getAppDB } = require("@budibase/backend-core/context")
 import { isTest } from "../../../environment"
-import { cleanupAttachments } from "../../../utilities/rowProcessor"
+import {
+  cleanupAttachments,
+  fixAutoColumnSubType,
+} from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
-import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+import { Table } from "../../../definitions/common"
+import { quotas } from "@budibase/pro"
+
+function checkAutoColumns(table: Table, oldTable: Table) {
+  if (!table.schema) {
+    return table
+  }
+  for (let [key, schema] of Object.entries(table.schema)) {
+    if (!schema.autocolumn || schema.subtype) {
+      continue
+    }
+    const oldSchema = oldTable && oldTable.schema[key]
+    if (oldSchema && oldSchema.subtype) {
+      table.schema[key].subtype = oldSchema.subtype
+    } else {
+      table.schema[key] = fixAutoColumnSubType(schema)
+    }
+  }
+  return table
+}
 
 export async function save(ctx: any) {
   const db = getAppDB()
@@ -29,9 +51,12 @@ export async function save(ctx: any) {
     oldTable = await db.get(ctx.request.body._id)
   }
 
+  // check all types are correct
   if (hasTypeChanged(tableToSave, oldTable)) {
     ctx.throw(400, "A column type has changed.")
   }
+  // check that subtypes have been maintained
+  tableToSave = checkAutoColumns(tableToSave, oldTable)
 
   // saving a table is a complex operation, involving many different steps, this
   // has been broken out into a utility to make it more obvious/easier to manipulate

packages/server/src/api/controllers/webhook.js

@@ -54,7 +54,7 @@ exports.destroy = async ctx => {
 }
 
 exports.buildSchema = async ctx => {
-  updateAppId(ctx.params.instance)
+  await updateAppId(ctx.params.instance)
   const db = getAppDB()
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
@@ -80,7 +80,7 @@ exports.buildSchema = async ctx => {
 
 exports.trigger = async ctx => {
   const prodAppId = getProdAppID(ctx.params.instance)
-  updateAppId(prodAppId)
+  await updateAppId(prodAppId)
   try {
     const db = getAppDB()
     const webhook = await db.get(ctx.params.id)

packages/server/src/api/routes/tests/misc.spec.js

@@ -43,6 +43,7 @@ describe("run misc tests", () => {
 
   describe("test table utilities", () => {
     it("should be able to import a CSV", async () => {
+      return config.doInContext(null, async () => {
         const table = await config.createTable({
           name: "table",
           type: "table",
@@ -93,3 +94,4 @@ describe("run misc tests", () => {
       })
     })
   })
+})

packages/server/src/api/routes/tests/routing.spec.js

@@ -2,7 +2,6 @@ const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint, runInProd } = require("./utilities/TestFunctions")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
-const { doInAppContext } = require("@budibase/backend-core/context")
 
 const route = "/test"
 

packages/server/src/api/routes/tests/row.spec.js

@@ -2,6 +2,7 @@ const { outputProcessing } = require("../../../utilities/rowProcessor")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 const { doInAppContext } = require("@budibase/backend-core/context")
+const { doInTenant } = require("@budibase/backend-core/tenancy")
 
 // mock the fetch for the search system
 jest.mock("node-fetch")
@@ -340,6 +341,7 @@ describe("/rows", () => {
 
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
+      const { table, firstRow, secondRow } = await doInTenant(setup.structures.TENANT_ID, async () => {
         const table = await config.createLinkedTable()
         const firstRow = await config.createRow({
           name: "Test Contact",
@@ -352,6 +354,8 @@ describe("/rows", () => {
           link: [{_id: firstRow._id}],
           tableId: table._id,
         })
+        return { table, firstRow, secondRow }
+      })
 
       // test basic enrichment
       const resBasic = await request

packages/server/src/api/routes/tests/table.spec.js

@@ -1,4 +1,5 @@
-const { checkBuilderEndpoint, getDB } = require("./utilities/TestFunctions")
+const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { getAppDB } = require("@budibase/backend-core/context")
 const setup = require("./utilities")
 const { basicTable } = setup.structures
 
@@ -122,7 +123,7 @@ describe("/tables", () => {
 
   describe("indexing", () => {
     it("should be able to create a table with indexes", async () => {
-      const db = getDB(config)
+      const db = getAppDB(config)
       const indexCount = (await db.getIndexes()).total_rows
       const table = basicTable()
       table.indexes = ["name"]

packages/server/src/api/routes/tests/utilities/TestFunctions.ts

@@ -2,6 +2,7 @@ import * as rowController from "../../../controllers/row"
 import * as appController from "../../../controllers/application"
 import { AppStatus } from "../../../../db/utils"
 import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/roles"
+import { doInTenant } from "@budibase/backend-core/tenancy"
 import { TENANT_ID } from "../../../../tests/utilities/structures"
 import { getAppDB, doInAppContext } from "@budibase/backend-core/context"
 import * as env from "../../../../environment"
@@ -32,6 +33,7 @@ export const getAllTableRows = async (config: any) => {
 }
 
 export const clearAllApps = async (tenantId = TENANT_ID) => {
+  await doInTenant(tenantId, async () => {
     const req: any = { query: { status: AppStatus.DEV }, user: { tenantId } }
     await appController.fetch(req)
     const apps = req.body
@@ -43,6 +45,7 @@ export const clearAllApps = async (tenantId = TENANT_ID) => {
       const req = new Request(null, { appId })
       await runRequest(appId, appController.destroy, req)
     }
+  })
 }
 
 export const clearAllAutomations = async (config: any) => {

packages/server/src/app.ts

@@ -1,8 +1,8 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 import * as env from "./environment"
-const CouchDB = require("./db")
+import db from "./db"
-require("@budibase/backend-core").init(CouchDB)
+db.init()
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")

packages/server/src/automations/steps/bash.js

@@ -1,6 +1,7 @@
 const { execSync } = require("child_process")
 const { processStringSync } = require("@budibase/string-templates")
 const automationUtils = require("../automationUtils")
+const environment = require("../../environment")
 
 exports.definition = {
   name: "Bash Scripting",
@@ -51,7 +52,9 @@ exports.run = async function ({ inputs, context }) {
     let stdout,
       success = true
     try {
-      stdout = execSync(command, { timeout: 500 }).toString()
+      stdout = execSync(command, {
+        timeout: environment.QUERY_THREAD_TIMEOUT || 500,
+      }).toString()
     } catch (err) {
       stdout = err.message
       success = false

packages/server/src/automations/tests/deleteRow.spec.ts

@@ -26,7 +26,7 @@ describe("test the delete row action", () => {
     expect(res.row._id).toEqual(row._id)
     let error
     try {
-      await config.getRow(table._id, res.id)
+      await config.getRow(table._id, res.row._id)
     } catch (err) {
       error = err
     }

packages/server/src/automations/tests/utilities/index.js

@@ -1,4 +1,6 @@
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { TENANT_ID } = require("../../../tests/utilities/structures")
+const { doInTenant } = require("@budibase/backend-core/tenancy")
 const actions = require("../../actions")
 const emitter = require("../../../events/index")
 const env = require("../../../environment")
@@ -31,6 +33,7 @@ exports.runInProd = async fn => {
 }
 
 exports.runStep = async function runStep(stepId, inputs) {
+  return doInTenant(TENANT_ID, async () => {
     let step = await actions.getAction(stepId)
     expect(step).toBeDefined()
     return step({
@@ -40,6 +43,7 @@ exports.runStep = async function runStep(stepId, inputs) {
       apiKey: exports.apiKey,
       emitter,
     })
+  })
 }
 
 exports.apiKey = "test"

packages/server/src/automations/utils.ts

@@ -1,12 +1,11 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
 import * as webhooks from "../api/controllers/webhook"
-import CouchDB from "../db"
 import { queue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
 import { MetadataTypes, WebhookType } from "../constants"
-import { getProdAppID } from "@budibase/backend-core/db"
+import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
 import { cloneDeep } from "lodash/fp"
 import { getAppDB, getAppId } from "@budibase/backend-core/context"
 import { tenancy } from "@budibase/backend-core"
@@ -113,10 +112,11 @@ export async function enableCronTrigger(appId: any, automation: any) {
     // can't use getAppDB here as this is likely to be called from dev app,
     // but this call could be for dev app or prod app, need to just use what
     // was passed in
-    const db = new CouchDB(appId)
+    await doWithDB(appId, async (db: any) => {
       const response = await db.put(automation)
       automation._id = response.id
       automation._rev = response.rev
+    })
   }
   return automation
 }

packages/server/src/constants/index.js

@@ -47,7 +47,11 @@ exports.FieldTypes = {
 
 exports.CanSwitchTypes = [
   [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
-  [exports.FieldTypes.STRING, exports.FieldTypes.OPTIONS],
+  [
+    exports.FieldTypes.STRING,
+    exports.FieldTypes.OPTIONS,
+    exports.FieldTypes.LONGFORM,
+  ],
   [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
 ]
 
@@ -162,6 +166,14 @@ exports.AutoFieldSubTypes = {
   AUTO_ID: "autoID",
 }
 
+exports.AutoFieldDefaultNames = {
+  CREATED_BY: "Created By",
+  CREATED_AT: "Created At",
+  UPDATED_BY: "Updated By",
+  UPDATED_AT: "Updated At",
+  AUTO_ID: "Auto ID",
+}
+
 exports.OBJ_STORE_DIRECTORY = "/prod-budi-app-assets"
 exports.BaseQueryVerbs = {
   CREATE: "create",

packages/server/src/db/client.js

@@ -1,31 +0,0 @@
-const PouchDB = require("pouchdb")
-const { getCouchUrl } = require("@budibase/backend-core/db")
-const replicationStream = require("pouchdb-replication-stream")
-const allDbs = require("pouchdb-all-dbs")
-const find = require("pouchdb-find")
-const env = require("../environment")
-
-const COUCH_DB_URL = getCouchUrl() || "http://localhost:4005"
-
-PouchDB.plugin(replicationStream.plugin)
-PouchDB.plugin(find)
-PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
-
-let POUCH_DB_DEFAULTS = {
-  prefix: COUCH_DB_URL,
-}
-
-if (env.isTest()) {
-  PouchDB.plugin(require("pouchdb-adapter-memory"))
-  POUCH_DB_DEFAULTS = {
-    prefix: undefined,
-    adapter: "memory",
-  }
-}
-
-const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS)
-
-// have to still have pouch alldbs for testing
-allDbs(Pouch)
-
-module.exports = Pouch

packages/server/src/db/inMemoryView.js

@@ -1,18 +1,16 @@
-const PouchDB = require("pouchdb")
-const memory = require("pouchdb-adapter-memory")
 const newid = require("./newid")
 
-PouchDB.plugin(memory)
+// bypass the main application db config
-const Pouch = PouchDB.defaults({
+// use in memory pouchdb directly
-  prefix: undefined,
+const { getPouch, closeDB } = require("@budibase/backend-core/db")
-  adapter: "memory",
+const Pouch = getPouch({ inMemory: true })
-})
 
 exports.runView = async (view, calculation, group, data) => {
   // use a different ID each time for the DB, make sure they
   // are always unique for each query, don't want overlap
   // which could cause 409s
   const db = new Pouch(newid())
+  try {
     // write all the docs to the in memory Pouch (remove revs)
     await db.bulkDocs(
       data.map(row => ({
@@ -43,6 +41,9 @@ exports.runView = async (view, calculation, group, data) => {
         row._rev = found._rev
       }
     }
-  await db.destroy()
     return response
+  } finally {
+    await db.destroy()
+    await closeDB(db)
+  }
 }

packages/server/src/db/index.js

@@ -1,3 +1,16 @@
-const client = require("./client")
+const core = require("@budibase/backend-core")
+const env = require("../environment")
 
-module.exports = client
+exports.init = () => {
+  const dbConfig = {
+    replication: true,
+    find: true,
+  }
+
+  if (env.isTest()) {
+    dbConfig.inMemory = true
+    dbConfig.allDbs = true
+  }
+
+  core.init({ db: dbConfig })
+}

packages/server/src/db/linkedRows/LinkController.js

@@ -376,6 +376,7 @@ class LinkController {
 
         if (field.autocolumn) {
           linkedField.autocolumn = field.autocolumn
+          linkedField.subtype = field.subtype
         }
 
         // check the linked table to make sure we aren't overwriting an existing column