Merge branch 'develop' of github.com:Budibase/budibase into new-design-ui

.github/workflows/release-develop.yml

@@ -1,4 +1,5 @@
 name: Budibase Release Staging
+concurrency: release-develop
 
 on: 
  push:

.github/workflows/release-selfhost.yml

@@ -87,3 +87,10 @@ jobs:
             packages/cli/build/cli-macos 
             packages/server/specs/openapi.yaml
             packages/server/specs/openapi.json
+
+      - name: Discord Webhook Action
+        uses: tsickert/discord-webhook@v4.0.0
+        with:
+          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host."
+          embed-title: ${{ env.RELEASE_VERSION }}

.github/workflows/release.yml

@@ -1,4 +1,5 @@
 name: Budibase Release
+concurrency: release
 
 on:
  push:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -3,6 +3,8 @@
   "private": true,
   "devDependencies": {
     "@rollup/plugin-json": "^4.0.2",
+    "@types/mongodb": "3.6.3",
+    "@typescript-eslint/parser": "4.28.0",
     "babel-eslint": "^10.0.3",
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
@@ -16,7 +18,6 @@
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
     "svelte": "^3.38.2",
-    "@typescript-eslint/parser": "4.28.0",
     "typescript": "4.5.5"
   },
   "scripts": {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",
@@ -13,7 +13,7 @@
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.901.0",
     "bcryptjs": "^2.4.3",
-    "cls-hooked": "^4.2.2",
+    "emitter-listener": "^1.1.2",
     "ioredis": "^4.27.1",
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",

packages/backend-core/src/clshooked/index.js

@@ -0,0 +1,650 @@
+const util = require("util")
+const assert = require("assert")
+const wrapEmitter = require("emitter-listener")
+const async_hooks = require("async_hooks")
+
+const CONTEXTS_SYMBOL = "cls@contexts"
+const ERROR_SYMBOL = "error@context"
+
+const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED
+
+let currentUid = -1
+
+module.exports = {
+  getNamespace: getNamespace,
+  createNamespace: createNamespace,
+  destroyNamespace: destroyNamespace,
+  reset: reset,
+  ERROR_SYMBOL: ERROR_SYMBOL,
+}
+
+function Namespace(name) {
+  this.name = name
+  // changed in 2.7: no default context
+  this.active = null
+  this._set = []
+  this.id = null
+  this._contexts = new Map()
+  this._indent = 0
+  this._hook = null
+}
+
+Namespace.prototype.set = function set(key, value) {
+  if (!this.active) {
+    throw new Error(
+      "No context available. ns.run() or ns.bind() must be called first."
+    )
+  }
+
+  this.active[key] = value
+
+  if (DEBUG_CLS_HOOKED) {
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      indentStr +
+        "CONTEXT-SET KEY:" +
+        key +
+        "=" +
+        value +
+        " in ns:" +
+        this.name +
+        " currentUid:" +
+        currentUid +
+        " active:" +
+        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
+    )
+  }
+
+  return value
+}
+
+Namespace.prototype.get = function get(key) {
+  if (!this.active) {
+    if (DEBUG_CLS_HOOKED) {
+      const asyncHooksCurrentId = async_hooks.currentId()
+      const triggerId = async_hooks.triggerAsyncId()
+      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+      debug2(
+        `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}`
+      )
+    }
+    return undefined
+  }
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      indentStr +
+        "CONTEXT-GETTING KEY:" +
+        key +
+        "=" +
+        this.active[key] +
+        " (" +
+        this.name +
+        ") currentUid:" +
+        currentUid +
+        " active:" +
+        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
+    )
+    debug2(
+      `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${
+        this.active[key]
+      } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
+        this._set.length
+      } active:${util.inspect(this.active)}`
+    )
+  }
+  return this.active[key]
+}
+
+Namespace.prototype.createContext = function createContext() {
+  // Prototype inherit existing context if created a new child context within existing context.
+  let context = Object.create(this.active ? this.active : Object.prototype)
+  context._ns_name = this.name
+  context.id = currentUid
+
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      `${indentStr}CONTEXT-CREATED Context: (${
+        this.name
+      }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
+        this._set.length
+      } context:${util.inspect(context, {
+        showHidden: true,
+        depth: 2,
+        colors: true,
+      })}`
+    )
+  }
+
+  return context
+}
+
+Namespace.prototype.run = function run(fn) {
+  let context = this.createContext()
+  this.enter(context)
+
+  try {
+    if (DEBUG_CLS_HOOKED) {
+      const triggerId = async_hooks.triggerAsyncId()
+      const asyncHooksCurrentId = async_hooks.executionAsyncId()
+      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+      debug2(
+        `${indentStr}CONTEXT-RUN BEGIN: (${
+          this.name
+        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+          this._set.length
+        } context:${util.inspect(context)}`
+      )
+    }
+    fn(context)
+    return context
+  } catch (exception) {
+    if (exception) {
+      exception[ERROR_SYMBOL] = context
+    }
+    throw exception
+  } finally {
+    if (DEBUG_CLS_HOOKED) {
+      const triggerId = async_hooks.triggerAsyncId()
+      const asyncHooksCurrentId = async_hooks.executionAsyncId()
+      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+      debug2(
+        `${indentStr}CONTEXT-RUN END: (${
+          this.name
+        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+          this._set.length
+        } ${util.inspect(context)}`
+      )
+    }
+    this.exit(context)
+  }
+}
+
+Namespace.prototype.runAndReturn = function runAndReturn(fn) {
+  let value
+  this.run(function (context) {
+    value = fn(context)
+  })
+  return value
+}
+
+/**
+ * Uses global Promise and assumes Promise is cls friendly or wrapped already.
+ * @param {function} fn
+ * @returns {*}
+ */
+Namespace.prototype.runPromise = function runPromise(fn) {
+  let context = this.createContext()
+  this.enter(context)
+
+  let promise = fn(context)
+  if (!promise || !promise.then || !promise.catch) {
+    throw new Error("fn must return a promise.")
+  }
+
+  if (DEBUG_CLS_HOOKED) {
+    debug2(
+      "CONTEXT-runPromise BEFORE: (" +
+        this.name +
+        ") currentUid:" +
+        currentUid +
+        " len:" +
+        this._set.length +
+        " " +
+        util.inspect(context)
+    )
+  }
+
+  return promise
+    .then(result => {
+      if (DEBUG_CLS_HOOKED) {
+        debug2(
+          "CONTEXT-runPromise AFTER then: (" +
+            this.name +
+            ") currentUid:" +
+            currentUid +
+            " len:" +
+            this._set.length +
+            " " +
+            util.inspect(context)
+        )
+      }
+      this.exit(context)
+      return result
+    })
+    .catch(err => {
+      err[ERROR_SYMBOL] = context
+      if (DEBUG_CLS_HOOKED) {
+        debug2(
+          "CONTEXT-runPromise AFTER catch: (" +
+            this.name +
+            ") currentUid:" +
+            currentUid +
+            " len:" +
+            this._set.length +
+            " " +
+            util.inspect(context)
+        )
+      }
+      this.exit(context)
+      throw err
+    })
+}
+
+Namespace.prototype.bind = function bindFactory(fn, context) {
+  if (!context) {
+    if (!this.active) {
+      context = this.createContext()
+    } else {
+      context = this.active
+    }
+  }
+
+  let self = this
+  return function clsBind() {
+    self.enter(context)
+    try {
+      return fn.apply(this, arguments)
+    } catch (exception) {
+      if (exception) {
+        exception[ERROR_SYMBOL] = context
+      }
+      throw exception
+    } finally {
+      self.exit(context)
+    }
+  }
+}
+
+Namespace.prototype.enter = function enter(context) {
+  assert.ok(context, "context must be provided for entering")
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      `${indentStr}CONTEXT-ENTER: (${
+        this.name
+      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+        this._set.length
+      } ${util.inspect(context)}`
+    )
+  }
+
+  this._set.push(this.active)
+  this.active = context
+}
+
+Namespace.prototype.exit = function exit(context) {
+  assert.ok(context, "context must be provided for exiting")
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      `${indentStr}CONTEXT-EXIT: (${
+        this.name
+      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+        this._set.length
+      } ${util.inspect(context)}`
+    )
+  }
+
+  // Fast path for most exits that are at the top of the stack
+  if (this.active === context) {
+    assert.ok(this._set.length, "can't remove top context")
+    this.active = this._set.pop()
+    return
+  }
+
+  // Fast search in the stack using lastIndexOf
+  let index = this._set.lastIndexOf(context)
+
+  if (index < 0) {
+    if (DEBUG_CLS_HOOKED) {
+      debug2(
+        "??ERROR?? context exiting but not entered - ignoring: " +
+          util.inspect(context)
+      )
+    }
+    assert.ok(
+      index >= 0,
+      "context not currently entered; can't exit. \n" +
+        util.inspect(this) +
+        "\n" +
+        util.inspect(context)
+    )
+  } else {
+    assert.ok(index, "can't remove top context")
+    this._set.splice(index, 1)
+  }
+}
+
+Namespace.prototype.bindEmitter = function bindEmitter(emitter) {
+  assert.ok(
+    emitter.on && emitter.addListener && emitter.emit,
+    "can only bind real EEs"
+  )
+
+  let namespace = this
+  let thisSymbol = "context@" + this.name
+
+  // Capture the context active at the time the emitter is bound.
+  function attach(listener) {
+    if (!listener) {
+      return
+    }
+    if (!listener[CONTEXTS_SYMBOL]) {
+      listener[CONTEXTS_SYMBOL] = Object.create(null)
+    }
+
+    listener[CONTEXTS_SYMBOL][thisSymbol] = {
+      namespace: namespace,
+      context: namespace.active,
+    }
+  }
+
+  // At emit time, bind the listener within the correct context.
+  function bind(unwrapped) {
+    if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) {
+      return unwrapped
+    }
+
+    let wrapped = unwrapped
+    let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL]
+    Object.keys(unwrappedContexts).forEach(function (name) {
+      let thunk = unwrappedContexts[name]
+      wrapped = thunk.namespace.bind(wrapped, thunk.context)
+    })
+    return wrapped
+  }
+
+  wrapEmitter(emitter, attach, bind)
+}
+
+/**
+ * If an error comes out of a namespace, it will have a context attached to it.
+ * This function knows how to find it.
+ *
+ * @param {Error} exception Possibly annotated error.
+ */
+Namespace.prototype.fromException = function fromException(exception) {
+  return exception[ERROR_SYMBOL]
+}
+
+function getNamespace(name) {
+  return process.namespaces[name]
+}
+
+function createNamespace(name) {
+  assert.ok(name, "namespace must be given a name.")
+
+  if (DEBUG_CLS_HOOKED) {
+    debug2(`NS-CREATING NAMESPACE (${name})`)
+  }
+  let namespace = new Namespace(name)
+  namespace.id = currentUid
+
+  const hook = async_hooks.createHook({
+    init(asyncId, type, triggerId, resource) {
+      currentUid = async_hooks.executionAsyncId()
+
+      //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec
+      // let initContext = namespace.active;
+      // if(!initContext && triggerId) {
+      //   let parentContext = namespace._contexts.get(triggerId);
+      //   if (parentContext) {
+      //     namespace.active = parentContext;
+      //     namespace._contexts.set(currentUid, parentContext);
+      //     if (DEBUG_CLS_HOOKED) {
+      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
+      //       debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
+      //     }
+      //   } else if (DEBUG_CLS_HOOKED) {
+      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
+      //       debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
+      //     }
+      // }else {
+      //   namespace._contexts.set(currentUid, namespace.active);
+      //   if (DEBUG_CLS_HOOKED) {
+      //     const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
+      //     debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
+      //   }
+      // }
+      if (namespace.active) {
+        namespace._contexts.set(asyncId, namespace.active)
+
+        if (DEBUG_CLS_HOOKED) {
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} resource:${resource}`
+          )
+        }
+      } else if (currentUid === 0) {
+        // CurrentId will be 0 when triggered from C++. Promise events
+        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
+        const triggerId = async_hooks.triggerAsyncId()
+        const triggerIdContext = namespace._contexts.get(triggerId)
+        if (triggerIdContext) {
+          namespace._contexts.set(asyncId, triggerIdContext)
+          if (DEBUG_CLS_HOOKED) {
+            const indentStr = " ".repeat(
+              namespace._indent < 0 ? 0 : namespace._indent
+            )
+            debug2(
+              `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+                namespace.active,
+                { showHidden: true, depth: 2, colors: true }
+              )} resource:${resource}`
+            )
+          }
+        } else if (DEBUG_CLS_HOOKED) {
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} resource:${resource}`
+          )
+        }
+      }
+
+      if (DEBUG_CLS_HOOKED && type === "PROMISE") {
+        debug2(util.inspect(resource, { showHidden: true }))
+        const parentId = resource.parentId
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} resource:${resource}`
+        )
+      }
+    },
+    before(asyncId) {
+      currentUid = async_hooks.executionAsyncId()
+      let context
+
+      /*
+      if(currentUid === 0){
+        // CurrentId will be 0 when triggered from C++. Promise events
+        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
+        //const triggerId = async_hooks.triggerAsyncId();
+        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
+      }else{
+        context = namespace._contexts.get(currentUid);
+      }
+      */
+
+      //HACK to work with promises until they are fixed in node > 8.1.1
+      context =
+        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
+
+      if (context) {
+        if (DEBUG_CLS_HOOKED) {
+          const triggerId = async_hooks.triggerAsyncId()
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} context:${util.inspect(context)}`
+          )
+          namespace._indent += 2
+        }
+
+        namespace.enter(context)
+      } else if (DEBUG_CLS_HOOKED) {
+        const triggerId = async_hooks.triggerAsyncId()
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} namespace._contexts:${util.inspect(namespace._contexts, {
+            showHidden: true,
+            depth: 2,
+            colors: true,
+          })}`
+        )
+        namespace._indent += 2
+      }
+    },
+    after(asyncId) {
+      currentUid = async_hooks.executionAsyncId()
+      let context // = namespace._contexts.get(currentUid);
+      /*
+      if(currentUid === 0){
+        // CurrentId will be 0 when triggered from C++. Promise events
+        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
+        //const triggerId = async_hooks.triggerAsyncId();
+        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
+      }else{
+        context = namespace._contexts.get(currentUid);
+      }
+      */
+      //HACK to work with promises until they are fixed in node > 8.1.1
+      context =
+        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
+
+      if (context) {
+        if (DEBUG_CLS_HOOKED) {
+          const triggerId = async_hooks.triggerAsyncId()
+          namespace._indent -= 2
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} context:${util.inspect(context)}`
+          )
+        }
+
+        namespace.exit(context)
+      } else if (DEBUG_CLS_HOOKED) {
+        const triggerId = async_hooks.triggerAsyncId()
+        namespace._indent -= 2
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} context:${util.inspect(context)}`
+        )
+      }
+    },
+    destroy(asyncId) {
+      currentUid = async_hooks.executionAsyncId()
+      if (DEBUG_CLS_HOOKED) {
+        const triggerId = async_hooks.triggerAsyncId()
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} context:${util.inspect(namespace._contexts.get(currentUid))}`
+        )
+      }
+
+      namespace._contexts.delete(asyncId)
+    },
+  })
+
+  hook.enable()
+  namespace._hook = hook
+
+  process.namespaces[name] = namespace
+  return namespace
+}
+
+function destroyNamespace(name) {
+  let namespace = getNamespace(name)
+
+  assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"')
+  assert.ok(
+    namespace.id,
+    "don't assign to process.namespaces directly! " + util.inspect(namespace)
+  )
+
+  namespace._hook.disable()
+  namespace._contexts = null
+  process.namespaces[name] = null
+}
+
+function reset() {
+  // must unregister async listeners
+  if (process.namespaces) {
+    Object.keys(process.namespaces).forEach(function (name) {
+      destroyNamespace(name)
+    })
+  }
+  process.namespaces = Object.create(null)
+}
+
+process.namespaces = process.namespaces || {}
+
+//const fs = require('fs');
+function debug2(...args) {
+  if (DEBUG_CLS_HOOKED) {
+    //fs.writeSync(1, `${util.format(...args)}\n`);
+    process._rawDebug(`${util.format(...args)}`)
+  }
+}
+
+/*function getFunctionName(fn) {
+  if (!fn) {
+    return fn;
+  }
+  if (typeof fn === 'function') {
+    if (fn.name) {
+      return fn.name;
+    }
+    return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1];
+  } else if (fn.constructor && fn.constructor.name) {
+    return fn.constructor.name;
+  }
+}*/

packages/backend-core/src/context/FunctionContext.js

@@ -1,84 +1,47 @@
-const cls = require("cls-hooked")
+const cls = require("../clshooked")
 const { newid } = require("../hashing")
 
 const REQUEST_ID_KEY = "requestId"
+const MAIN_CTX = cls.createNamespace("main")
+
+function getContextStorage(namespace) {
+  if (namespace && namespace.active) {
+    let contextData = namespace.active
+    delete contextData.id
+    delete contextData._ns_name
+    return contextData
+  }
+  return {}
+}
 
 class FunctionContext {
-  static getMiddleware(
+  static run(callback) {
-    updateCtxFn = null,
+    return MAIN_CTX.runAndReturn(async () => {
-    destroyFn = null,
+      const namespaceId = newid()
-    contextName = "session"
+      MAIN_CTX.set(REQUEST_ID_KEY, namespaceId)
-  ) {
+      const namespace = cls.createNamespace(namespaceId)
-    const namespace = this.createNamespace(contextName)
+      let response = await namespace.runAndReturn(callback)
-
+      cls.destroyNamespace(namespaceId)
-    return async function (ctx, next) {
+      return response
-      await new Promise(
+    })
-        namespace.bind(function (resolve, reject) {
-          // store a contextual request ID that can be used anywhere (audit logs)
-          namespace.set(REQUEST_ID_KEY, newid())
-          namespace.bindEmitter(ctx.req)
-          namespace.bindEmitter(ctx.res)
-
-          if (updateCtxFn) {
-            updateCtxFn(ctx)
-          }
-          next()
-            .then(resolve)
-            .catch(reject)
-            .finally(() => {
-              if (destroyFn) {
-                return destroyFn(ctx)
-              }
-            })
-        })
-      )
-    }
   }
 
-  static run(callback, contextName = "session") {
+  static setOnContext(key, value) {
-    const namespace = this.createNamespace(contextName)
+    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
-
+    const namespace = cls.getNamespace(namespaceId)
-    return namespace.runAndReturn(callback)
-  }
-
-  static setOnContext(key, value, contextName = "session") {
-    const namespace = this.createNamespace(contextName)
     namespace.set(key, value)
   }
 
-  static getContextStorage() {
-    if (this._namespace && this._namespace.active) {
-      let contextData = this._namespace.active
-      delete contextData.id
-      delete contextData._ns_name
-      return contextData
-    }
-
-    return {}
-  }
-
   static getFromContext(key) {
-    const context = this.getContextStorage()
+    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
+    const namespace = cls.getNamespace(namespaceId)
+    const context = getContextStorage(namespace)
     if (context) {
       return context[key]
     } else {
       return null
     }
   }
-
-  static destroyNamespace(name = "session") {
-    if (this._namespace) {
-      cls.destroyNamespace(name)
-      this._namespace = null
-    }
-  }
-
-  static createNamespace(name = "session") {
-    if (!this._namespace) {
-      this._namespace = cls.createNamespace(name)
-    }
-    return this._namespace
-  }
 }
 
 module.exports = FunctionContext

packages/backend-core/src/context/index.js

@@ -55,6 +55,15 @@ async function closeAppDBs() {
   }
 }
 
+exports.closeTenancy = async () => {
+  if (env.USE_COUCH) {
+    await closeDB(exports.getGlobalDB())
+  }
+  // clear from context now that database is closed/task is finished
+  cls.setOnContext(ContextKeys.TENANT_ID, null)
+  cls.setOnContext(ContextKeys.GLOBAL_DB, null)
+}
+
 exports.isDefaultTenant = () => {
   return exports.getTenantId() === exports.DEFAULT_TENANT_ID
 }
@@ -82,12 +91,7 @@ exports.doInTenant = (tenantId, task) => {
     } finally {
       const using = cls.getFromContext(ContextKeys.IN_USE)
       if (!using || using <= 1) {
-        if (env.USE_COUCH) {
+        await exports.closeTenancy()
-          await closeDB(exports.getGlobalDB())
-        }
-        // clear from context now that database is closed/task is finished
-        cls.setOnContext(ContextKeys.TENANT_ID, null)
-        cls.setOnContext(ContextKeys.GLOBAL_DB, null)
       } else {
         cls.setOnContext(using - 1)
       }

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,7 +1,7 @@
 const google = require("../google")
 const { Cookies, Configs } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
-const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
+const { getScopedConfig } = require("../../../db/utils")
 const { doWithDB } = require("../../../db")
 const environment = require("../../../environment")
 const { getGlobalDB } = require("../../../tenancy")
@@ -21,20 +21,28 @@ async function fetchGoogleCreds() {
   )
 }
 
-async function platformUrl() {
+async function getPlatformUrl() {
+  let platformUrl = environment.PLATFORM_URL || "http://localhost:10000"
+
   const db = getGlobalDB()
-  const publicConfig = await getScopedConfig(db, {
+  const settings = await getScopedConfig(db, {
     type: Configs.SETTINGS,
   })
-  return getPlatformUrl(publicConfig)
+
+  // self hosted - check for platform url override
+  if (settings && settings.platformUrl) {
+    platformUrl = settings.platformUrl
+  }
+
+  return platformUrl
 }
 
 async function preAuth(passport, ctx, next) {
   // get the relevant config
   const googleConfig = await fetchGoogleCreds()
-  const platUrl = await platformUrl()
+  const platformUrl = await getPlatformUrl()
 
-  let callbackUrl = `${platUrl}/api/global/auth/datasource/google/callback`
+  let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
   const strategy = await google.strategyFactory(googleConfig, callbackUrl)
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
@@ -51,9 +59,9 @@ async function preAuth(passport, ctx, next) {
 async function postAuth(passport, ctx, next) {
   // get the relevant config
   const config = await fetchGoogleCreds()
-  const platUrl = await platformUrl()
+  const platformUrl = await getPlatformUrl()
 
-  let callbackUrl = `${platUrl}/api/global/auth/datasource/google/callback`
+  let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
   const strategy = await google.strategyFactory(
     config,
     callbackUrl,

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -71,7 +71,7 @@ describe("oidc", () => {
   
   describe("authenticate", () => {    
     afterEach(() => {
-      jest.clearAllMocks();
+      jest.clearAllMocks()
     });
 
     // mock third party common authentication
@@ -80,10 +80,10 @@ describe("oidc", () => {
     
     // mock the passport callback
     const mockDone = jest.fn()
+    const mockSaveUserFn = jest.fn()
 
     async function doAuthenticate() {
       const oidc = require("../oidc")
-      const mockSaveUserFn = jest.fn()
       const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
 
       await authenticate(
@@ -105,11 +105,13 @@ describe("oidc", () => {
       expect(authenticateThirdParty).toHaveBeenCalledWith(
         user,
         false, 
-        mockDone)
+        mockDone,
+        mockSaveUserFn,
+      )
     }
 
     it("delegates authentication to third party common", async () => {
-      doTest()
+      await doTest()
     })
 
     it("uses JWT email to get email", async () => {
@@ -118,7 +120,7 @@ describe("oidc", () => {
         email : "mock@budibase.com"
       }
 
-      doTest()
+      await doTest()
     })
   
     it("uses JWT username to get email", async () => {
@@ -127,7 +129,7 @@ describe("oidc", () => {
         preferred_username : "mock@budibase.com"
       }
 
-      doTest()
+      await doTest()
     })
 
     it("uses JWT invalid username to get email", async () => {

packages/backend-core/src/middleware/tenancy.js

@@ -1,6 +1,5 @@
-const { setTenantId, setGlobalDB, getGlobalDB } = require("../tenancy")
+const { setTenantId, setGlobalDB, closeTenancy } = require("../tenancy")
-const { closeDB } = require("../db")
+const cls = require("../context/FunctionContext")
-const ContextFactory = require("../context/FunctionContext")
 const { buildMatcherRegex, matches } = require("./matchers")
 
 module.exports = (
@@ -11,17 +10,16 @@ module.exports = (
   const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
   const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
 
-  const updateCtxFn = ctx => {
+  return async function (ctx, next) {
-    const allowNoTenant =
+    return cls.run(async () => {
-      opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
+      const allowNoTenant =
-    const allowQs = !!matches(ctx, allowQsOptions)
+        opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
-    const tenantId = setTenantId(ctx, { allowQs, allowNoTenant })
+      const allowQs = !!matches(ctx, allowQsOptions)
-    setGlobalDB(tenantId)
+      const tenantId = setTenantId(ctx, { allowQs, allowNoTenant })
+      setGlobalDB(tenantId)
+      const res = await next()
+      await closeTenancy()
+      return res
+    })
   }
-  const destroyFn = async () => {
-    const db = getGlobalDB()
-    await closeDB(db)
-  }
-
-  return ContextFactory.getMiddleware(updateCtxFn, destroyFn)
 }

packages/backend-core/src/security/permissions.js

@@ -96,6 +96,7 @@ const BUILTIN_PERMISSIONS = {
       new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE),
       new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
       new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
+      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
     ],
   },
   POWER: {

packages/backend-core/yarn.lock

@@ -805,13 +805,6 @@ ast-types@0.9.6:
   resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9"
   integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk=
 
-async-hook-jl@^1.7.6:
-  version "1.7.6"
-  resolved "https://registry.yarnpkg.com/async-hook-jl/-/async-hook-jl-1.7.6.tgz#4fd25c2f864dbaf279c610d73bf97b1b28595e68"
-  integrity sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==
-  dependencies:
-    stack-chain "^1.3.7"
-
 async@~2.1.4:
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc"
@@ -1205,15 +1198,6 @@ clone-buffer@1.0.0:
   resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58"
   integrity sha1-4+JbIHrE5wGvch4staFnksrD3Fg=
 
-cls-hooked@^4.2.2:
-  version "4.2.2"
-  resolved "https://registry.yarnpkg.com/cls-hooked/-/cls-hooked-4.2.2.tgz#ad2e9a4092680cdaffeb2d3551da0e225eae1908"
-  integrity sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==
-  dependencies:
-    async-hook-jl "^1.7.6"
-    emitter-listener "^1.0.1"
-    semver "^5.4.1"
-
 cluster-key-slot@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
@@ -1533,7 +1517,7 @@ electron-to-chromium@^1.3.896:
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.900.tgz#5be2c5818a2a012c511b4b43e87b6ab7a296d4f5"
   integrity sha512-SuXbQD8D4EjsaBaJJxySHbC+zq8JrFfxtb4GIr4E9n1BcROyMcRrJCYQNpJ9N+Wjf5mFp7Wp0OHykd14JNEzzQ==
 
-emitter-listener@^1.0.1:
+emitter-listener@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8"
   integrity sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==
@@ -4466,7 +4450,7 @@ saxes@^5.0.1:
   dependencies:
     xmlchars "^2.2.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.6.0:
+"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
@@ -4706,11 +4690,6 @@ sshpk@^1.7.0:
     jsbn "~0.1.0"
     tweetnacl "~0.14.0"
 
-stack-chain@^1.3.7:
-  version "1.3.7"
-  resolved "https://registry.yarnpkg.com/stack-chain/-/stack-chain-1.3.7.tgz#d192c9ff4ea6a22c94c4dd459171e3f00cea1285"
-  integrity sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=
-
 stack-utils@^2.0.2:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.5.tgz#d25265fca995154659dbbfba3b49254778d2fdd5"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.148-alpha.1",
+    "@budibase/string-templates": "^1.0.164-alpha.0",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/bbui/src/Form/Core/CheckboxGroup.svelte

@@ -0,0 +1,68 @@
+<script>
+  import "@spectrum-css/fieldgroup/dist/index-vars.css"
+  import "@spectrum-css/radio/dist/index-vars.css"
+  import { createEventDispatcher } from "svelte"
+
+  export let direction = "vertical"
+  export let value = []
+  export let options = []
+  export let error = null
+  export let disabled = false
+  export let getOptionLabel = option => option
+  export let getOptionValue = option => option
+
+  const dispatch = createEventDispatcher()
+  const onChange = e => {
+    let tempValue = value
+    let isChecked = e.target.checked
+    if (!tempValue.includes(e.target.value) && isChecked) {
+      tempValue.push(e.target.value)
+    }
+    value = tempValue
+    dispatch(
+      "change",
+      tempValue.filter(val => val !== e.target.value || isChecked)
+    )
+  }
+</script>
+
+<div class={`spectrum-FieldGroup spectrum-FieldGroup--${direction}`}>
+  {#if options && Array.isArray(options)}
+    {#each options as option}
+      <div
+        title={getOptionLabel(option)}
+        class="spectrum-Checkbox spectrum-FieldGroup-item"
+        class:is-invalid={!!error}
+      >
+        <label
+          class="spectrum-Checkbox spectrum-Checkbox--sizeM spectrum-FieldGroup-item"
+        >
+          <input
+            on:change={onChange}
+            value={getOptionValue(option)}
+            type="checkbox"
+            class="spectrum-Checkbox-input"
+            {disabled}
+            checked={value.includes(getOptionValue(option))}
+          />
+          <span class="spectrum-Checkbox-box">
+            <svg
+              class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Checkbox-checkmark"
+              focusable="false"
+              aria-hidden="true"
+            >
+              <use xlink:href="#spectrum-css-icon-Checkmark100" />
+            </svg>
+          </span>
+          <span class="spectrum-Checkbox-label">{getOptionLabel(option)}</span>
+        </label>
+      </div>
+    {/each}
+  {/if}
+</div>
+
+<style>
+  .spectrum-Checkbox-input {
+    opacity: 0;
+  }
+</style>

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -58,6 +58,11 @@
     if (timeOnly) {
       newValue = `2000-01-01T${newValue.split("T")[1]}`
     }
+    // date only, offset for timezone so always right date
+    else if (!enableTime) {
+      const offset = dates[0].getTimezoneOffset() * 60000
+      newValue = new Date(dates[0].getTime() - offset).toISOString()
+    }
     dispatch("change", newValue)
   }
 
@@ -156,8 +161,8 @@
         <input
           data-input
           type="text"
-          {disabled}
           class="spectrum-Textfield-input spectrum-InputGroup-input"
+          class:is-disabled={disabled}
           {placeholder}
           {id}
           {value}
@@ -167,7 +172,7 @@
         type="button"
         class="spectrum-Picker spectrum-Picker--sizeM spectrum-InputGroup-button"
         tabindex="-1"
-        {disabled}
+        class:is-disabled={disabled}
         class:is-invalid={!!error}
         on:click={flatpickr?.open}
       >
@@ -212,4 +217,7 @@
   :global(.flatpickr-calendar) {
     font-family: "Source Sans Pro", sans-serif;
   }
+  .is-disabled {
+    pointer-events: none !important;
+  }
 </style>

packages/bbui/src/Form/Core/Picker.svelte

@@ -47,7 +47,7 @@
       return
     }
     searchTerm = null
-    open = true
+    open = !open
   }
 
   const getSortedOptions = (options, getLabel, sort) => {
@@ -75,104 +75,77 @@
   }
 </script>
 
-<button
+<div use:clickOutside={() => (open = false)}>
-  {id}
+  <button
-  class="spectrum-Picker spectrum-Picker--sizeM"
+    {id}
-  class:spectrum-Picker--quiet={quiet}
+    class="spectrum-Picker spectrum-Picker--sizeM"
-  {disabled}
+    class:spectrum-Picker--quiet={quiet}
-  class:is-invalid={!!error}
+    {disabled}
-  class:is-open={open}
+    class:is-invalid={!!error}
-  aria-haspopup="listbox"
+    class:is-open={open}
-  on:mousedown={onClick}
+    aria-haspopup="listbox"
->
+    on:mousedown={onClick}
-  {#if fieldIcon}
-    <span class="option-icon">
-      <Icon name={fieldIcon} />
-    </span>
-  {/if}
-  <span
-    class="spectrum-Picker-label"
-    class:is-placeholder={isPlaceholder}
-    class:auto-width={autoWidth}
   >
-    {fieldText}
+    {#if fieldIcon}
-  </span>
+      <span class="option-icon">
-  {#if error}
+        <Icon name={fieldIcon} />
+      </span>
+    {/if}
+    <span
+      class="spectrum-Picker-label"
+      class:is-placeholder={isPlaceholder}
+      class:auto-width={autoWidth}
+    >
+      {fieldText}
+    </span>
+    {#if error}
+      <svg
+        class="spectrum-Icon spectrum-Icon--sizeM spectrum-Picker-validationIcon"
+        focusable="false"
+        aria-hidden="true"
+        aria-label="Folder"
+      >
+        <use xlink:href="#spectrum-icon-18-Alert" />
+      </svg>
+    {/if}
+    {#if fieldColour}
+      <span class="option-colour">
+        <StatusLight size="L" color={fieldColour} />
+      </span>
+    {/if}
     <svg
-      class="spectrum-Icon spectrum-Icon--sizeM spectrum-Picker-validationIcon"
+      class="spectrum-Icon spectrum-UIIcon-ChevronDown100 spectrum-Picker-menuIcon"
       focusable="false"
       aria-hidden="true"
-      aria-label="Folder"
     >
-      <use xlink:href="#spectrum-icon-18-Alert" />
+      <use xlink:href="#spectrum-css-icon-Chevron100" />
     </svg>
-  {/if}
+  </button>
-  {#if fieldColour}
+  {#if open}
-    <span class="option-colour">
+    <div
-      <StatusLight size="L" color={fieldColour} />
+      transition:fly|local={{ y: -20, duration: 200 }}
-    </span>
+      class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
-  {/if}
+      class:auto-width={autoWidth}
-  <svg
+    >
-    class="spectrum-Icon spectrum-UIIcon-ChevronDown100 spectrum-Picker-menuIcon"
+      {#if autocomplete}
-    focusable="false"
+        <Search
-    aria-hidden="true"
+          value={searchTerm}
-  >
+          on:change={event => (searchTerm = event.detail)}
-    <use xlink:href="#spectrum-css-icon-Chevron100" />
+          {disabled}
-  </svg>
+          placeholder="Search"
-</button>
+        />
-{#if open}
-  <div
-    use:clickOutside={() => (open = false)}
-    transition:fly|local={{ y: -20, duration: 200 }}
-    class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
-    class:auto-width={autoWidth}
-  >
-    {#if autocomplete}
-      <Search
-        value={searchTerm}
-        on:change={event => (searchTerm = event.detail)}
-        {disabled}
-        placeholder="Search"
-      />
-    {/if}
-    <ul class="spectrum-Menu" role="listbox">
-      {#if placeholderOption}
-        <li
-          class="spectrum-Menu-item placeholder"
-          class:is-selected={isPlaceholder}
-          role="option"
-          aria-selected="true"
-          tabindex="0"
-          on:click={() => onSelectOption(null)}
-        >
-          <span class="spectrum-Menu-itemLabel">{placeholderOption}</span>
-          <svg
-            class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
-            focusable="false"
-            aria-hidden="true"
-          >
-            <use xlink:href="#spectrum-css-icon-Checkmark100" />
-          </svg>
-        </li>
       {/if}
-      {#if filteredOptions.length}
+      <ul class="spectrum-Menu" role="listbox">
-        {#each filteredOptions as option, idx}
+        {#if placeholderOption}
           <li
-            class="spectrum-Menu-item"
+            class="spectrum-Menu-item placeholder"
-            class:is-selected={isOptionSelected(getOptionValue(option, idx))}
+            class:is-selected={isPlaceholder}
             role="option"
             aria-selected="true"
             tabindex="0"
-            on:click={() => onSelectOption(getOptionValue(option, idx))}
+            on:click={() => onSelectOption(null)}
           >
-            {#if getOptionIcon(option, idx)}
+            <span class="spectrum-Menu-itemLabel">{placeholderOption}</span>
-              <span class="option-icon">
-                <Icon name={getOptionIcon(option, idx)} />
-              </span>
-            {/if}
-            <span class="spectrum-Menu-itemLabel">
-              {getOptionLabel(option, idx)}
-            </span>
             <svg
               class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
               focusable="false"
@@ -180,17 +153,45 @@
             >
               <use xlink:href="#spectrum-css-icon-Checkmark100" />
             </svg>
-            {#if getOptionColour(option, idx)}
-              <span class="option-colour">
-                <StatusLight size="L" color={getOptionColour(option, idx)} />
-              </span>
-            {/if}
           </li>
-        {/each}
+        {/if}
-      {/if}
+        {#if filteredOptions.length}
-    </ul>
+          {#each filteredOptions as option, idx}
-  </div>
+            <li
-{/if}
+              class="spectrum-Menu-item"
+              class:is-selected={isOptionSelected(getOptionValue(option, idx))}
+              role="option"
+              aria-selected="true"
+              tabindex="0"
+              on:click={() => onSelectOption(getOptionValue(option, idx))}
+            >
+              {#if getOptionIcon(option, idx)}
+                <span class="option-icon">
+                  <Icon name={getOptionIcon(option, idx)} />
+                </span>
+              {/if}
+              <span class="spectrum-Menu-itemLabel">
+                {getOptionLabel(option, idx)}
+              </span>
+              <svg
+                class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+                focusable="false"
+                aria-hidden="true"
+              >
+                <use xlink:href="#spectrum-css-icon-Checkmark100" />
+              </svg>
+              {#if getOptionColour(option, idx)}
+                <span class="option-colour">
+                  <StatusLight size="L" color={getOptionColour(option, idx)} />
+                </span>
+              {/if}
+            </li>
+          {/each}
+        {/if}
+      </ul>
+    </div>
+  {/if}
+</div>
 
 <style>
   .spectrum-Popover {

packages/bbui/src/Form/Core/TextField.svelte

@@ -1,6 +1,6 @@
 <script>
   import "@spectrum-css/textfield/dist/index-vars.css"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
 
   export let value = null
   export let placeholder = null
@@ -13,8 +13,11 @@
   export let quiet = false
   export let dataCy
   export let align
+  export let autofocus = false
 
   const dispatch = createEventDispatcher()
+
+  let field
   let focus = false
 
   const updateValue = newValue => {
@@ -58,6 +61,11 @@
       updateValue(event.target.value)
     }
   }
+
+  onMount(() => {
+    focus = autofocus
+    if (focus) field.focus()
+  })
 </script>
 
 <div
@@ -77,6 +85,7 @@
     </svg>
   {/if}
   <input
+    bind:this={field}
     {disabled}
     {readonly}
     {id}

packages/bbui/src/Form/Core/index.js

@@ -3,6 +3,7 @@ export { default as CoreSelect } from "./Select.svelte"
 export { default as CoreMultiselect } from "./Multiselect.svelte"
 export { default as CoreCheckbox } from "./Checkbox.svelte"
 export { default as CoreRadioGroup } from "./RadioGroup.svelte"
+export { default as CoreCheckboxGroup } from "./CheckboxGroup.svelte"
 export { default as CoreTextArea } from "./TextArea.svelte"
 export { default as CoreCombobox } from "./Combobox.svelte"
 export { default as CoreSwitch } from "./Switch.svelte"

packages/bbui/src/Form/Input.svelte

@@ -14,6 +14,7 @@
   export let updateOnChange = true
   export let quiet = false
   export let dataCy
+  export let autofocus
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -33,6 +34,7 @@
     {placeholder}
     {type}
     {quiet}
+    {autofocus}
     on:change={onChange}
     on:click
     on:input

packages/builder/cypress.json

@@ -1,11 +1,15 @@
 {
   "baseUrl": "http://localhost:4100",
-  "video": false,
+  "video": true,
   "projectId": "bmbemn",
   "env": {
     "PORT": "4100",
     "WORKER_PORT": "4200",
     "JWT_SECRET": "test",
     "HOST_IP": ""
+  },
+  "retries": {
+    "runMode": 2,
+    "openMode": 0
   }
 }

packages/builder/cypress/integration/createApp.spec.js

@@ -1,4 +1,5 @@
 import filterTests from '../support/filterTests'
+const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
   context("Create an Application", () => {
@@ -10,14 +11,14 @@ filterTests(['smoke', 'all'], () => {
 
     if (!(Cypress.env("TEST_ENV"))) {
       it("should show the new user UI/UX", () => {
-        cy.visit(`${Cypress.config().baseUrl}/builder`)
+        cy.visit(`${Cypress.config().baseUrl}/builder/portal/apps/create`) //added /portal/apps/create
-        cy.get(`[data-cy="create-app-btn"]`).contains('Start from scratch').should("exist")
+        cy.get(interact.CREATE_APP_BUTTON).contains('Start from scratch').should("exist")
-        cy.get(`[data-cy="import-app-btn"]`).should("exist")
+        cy.get(interact.CREATE_APP_BUTTON).should("exist")
         
-        cy.get(".template-category-filters").should("exist")
+        cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist")
-        cy.get(".template-categories").should("exist")
+        cy.get(interact.TEMPLATE_CATEGORY).should("exist")
         
-        cy.get(".appTable").should("not.exist")
+        cy.get(interact.APP_TABLE).should("not.exist")
       })
     }
 
@@ -29,21 +30,21 @@ filterTests(['smoke', 'all'], () => {
         .its("body")
         .then(val => {
           if (val.length > 0) {
-            cy.get(".spectrum-Button").contains("Templates").click({force: true})
+            cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Templates").click({force: true})
           }
         })
 
-      cy.get(".template-category-filters").should("exist")
+      cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist")
-      cy.get(".template-categories").should("exist")
+      cy.get(interact.TEMPLATE_CATEGORY).should("exist")
       
-      cy.get(".template-category").its('length').should('be.gt', 1)
+      cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).its('length').should('be.gt', 1)
-      cy.get(".template-category-filters .spectrum-ActionButton").its('length').should('be.gt', 2)
+      cy.get(interact.TEMPLATE_CATEGORY_FILTER_ACTIONBUTTON).its('length').should('be.gt', 2)
       
-      cy.get(".template-category-filters .spectrum-ActionButton").eq(1).click()
+      cy.get(interact.TEMPLATE_CATEGORY_FILTER_ACTIONBUTTON).eq(1).click()
-      cy.get(".template-category").should('have.length', 1)
+      cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).should('have.length', 1)
 
-      cy.get(".template-category-filters .spectrum-ActionButton").eq(0).click()
+      cy.get(interact.TEMPLATE_CATEGORY_FILTER_ACTIONBUTTON).eq(0).click()
-      cy.get(".template-category").its('length').should('be.gt', 1)
+      cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).its('length').should('be.gt', 1)
     })
 
     it("should enforce a valid url before submission", () => {
@@ -51,37 +52,40 @@ filterTests(['smoke', 'all'], () => {
       cy.wait(500)
 
       // Start create app process. If apps already exist, click second button
-      cy.get(`[data-cy="create-app-btn"]`).click({ force: true })
+      cy.get(interact.CREATE_APP_BUTTON).click({ force: true })
       cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`)
         .its("body")
         .then(val => {
           if (val.length > 0) {
-            cy.get(`[data-cy="create-app-btn"]`).click({ force: true })
+            cy.get(interact.CREATE_APP_BUTTON).click({ force: true })
           }
         })
 
       const appName = "Cypress Tests"
-      cy.get(".spectrum-Modal").within(() => {
+      cy.get(interact.SPECTRUM_MODAL).within(() => {
+
+        cy.get(interact.APP_NAME_INPUT).eq(0).should('have.focus')
 
         //Auto fill
-        cy.get("input").eq(0).type(appName).should("have.value", appName).blur()
+        cy.get(interact.APP_NAME_INPUT).eq(0).clear()
-        cy.get("input").eq(1).should("have.value", "/cypress-tests")
+        cy.get(interact.APP_NAME_INPUT).eq(0).type(appName).should("have.value", appName).blur()
-        cy.get(".spectrum-ButtonGroup").contains("Create app").should('not.be.disabled')
+        cy.get(interact.APP_NAME_INPUT).eq(1).should("have.value", "/cypress-tests")
+        cy.get(interact.SPECTRUM_BUTTON_GROUP).contains("Create app").should('not.be.disabled')
 
         //Empty the app url - disabled create
-        cy.get("input").eq(1).clear().blur()
+        cy.get(interact.APP_NAME_INPUT).eq(1).clear().blur()
-        cy.get(".spectrum-ButtonGroup").contains("Create app").should('be.disabled')
+        cy.get(interact.SPECTRUM_BUTTON_GROUP).contains("Create app").should('be.disabled')
 
         //Invalid url
-        cy.get("input").eq(1).type("/new app-url").blur()
+        cy.get(interact.APP_NAME_INPUT).eq(1).type("/new app-url").blur()
-        cy.get(".spectrum-ButtonGroup").contains("Create app").should('be.disabled')
+        cy.get(interact.SPECTRUM_BUTTON_GROUP).contains("Create app").should('be.disabled')
 
         //Specifically alter the url
-        cy.get("input").eq(1).clear()
+        cy.get(interact.APP_NAME_INPUT).eq(1).clear()
-        cy.get("input").eq(1).type("another-app-name").blur()
+        cy.get(interact.APP_NAME_INPUT).eq(1).type("another-app-name").blur()
-        cy.get("input").eq(1).should("have.value", "/another-app-name")
+        cy.get(interact.APP_NAME_INPUT).eq(1).should("have.value", "/another-app-name")
-        cy.get("input").eq(0).should("have.value", appName)
+        cy.get(interact.APP_NAME_INPUT).eq(0).should("have.value", appName)
-        cy.get(".spectrum-ButtonGroup").contains("Create app").should('not.be.disabled')
+        cy.get(interact.SPECTRUM_BUTTON_GROUP).contains("Create app").should('not.be.disabled')
 
       })
     })
@@ -97,6 +101,75 @@ filterTests(['smoke', 'all'], () => {
       cy.deleteApp(appName)
     })
 
+    it("should create the first application from scratch with a default name", () => {
+      cy.createApp()
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.wait(1000)
+
+      cy.applicationInAppTable("My app")
+      cy.deleteApp("My app")
+    })
+
+    it("should create the first application from scratch, using the users first name as the default app name", () => {
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+
+      cy.updateUserInformation("Ted", "Userman")
+      
+      cy.createApp()
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.wait(1000)
+
+      cy.applicationInAppTable("Teds app")
+      cy.deleteApp("Teds app")
+
+      //Accomodate names that end in 'S'
+      cy.updateUserInformation("Chris", "Userman")
+      
+      cy.createApp()
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.wait(1000)
+
+      cy.applicationInAppTable("Chris app")
+      cy.deleteApp("Chris app")
+
+      cy.updateUserInformation("", "")
+    })
+
+    it("should create an application from an export", () => {
+      const exportedApp = 'cypress/fixtures/exported-app.txt'
+
+      cy.importApp(exportedApp, "")
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+
+      cy.applicationInAppTable("My app")
+      
+      cy.get(".appTable .name").eq(0).click()
+
+      cy.deleteApp("My app")
+    })
+
+    it("should create an application from an export, using the users first name as the default app name", () => {
+      const exportedApp = 'cypress/fixtures/exported-app.txt'
+
+      cy.updateUserInformation("Ted", "Userman")
+
+      cy.importApp(exportedApp, "")
+
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+
+      cy.applicationInAppTable("Teds app")
+      
+      cy.get(".appTable .name").eq(0).click()
+
+      cy.deleteApp("Teds app")
+
+      cy.updateUserInformation("", "")
+    })
+
     it("should generate the first application from a template", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.wait(500)
@@ -106,15 +179,15 @@ filterTests(['smoke', 'all'], () => {
         .its("body")
         .then(val => {
           if (val.length > 0) {
-            cy.get(`[data-cy="create-app-btn"]`).click({ force: true })
+            cy.get(interact.CREATE_APP_BUTTON).click({ force: true })
           }
         })
 
-      cy.get(".template-category-filters").should("exist")
+      cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist")
-      cy.get(".template-categories").should("exist")
+      cy.get(interact.TEMPLATE_CATEGORY).should("exist")
 
       // Select template
-      cy.get('.template-category').eq(0).within(() => {
+      cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).eq(0).within(() => {
         const card = cy.get('.template-card').eq(0).should("exist");
         const cardOverlay = card.get('.template-thumbnail-action-overlay').should("exist")
         cardOverlay.invoke("show")
@@ -128,14 +201,14 @@ filterTests(['smoke', 'all'], () => {
       templateName.invoke('text')
       .then(templateNameText => {
         const templateNameParsed = "/"+templateNameText.toLowerCase().replace(/\s+/g, "-")
-        cy.get(".spectrum-Modal input").eq(0).should("have.value", templateNameText)
+        cy.get(interact.SPECTRUM_MODAL_INPUT).eq(0).should("have.value", templateNameText)
-        cy.get(".spectrum-Modal input").eq(1).should("have.value", templateNameParsed)
+        cy.get(interact.SPECTRUM_MODAL_INPUT).eq(1).should("have.value", templateNameParsed)
 
         cy.get(".spectrum-Modal .spectrum-ButtonGroup").contains("Create app").click()
         cy.wait(5000)
         
         cy.visit(`${Cypress.config().baseUrl}/builder`)
-        cy.wait(1000)
+        cy.wait(2000)
 
         cy.applicationInAppTable(templateNameText)
         cy.deleteApp(templateNameText) 

packages/builder/cypress/integration/createBinding.spec.js

@@ -24,7 +24,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("should add a URL param binding", () => {
+    xit("should add a URL param binding", () => {
       const paramName = "foo"
       cy.createScreen(`/test/:${paramName}`)
       cy.addComponent("Elements", "Paragraph").then(componentId => {

packages/builder/cypress/integration/datasources/postgreSql.spec.js

@@ -11,7 +11,7 @@ filterTests(["all"], () => {
       const queryName = "Cypress Test Query"
       const queryRename = "CT Query Rename"
 
-      it("Should add PostgreSQL data source without configuration", () => {
+      xit("Should add PostgreSQL data source without configuration", () => {
         // Select PostgreSQL data source
         cy.selectExternalDatasource(datasource)
         // Attempt to fetch tables without applying configuration
@@ -107,7 +107,7 @@ filterTests(["all"], () => {
       })
 
       it("should delete a relationship", () => {
-        cy.get(".hierarchy-items-container").contains("PostgreSQL-2").click()
+        cy.get(".hierarchy-items-container").contains("PostgreSQL").click()
         cy.reload()
         // Delete one relationship
         cy.get(".spectrum-Table")
@@ -155,7 +155,7 @@ filterTests(["all"], () => {
 
       it("should switch to schema with no tables", () => {
         // Switch Schema - To one without any tables
-        cy.get(".hierarchy-items-container").contains("PostgreSQL-2").click()
+        cy.get(".hierarchy-items-container").contains("PostgreSQL").click()
         switchSchema("randomText")
 
         // No tables displayed

packages/builder/cypress/support/commands.js

@@ -18,14 +18,14 @@ Cypress.Commands.add("login", () => {
       cy.get("input").first().type("test@test.com")
       cy.get('input[type="password"]').first().type("test")
       cy.get('input[type="password"]').eq(1).type("test")
-      cy.contains("Create super admin user").click()
+      cy.contains("Create super admin user").click({ force: true })
     }
     if (url.includes("builder/auth/login") || url.includes("builder/admin")) {
       // login
       cy.contains("Sign in to Budibase").then(() => {
         cy.get("input").first().type("test@test.com")
         cy.get('input[type="password"]').type("test")
-        cy.get("button").first().click()
+        cy.get("button").first().click({ force: true })
         cy.wait(1000)
       })
     }
@@ -39,6 +39,71 @@ Cypress.Commands.add("closeModal", () => {
   })
 })
 
+Cypress.Commands.add("importApp", (exportFilePath, name) => {
+  cy.visit(`${Cypress.config().baseUrl}/builder`)
+
+  cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`)
+    .its("body")
+    .then(val => {
+      if (val.length > 0) {
+        cy.get(`[data-cy="create-app-btn"]`).click({ force: true })
+        cy.wait(500)
+      }
+      cy.get(`[data-cy="import-app-btn"]`).click({ force: true })
+    })
+
+  cy.get(".spectrum-Modal").within(() => {
+    cy.get("input").eq(1).should("have.focus")
+
+    cy.get(".spectrum-Dropzone").selectFile(exportFilePath, {
+      action: "drag-drop",
+    })
+
+    cy.get(".gallery .filename").contains("exported-app.txt")
+
+    if (name && name != "") {
+      cy.get("input").eq(0).type(name).should("have.value", name).blur()
+    }
+    cy.get(".confirm-wrap button")
+      .should("not.be.disabled")
+      .click({ force: true })
+    cy.wait(5000)
+  })
+})
+
+Cypress.Commands.add("updateUserInformation", (firstName, lastName) => {
+  cy.get(".user-dropdown .avatar > .icon").click({ force: true })
+
+  cy.get(".spectrum-Popover[data-cy='user-menu']").within(() => {
+    cy.get("li[data-cy='user-info']").click({ force: true })
+  })
+
+  cy.get(".spectrum-Modal.is-open").within(() => {
+    cy.get("[data-cy='user-first-name']").clear()
+
+    if (!firstName || firstName == "") {
+      cy.get("[data-cy='user-first-name']").invoke("val").should("be.empty")
+    } else {
+      cy.get("[data-cy='user-first-name']")
+        .type(firstName)
+        .should("have.value", firstName)
+        .blur()
+    }
+
+    cy.get("[data-cy='user-last-name']").clear()
+
+    if (!lastName || lastName == "") {
+      cy.get("[data-cy='user-last-name']").invoke("val").should("be.empty")
+    } else {
+      cy.get("[data-cy='user-last-name']")
+        .type(lastName)
+        .should("have.value", lastName)
+        .blur()
+    }
+    cy.get("button").contains("Update information").click({ force: true })
+  })
+})
+
 Cypress.Commands.add("createApp", (name, addDefaultTable) => {
   const shouldCreateDefaultTable =
     typeof addDefaultTable != "boolean" ? true : addDefaultTable
@@ -57,8 +122,14 @@ Cypress.Commands.add("createApp", (name, addDefaultTable) => {
     })
 
   cy.get(".spectrum-Modal").within(() => {
-    cy.get("input").eq(0).type(name).should("have.value", name).blur()
+    cy.get("input").eq(0).should("have.focus")
-    cy.get(".spectrum-ButtonGroup").contains("Create app").click()
+    if (name && name != "") {
+      cy.get("input").eq(0).clear()
+      cy.get("input").eq(0).type(name).should("have.value", name).blur()
+    }
+    cy.get(".spectrum-ButtonGroup")
+      .contains("Create app")
+      .click({ force: true })
     cy.wait(10000)
   })
   if (shouldCreateDefaultTable) {
@@ -75,9 +146,6 @@ Cypress.Commands.add("deleteApp", name => {
       const findAppName = val.some(val => val.name == name)
       if (findAppName) {
         if (val.length > 0) {
-          if (Cypress.env("TEST_ENV")) {
-            cy.searchForApplication(name)
-          }
           const appId = val.reduce((acc, app) => {
             if (name === app.name) {
               acc = app.appId
@@ -92,7 +160,7 @@ Cypress.Commands.add("deleteApp", name => {
           const appIdParsed = appId.split("_").pop()
           const actionEleId = `[data-cy=row_actions_${appIdParsed}]`
           cy.get(actionEleId).within(() => {
-            cy.get(".spectrum-Icon").eq(0).click()
+            cy.get(".spectrum-Icon").eq(0).click({ force: true })
           })
           cy.get(".spectrum-Menu").then($menu => {
             if ($menu.text().includes("Unpublish")) {
@@ -102,7 +170,7 @@ Cypress.Commands.add("deleteApp", name => {
           })
 
           cy.get(actionEleId).within(() => {
-            cy.get(".spectrum-Icon").eq(0).click()
+            cy.get(".spectrum-Icon").eq(0).click({ force: true })
           })
           cy.get(".spectrum-Menu").contains("Delete").click()
           cy.get(".spectrum-Dialog-grid").within(() => {
@@ -128,7 +196,7 @@ Cypress.Commands.add("deleteAllApps", () => {
         const appIdParsed = val[i].appId.split("_").pop()
         const actionEleId = `[data-cy=row_actions_${appIdParsed}]`
         cy.get(actionEleId).within(() => {
-          cy.get(".spectrum-Icon").eq(0).click()
+          cy.get(".spectrum-Icon").eq(0).click({ force: true })
         })
 
         cy.get(".spectrum-Menu").contains("Delete").click()
@@ -145,6 +213,7 @@ Cypress.Commands.add("createTestApp", () => {
   const appName = "Cypress Tests"
   cy.deleteApp(appName)
   cy.createApp(appName, "This app is used for Cypress testing.")
+  //cy.createScreen("home")
 })
 
 Cypress.Commands.add("createTestTableWithData", () => {
@@ -245,12 +314,12 @@ Cypress.Commands.add("createUser", email => {
 
 Cypress.Commands.add("addComponent", (category, component) => {
   if (category) {
-    cy.get(`[data-cy="category-${category}"]`).click()
+    cy.get(`[data-cy="category-${category}"]`).click({ force: true })
   }
   if (component) {
-    cy.get(`[data-cy="component-${component}"]`).click()
+    cy.get(`[data-cy="component-${component}"]`).click({ force: true })
   }
-  cy.wait(1000)
+  cy.wait(2000)
   cy.location().then(loc => {
     const params = loc.pathname.split("/")
     const componentId = params[params.length - 1]
@@ -573,7 +642,7 @@ Cypress.Commands.add("addDatasourceConfig", (datasource, skipFetch) => {
       cy.get(".spectrum-Button")
         .contains("Save and fetch tables")
         .click({ force: true })
-      cy.wait(1000)
+      cy.wait(3000)
     })
   }
 })

packages/builder/cypress/support/interact.js

@@ -0,0 +1,13 @@
+// createApp test
+export const CREATE_APP_BUTTON = '[data-cy="create-app-btn"]'
+export const TEMPLATE_CATEGORY_FILTER = ".template-category-filters"
+export const TEMPLATE_CATEGORY = ".template-categories"
+export const APP_TABLE = ".appTable"
+export const SPECTRUM_BUTTON_TEMPLATE = ".spectrum-Button"
+export const TEMPLATE_CATEGORY_ACTIONGROUP = ".template-category"
+export const TEMPLATE_CATEGORY_FILTER_ACTIONBUTTON =
+  ".template-category-filters .spectrum-ActionButton"
+export const SPECTRUM_MODAL = ".spectrum-Modal"
+export const APP_NAME_INPUT = "input" // we need to update this with atribute cy-data
+export const SPECTRUM_BUTTON_GROUP = ".spectrum-ButtonGroup"
+export const SPECTRUM_MODAL_INPUT = ".spectrum-Modal input"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -67,10 +67,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.148-alpha.1",
+    "@budibase/bbui": "^1.0.164-alpha.0",
-    "@budibase/client": "^1.0.148-alpha.1",
+    "@budibase/client": "^1.0.164-alpha.0",
-    "@budibase/frontend-core": "^1.0.148-alpha.1",
+    "@budibase/frontend-core": "^1.0.164-alpha.0",
-    "@budibase/string-templates": "^1.0.148-alpha.1",
+    "@budibase/string-templates": "^1.0.164-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -53,6 +53,7 @@
     : { schema: {} }
   $: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
   $: schemaFields = Object.values(schema || {})
+  $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
 
   const onChange = Utils.sequential(async (e, key) => {
     try {
@@ -330,6 +331,7 @@
               on:change={e => onChange(e, key)}
               {bindings}
               updateOnChange={false}
+              placeholder={value.customType === "queryLimit" ? queryLimit : ""}
             />
           </div>
         {/if}

packages/builder/src/components/automation/Shared/WebhookDisplay.svelte

@@ -12,4 +12,4 @@
   }
 </script>
 
-<CopyInput {value} copyValue={fullWebhookURL(value)} />
+<CopyInput value={fullWebhookURL(value)} />

packages/builder/src/components/backend/DataTable/formula.js

@@ -60,7 +60,7 @@ export function getBindings({
     )
 
     const label = path == null ? column : `${path}.0.${column}`
-    const binding = path == null ? `[${column}]` : `${path}.0.[${column}]`
+    const binding = path == null ? `[${column}]` : `[${path}].0.[${column}]`
     // only supply a description for relationship paths
     const description =
       path == null

packages/builder/src/components/common/inputs/CopyInput.svelte

@@ -3,7 +3,6 @@
 
   export let label = null
   export let value
-  export let copyValue
   export let dataCy = null
 
   const copyToClipboard = val => {
@@ -19,7 +18,7 @@
 
 <div data-cy={dataCy}>
   <Input readonly {value} {label} />
-  <div class="icon" on:click={() => copyToClipboard(value || copyValue)}>
+  <div class="icon" on:click={() => copyToClipboard(value)}>
     <Icon size="S" name="Copy" />
   </div>
 </div>

packages/builder/src/components/settings/UpdateUserInfoModal.svelte

@@ -27,6 +27,14 @@
     Personalise the platform by adding your first name and last name.
   </Body>
   <Input disabled bind:value={$auth.user.email} label="Email" />
-  <Input bind:value={$values.firstName} label="First name" />
+  <Input
-  <Input bind:value={$values.lastName} label="Last name" />
+    bind:value={$values.firstName}
+    label="First name"
+    dataCy="user-first-name"
+  />
+  <Input
+    bind:value={$values.lastName}
+    label="Last name"
+    dataCy="user-last-name"
+  />
 </ModalContent>

packages/builder/src/components/start/AppRow.svelte

@@ -19,6 +19,7 @@
   export let unpublishApp
   export let releaseLock
   export let editIcon
+  export let copyAppId
 </script>
 
 <div class="title">
@@ -102,6 +103,9 @@
       <MenuItem on:click={() => unpublishApp(app)} icon="GlobeRemove">
         Unpublish
       </MenuItem>
+      <MenuItem on:click={() => copyAppId(app)} icon="Copy">
+        Copy App ID
+      </MenuItem>
     {/if}
     {#if !app.deployed}
       <MenuItem on:click={() => updateApp(app)} icon="Edit">Edit</MenuItem>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -16,13 +16,26 @@
   export let template
 
   let creating = false
+  let defaultAppName
 
   const values = writable({ name: "", url: null })
   const validation = createValidationStore()
   $: validation.check($values)
 
   onMount(async () => {
-    $values.name = resolveAppName(template, $values.name)
+    const lastChar = $auth.user?.firstName
+      ? $auth.user?.firstName[$auth.user?.firstName.length - 1]
+      : null
+
+    defaultAppName =
+      lastChar && lastChar.toLowerCase() == "s"
+        ? `${$auth.user?.firstName} app`
+        : `${$auth.user.firstName}s app`
+
+    $values.name = resolveAppName(
+      template,
+      !$auth.user?.firstName ? "My app" : defaultAppName
+    )
     nameToUrl($values.name)
     await setupValidation()
   })
@@ -44,7 +57,7 @@
   }
 
   const resolveAppName = (template, name) => {
-    if (template && !name) {
+    if (template && !template.fromFile) {
       return template.name
     }
     return name ? name.trim() : null
@@ -83,7 +96,7 @@
       }
       data.append("useTemplate", template != null)
       if (template) {
-        data.append("templateName", template.name) //or here?
+        data.append("templateName", template.name)
         data.append("templateKey", template.key)
         data.append("templateFile", $values.file)
       }
@@ -159,15 +172,14 @@
     />
   {/if}
   <Input
+    autofocus={true}
     bind:value={$values.name}
     disabled={creating}
     error={$validation.touched.name && $validation.errors.name}
     on:blur={() => ($validation.touched.name = true)}
     on:change={nameToUrl($values.name)}
     label="Name"
-    placeholder={$auth.user?.firstName
+    placeholder={defaultAppName}
-      ? `${$auth.user.firstName}s app`
-      : "My app"}
   />
   <span>
     <Input

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/DynamicVariableModal.svelte

@@ -1,5 +1,8 @@
 <script>
   import { Input, ModalContent, Modal, Body } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+
+  const dispatch = createEventDispatcher()
 
   export let dynamicVariables
   export let datasource
@@ -35,6 +38,7 @@
     name = null
     binding = null
     dynamicVariables[copiedName] = copiedBinding
+    dispatch("change", dynamicVariables)
   }
 </script>
 

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte

@@ -299,6 +299,7 @@
   {dynamicVariables}
   bind:binding={varBinding}
   bind:this={addVariableModal}
+  on:change={saveQuery}
 />
 {#if query && queryConfig}
   <div class="inner">

packages/builder/src/pages/builder/apps/index.svelte

@@ -69,7 +69,7 @@
         <Layout noPadding>
           <div class="header">
             <img alt="logo" src={$organisation.logoUrl || Logo} />
-            <ActionMenu align="right">
+            <ActionMenu align="right" dataCy="user-menu">
               <div slot="control" class="avatar">
                 <Avatar
                   size="M"

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -160,7 +160,7 @@
           />
         </div>
         <div class="user-dropdown">
-          <ActionMenu align="right">
+          <ActionMenu align="right" dataCy="user-menu">
             <div slot="control" class="avatar">
               <Avatar
                 size="M"
@@ -169,7 +169,11 @@
               />
               <Icon size="XL" name="ChevronDown" />
             </div>
-            <MenuItem icon="UserEdit" on:click={() => userInfoModal.show()}>
+            <MenuItem
+              icon="UserEdit"
+              on:click={() => userInfoModal.show()}
+              dataCy={"user-info"}
+            >
               Update user information
             </MenuItem>
             {#if $auth.isBuilder}

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -12,6 +12,7 @@
     Body,
     Search,
     Divider,
+    Helpers,
   } from "@budibase/bbui"
   import TemplateDisplay from "components/common/TemplateDisplay.svelte"
   import Spinner from "components/common/Spinner.svelte"
@@ -261,6 +262,11 @@
     }
   }
 
+  const copyAppId = async app => {
+    await Helpers.copyToClipboard(app.prodId)
+    notifications.success("App ID copied to clipboard.")
+  }
+
   function createAppFromTemplateUrl(templateKey) {
     // validate the template key just to make sure
     const templateParts = templateKey.split("/")
@@ -394,6 +400,7 @@
           <div class="appTable">
             {#each filteredApps as app (app.appId)}
               <AppRow
+                {copyAppId}
                 {releaseLock}
                 {editIcon}
                 {app}

packages/builder/src/pages/builder/portal/manage/email/index.svelte

@@ -68,6 +68,26 @@
     }
   }
 
+  async function deleteSmtp() {
+    // Delete the SMTP config
+    try {
+      await API.deleteConfig({
+        id: smtpConfig._id,
+        rev: smtpConfig._rev,
+      })
+      smtpConfig = {
+        config: {},
+      }
+      await admin.getChecklist()
+      notifications.success(`Settings cleared`)
+      analytics.captureEvent(Events.SMTP.SAVED)
+    } catch (error) {
+      notifications.error(
+        `Failed to clear email settings, reason: ${error?.message || "Unknown"}`
+      )
+    }
+  }
+
   async function fetchSmtp() {
     loading = true
     try {
@@ -156,8 +176,15 @@
         </div>
       {/if}
     </Layout>
-    <div>
+    <div class="spectrum-ButtonGroup spectrum-Settings-buttonGroup">
       <Button cta on:click={saveSmtp}>Save</Button>
+      <Button
+        secondary
+        on:click={deleteSmtp}
+        disabled={!$admin.checklist.smtp.checked}
+      >
+        Reset
+      </Button>
     </div>
     <Divider />
     <Layout gap="XS" noPadding>
@@ -186,4 +213,8 @@
     grid-gap: var(--spacing-l);
     align-items: center;
   }
+  .spectrum-Settings-buttonGroup {
+    gap: var(--spectrum-global-dimension-static-size-200);
+    align-items: flex-end;
+  }
 </style>

packages/builder/src/pages/builder/portal/manage/users/_components/UpdateRolesModal.svelte

@@ -14,7 +14,10 @@
   let options = roles
     .filter(role => role._id !== "PUBLIC")
     .map(role => ({ value: role._id, label: role.name }))
-  options.push({ value: NO_ACCESS, label: "No Access" })
+
+  if (!user?.builder?.global) {
+    options.push({ value: NO_ACCESS, label: "No Access" })
+  }
   let selectedRole = user?.roles?.[app?._id]
 
   async function updateUserRoles() {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2338,7 +2338,11 @@
         "type": "boolean",
         "label": "Autocomplete",
         "key": "autocomplete",
-        "defaultValue": false
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "optionsType",
+          "value": "select"
+        }
       },
       {
         "type": "boolean",
@@ -2346,6 +2350,43 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "select",
+        "label": "Type",
+        "key": "optionsType",
+        "defaultValue": "select",
+        "placeholder": "Pick an options type",
+        "options": [
+          {
+            "label": "Select",
+            "value": "select"
+          },
+          {
+            "label": "Checkboxes",
+            "value": "checkbox"
+          }
+        ]
+      },
+      {
+        "type": "select",
+        "label": "Direction",
+        "key": "direction",
+        "defaultValue": "vertical",
+        "options": [
+          {
+            "label": "Horizontal",
+            "value": "horizontal"
+          },
+          {
+            "label": "Vertical",
+            "value": "vertical"
+          }
+        ],
+        "dependsOn": {
+          "setting": "optionsType",
+          "value": "checkbox"
+        }
+      },
       {
         "type": "select",
         "label": "Options source",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.148-alpha.1",
+    "@budibase/bbui": "^1.0.164-alpha.0",
-    "@budibase/frontend-core": "^1.0.148-alpha.1",
+    "@budibase/frontend-core": "^1.0.164-alpha.0",
-    "@budibase/string-templates": "^1.0.148-alpha.1",
+    "@budibase/string-templates": "^1.0.164-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/ClientApp.svelte

@@ -304,7 +304,8 @@
   @media print {
     #spectrum-root,
     #clip-root,
-    #app-root {
+    #app-root,
+    #app-body {
       overflow: visible !important;
     }
   }

packages/client/src/components/app/forms/MultiFieldSelect.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { CoreMultiselect } from "@budibase/bbui"
+  import { CoreMultiselect, CoreCheckboxGroup } from "@budibase/bbui"
   import Field from "./Field.svelte"
   import { getOptions } from "./optionsParser"
   export let field
@@ -15,6 +15,8 @@
   export let customOptions
   export let autocomplete = false
   export let onChange
+  export let optionsType = "select"
+  export let direction = "vertical"
 
   let fieldState
   let fieldApi
@@ -61,17 +63,31 @@
   bind:fieldSchema
 >
   {#if fieldState}
-    <CoreMultiselect
+    {#if !optionsType || optionsType === "select"}
-      value={fieldState.value || []}
+      <CoreMultiselect
-      error={fieldState.error}
+        value={fieldState.value || []}
-      getOptionLabel={flatOptions ? x => x : x => x.label}
+        error={fieldState.error}
-      getOptionValue={flatOptions ? x => x : x => x.value}
+        getOptionLabel={flatOptions ? x => x : x => x.label}
-      id={fieldState.fieldId}
+        getOptionValue={flatOptions ? x => x : x => x.value}
-      disabled={fieldState.disabled}
+        id={fieldState.fieldId}
-      on:change={handleChange}
+        disabled={fieldState.disabled}
-      {placeholder}
+        on:change={handleChange}
-      {options}
+        {placeholder}
-      {autocomplete}
+        {options}
-    />
+        {autocomplete}
+      />
+    {:else if optionsType === "checkbox"}
+      <CoreCheckboxGroup
+        value={fieldState.value || []}
+        id={fieldState.fieldId}
+        disabled={fieldState.disabled}
+        error={fieldState.error}
+        {options}
+        {direction}
+        on:change={handleChange}
+        getOptionLabel={flatOptions ? x => x : x => x.label}
+        getOptionValue={flatOptions ? x => x : x => x.value}
+      />
+    {/if}
   {/if}
 </Field>

packages/client/src/components/devtools/DevTools.svelte

@@ -66,4 +66,9 @@
   .tab-content {
     padding: 0 var(--spacing-xl);
   }
+  @media print {
+    .devtools {
+      display: none;
+    }
+  }
 </style>

packages/client/src/components/devtools/DevToolsHeader.svelte

@@ -71,4 +71,9 @@
   .dev-preview-header :global(.spectrum-Picker-label) {
     color: white !important;
   }
+  @media print {
+    .dev-preview-header {
+      display: none;
+    }
+  }
 </style>

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.148-alpha.1",
+    "@budibase/bbui": "^1.0.164-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/api/configs.js

@@ -20,6 +20,17 @@ export const buildConfigEndpoints = API => ({
     })
   },
 
+  /**
+   * Deletes a global config
+   * @param id the id of the config to delete
+   * @param rev the revision of the config to delete
+   */
+  deleteConfig: async ({ id, rev }) => {
+    return await API.delete({
+      url: `/api/global/configs/${id}/${rev}`,
+    })
+  },
+
   /**
    * Gets the config for a certain tenant.
    * @param tenantId the tenant ID to get the config for

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -10,6 +10,7 @@
   },
   "scripts": {
     "build": "rimraf dist/ && tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/ && yarn postbuild",
+    "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
@@ -68,10 +69,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.148-alpha.1",
+    "@budibase/backend-core": "^1.0.164-alpha.0",
-    "@budibase/client": "^1.0.148-alpha.1",
+    "@budibase/client": "^1.0.164-alpha.0",
-    "@budibase/pro": "1.0.148-alpha.1",
+    "@budibase/pro": "1.0.164-alpha.0",
-    "@budibase/string-templates": "^1.0.148-alpha.1",
+    "@budibase/string-templates": "^1.0.164-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/row/index.ts

@@ -31,7 +31,9 @@ export async function patch(ctx: any): Promise<any> {
     return save(ctx)
   }
   try {
-    const { row, table } = await pickApi(tableId).patch(ctx)
+    const { row, table } = await quotas.addQuery(() =>
+      pickApi(tableId).patch(ctx)
+    )
     ctx.status = 200
     ctx.eventEmitter &&
       ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
@@ -42,7 +44,7 @@ export async function patch(ctx: any): Promise<any> {
   }
 }
 
-const saveRow = async (ctx: any) => {
+export const save = async (ctx: any) => {
   const appId = ctx.appId
   const tableId = getTableId(ctx)
   const body = ctx.request.body
@@ -51,7 +53,9 @@ const saveRow = async (ctx: any) => {
     return patch(ctx)
   }
   try {
-    const { row, table } = await pickApi(tableId).save(ctx)
+    const { row, table } = await quotas.addRow(() =>
+      quotas.addQuery(() => pickApi(tableId).save(ctx))
+    )
     ctx.status = 200
     ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
     ctx.message = `${table.name} saved successfully`
@@ -61,14 +65,10 @@ const saveRow = async (ctx: any) => {
   }
 }
 
-export async function save(ctx: any) {
-  await quotas.addRow(() => saveRow(ctx))
-}
-
 export async function fetchView(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await pickApi(tableId).fetchView(ctx)
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx))
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -77,7 +77,7 @@ export async function fetchView(ctx: any) {
 export async function fetch(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await pickApi(tableId).fetch(ctx)
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx))
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -86,7 +86,7 @@ export async function fetch(ctx: any) {
 export async function find(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await pickApi(tableId).find(ctx)
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx))
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -98,14 +98,16 @@ export async function destroy(ctx: any) {
   const tableId = getTableId(ctx)
   let response, row
   if (inputs.rows) {
-    let { rows } = await pickApi(tableId).bulkDestroy(ctx)
+    let { rows } = await quotas.addQuery(() =>
+      pickApi(tableId).bulkDestroy(ctx)
+    )
     await quotas.removeRows(rows.length)
     response = rows
     for (let row of rows) {
       ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
     }
   } else {
-    let resp = await pickApi(tableId).destroy(ctx)
+    let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx))
     await quotas.removeRow()
     response = resp.response
     row = resp.row
@@ -121,7 +123,7 @@ export async function search(ctx: any) {
   const tableId = getTableId(ctx)
   try {
     ctx.status = 200
-    ctx.body = await pickApi(tableId).search(ctx)
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx))
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -139,7 +141,9 @@ export async function validate(ctx: any) {
 export async function fetchEnrichedRow(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx)
+    ctx.body = await quotas.addQuery(() =>
+      pickApi(tableId).fetchEnrichedRow(ctx)
+    )
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -148,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) {
 export const exportRows = async (ctx: any) => {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await pickApi(tableId).exportRows(ctx)
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx))
   } catch (err) {
     ctx.throw(400, err)
   }

packages/server/src/api/controllers/view/exporters.js

@@ -6,7 +6,7 @@ exports.csv = function (headers, rows) {
       .map(header => {
         let val = row[header]
         val =
-          typeof val === "object"
+          typeof val === "object" && !(val instanceof Date)
             ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
             : `"${val}"`
         return val.trim()

packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -1,9 +1,66 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
+Object {
+  "map": "function (doc) {
+      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      doc[\\"myField\\"] === undefined ||
+      doc[\\"myField\\"] === null ||
+      doc[\\"myField\\"] === \\"\\" ||
+      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+  )) && (doc[\\"age\\"] > 17)) {
+        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+      }
+    }",
+  "meta": Object {
+    "calculation": "stats",
+    "field": "myField",
+    "filters": Array [
+      Object {
+        "condition": "MT",
+        "key": "age",
+        "value": 17,
+      },
+    ],
+    "groupBy": undefined,
+    "schema": Object {
+      "avg": Object {
+        "type": "number",
+      },
+      "count": Object {
+        "type": "number",
+      },
+      "field": Object {
+        "type": "string",
+      },
+      "max": Object {
+        "type": "number",
+      },
+      "min": Object {
+        "type": "number",
+      },
+      "sum": Object {
+        "type": "number",
+      },
+      "sumsqr": Object {
+        "type": "number",
+      },
+    },
+    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
+  },
+  "reduce": "_stats",
+}
+`;
+
 exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
 Object {
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
+      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      doc[\\"myField\\"] === undefined ||
+      doc[\\"myField\\"] === null ||
+      doc[\\"myField\\"] === \\"\\" ||
+      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+  )) ) {
         emit(doc[\\"_id\\"], doc[\\"myField\\"]);
       }
     }",

packages/server/src/api/controllers/view/tests/viewBuilder.spec.js

@@ -44,4 +44,22 @@ describe("viewBuilder", () => {
       })).toMatchSnapshot()
     })
   })
+
+  describe("Calculate and filter", () => {
+    it("creates a view with the calculation statistics and filter schema", () => {
+      expect(viewTemplate({
+        "name": "Calculate View",
+        "field": "myField",
+        "calculation": "stats",
+        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
+        "filters": [
+          {
+            "value": 17,
+            "condition": "MT",
+            "key": "age",
+          }
+        ]
+      })).toMatchSnapshot()
+    })
+  })
 });

packages/server/src/api/controllers/view/utils.js

@@ -7,6 +7,7 @@ const {
 } = require("../../../db/utils")
 const env = require("../../../environment")
 const { getAppDB } = require("@budibase/backend-core/context")
+const viewBuilder = require("./viewBuilder")
 
 exports.getView = async viewName => {
   const db = getAppDB()
@@ -114,7 +115,8 @@ exports.deleteView = async viewName => {
 exports.migrateToInMemoryView = async (db, viewName) => {
   // delete the view initially
   const designDoc = await db.get("_design/database")
-  const view = designDoc.views[viewName]
+  // run the view back through the view builder to update it
+  const view = viewBuilder(designDoc.views[viewName].meta)
   delete designDoc.views[viewName]
   await db.put(designDoc)
   await exports.saveView(db, null, viewName, view)
@@ -123,7 +125,7 @@ exports.migrateToInMemoryView = async (db, viewName) => {
 exports.migrateToDesignView = async (db, viewName) => {
   let view = await db.get(generateMemoryViewID(viewName))
   const designDoc = await db.get("_design/database")
-  designDoc.views[viewName] = view.view
+  designDoc.views[viewName] = viewBuilder(view.view.meta)
   await db.put(designDoc)
   await db.remove(view._id, view._rev)
 }

packages/server/src/api/controllers/view/viewBuilder.js

@@ -10,6 +10,12 @@ const TOKEN_MAP = {
   OR: "||",
 }
 
+const CONDITIONS = {
+  EMPTY: "EMPTY",
+  NOT_EMPTY: "NOT_EMPTY",
+  CONTAINS: "CONTAINS",
+}
+
 const isEmptyExpression = key => {
   return `(
       doc["${key}"] === undefined ||
@@ -77,13 +83,13 @@ function parseFilterExpression(filters) {
       expression.push(TOKEN_MAP[filter.conjunction])
     }
 
-    if (filter.condition === "CONTAINS") {
+    if (filter.condition === CONDITIONS.CONTAINS) {
       expression.push(
         `doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
       )
-    } else if (filter.condition === "EMPTY") {
+    } else if (filter.condition === CONDITIONS.EMPTY) {
       expression.push(isEmptyExpression(filter.key))
-    } else if (filter.condition === "NOT_EMPTY") {
+    } else if (filter.condition === CONDITIONS.NOT_EMPTY) {
       expression.push(`!${isEmptyExpression(filter.key)}`)
     } else {
       const value =
@@ -125,22 +131,37 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction
   }
-  const parsedFilters = parseFilterExpression(filters)
-  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
 
-  const emitExpression = parseEmitExpression(field, groupBy)
+  let schema = null,
-
+    statFilter = null
-  const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
-
-  let schema = null
 
   if (calculation) {
     schema = {
       ...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
       ...SCHEMA_MAP[calculation],
     }
+    if (
+      !filters.find(
+        filter =>
+          filter.key === field && filter.condition === CONDITIONS.NOT_EMPTY
+      )
+    ) {
+      statFilter = parseFilterExpression([
+        { key: field, condition: CONDITIONS.NOT_EMPTY },
+      ])
+    }
   }
 
+  const parsedFilters = parseFilterExpression(filters)
+  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
+
+  const emitExpression = parseEmitExpression(field, groupBy)
+  const tableExpression = `doc.tableId === "${tableId}"`
+  const coreExpression = statFilter
+    ? `(${tableExpression} && ${statFilter})`
+    : tableExpression
+  const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
+
   return {
     meta: {
       field,
@@ -151,7 +172,7 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
       calculation,
     },
     map: `function (doc) {
-      if (doc.tableId === "${tableId}" ${filterExpression}) {
+      if (${coreExpression} ${filterExpression}) {
         ${emitExpression}
       }
     }`,

packages/server/src/api/routes/tests/row.spec.js

@@ -3,6 +3,7 @@ const setup = require("./utilities")
 const { basicRow } = setup.structures
 const { doInAppContext } = require("@budibase/backend-core/context")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
+const { quotas, QuotaUsageType, StaticQuotaName, MonthlyQuotaName } = require("@budibase/pro")
 
 // mock the fetch for the search system
 jest.mock("node-fetch")
@@ -28,9 +29,29 @@ describe("/rows", () => {
       .expect('Content-Type', /json/)
       .expect(status)
 
+  const getRowUsage = async () => {
+    return config.doInContext(null, () => quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
+  }
+
+  const getQueryUsage = async () => {
+    return config.doInContext(null, () => quotas.getCurrentUsageValue(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
+  }
+
+  const assertRowUsage = async (expected) => {
+    const usage = await getRowUsage()
+    expect(usage).toBe(expected)
+  }
+
+  const assertQueryUsage = async (expected) => {
+    const usage = await getQueryUsage()
+    expect(usage).toBe(expected)
+  }
 
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .post(`/api/${row.tableId}/rows`)
         .send(row)
@@ -40,10 +61,14 @@ describe("/rows", () => {
       expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
       expect(res.body.name).toEqual("Test Contact")
       expect(res.body._rev).toBeDefined()
+      await assertRowUsage(rowUsage + 1)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${table._id}/rows`)
@@ -59,10 +84,13 @@ describe("/rows", () => {
       
       expect(res.res.statusMessage).toEqual(`${table.name} updated successfully.`)
       expect(res.body.name).toEqual("Updated Name")
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should load a row", async () => {
       const existing = await config.createRow()
+      const queryUsage = await getQueryUsage()
 
       const res = await request
         .get(`/api/${table._id}/rows/${existing._id}`)
@@ -76,6 +104,7 @@ describe("/rows", () => {
         _rev: existing._rev,
         type: "row",
       })
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should list all rows for given tableId", async () => {
@@ -86,6 +115,7 @@ describe("/rows", () => {
       }
       await config.createRow()
       await config.createRow(newRow)
+      const queryUsage = await getQueryUsage()
 
       const res = await request
         .get(`/api/${table._id}/rows`)
@@ -96,15 +126,19 @@ describe("/rows", () => {
       expect(res.body.length).toBe(2)
       expect(res.body.find(r => r.name === newRow.name)).toBeDefined()
       expect(res.body.find(r => r.name === row.name)).toBeDefined()
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("load should return 404 when row does not exist", async () => {
       await config.createRow()
+      const queryUsage = await getQueryUsage()
+
       await request
         .get(`/api/${table._id}/rows/not-a-valid-id`)
         .set(config.defaultHeaders())
         .expect('Content-Type', /json/)
         .expect(404)
+      await assertQueryUsage(queryUsage) // no change
     })
 
     it("row values are coerced", async () => {
@@ -202,6 +236,9 @@ describe("/rows", () => {
     it("should update only the fields that are supplied", async () => {
       const existing = await config.createRow()
 
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .patch(`/api/${table._id}/rows`)
         .send({
@@ -222,10 +259,15 @@ describe("/rows", () => {
 
       expect(savedRow.body.description).toEqual(existing.description)
       expect(savedRow.body.name).toEqual("Updated Name")
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage + 2) // account for the second load
     })
 
     it("should throw an error when given improper types", async () => {
       const existing = await config.createRow()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       await request
         .patch(`/api/${table._id}/rows`)
         .send({
@@ -236,12 +278,18 @@ describe("/rows", () => {
         })
         .set(config.defaultHeaders())
         .expect(400)
+
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage)
     })
   })
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
       const createdRow = await config.createRow(row)
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .delete(`/api/${table._id}/rows`)
         .send({
@@ -253,11 +301,16 @@ describe("/rows", () => {
         .expect('Content-Type', /json/)
         .expect(200)
       expect(res.body[0]._id).toEqual(createdRow._id)
+      await assertRowUsage(rowUsage -1)
+      await assertQueryUsage(queryUsage +1)
     })
   })
 
   describe("validate", () => {
     it("should return no errors on valid row", async () => {
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .post(`/api/${table._id}/rows/validate`)
         .send({ name: "ivan" })
@@ -267,9 +320,14 @@ describe("/rows", () => {
       
       expect(res.body.valid).toBe(true)
       expect(Object.keys(res.body.errors)).toEqual([])
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage)
     })
 
     it("should errors on invalid row", async () => {
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .post(`/api/${table._id}/rows/validate`)
         .send({ name: 1 })
@@ -279,7 +337,8 @@ describe("/rows", () => {
       
       expect(res.body.valid).toBe(false)
       expect(Object.keys(res.body.errors)).toEqual(["name"])
-
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage)
     })
   })
 
@@ -287,6 +346,9 @@ describe("/rows", () => {
     it("should be able to delete a bulk set of rows", async () => {
       const row1 = await config.createRow()
       const row2 = await config.createRow()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .delete(`/api/${table._id}/rows`)
         .send({
@@ -298,14 +360,20 @@ describe("/rows", () => {
         .set(config.defaultHeaders())
         .expect('Content-Type', /json/)
         .expect(200)
+
       expect(res.body.length).toEqual(2)
       await loadRow(row1._id, 404)
+      await assertRowUsage(rowUsage - 2)
+      await assertQueryUsage(queryUsage +1)
     })
   })
 
   describe("fetchView", () => {
     it("should be able to fetch tables contents via 'view'", async () => {
       const row = await config.createRow()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .get(`/api/views/${table._id}`)
         .set(config.defaultHeaders())
@@ -313,18 +381,29 @@ describe("/rows", () => {
         .expect(200)
       expect(res.body.length).toEqual(1)
       expect(res.body[0]._id).toEqual(row._id)
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage +1)
     })
 
     it("should throw an error if view doesn't exist", async () => {
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       await request
         .get(`/api/views/derp`)
         .set(config.defaultHeaders())
         .expect(404)
+
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage)
     })
 
     it("should be able to run on a view", async () => {
       const view = await config.createView()
       const row = await config.createRow()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
       const res = await request
         .get(`/api/views/${view.name}`)
         .set(config.defaultHeaders())
@@ -332,13 +411,12 @@ describe("/rows", () => {
         .expect(200)
       expect(res.body.length).toEqual(1)
       expect(res.body[0]._id).toEqual(row._id)
+
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage + 1)
     })
   })
 
-  describe("user testing", () => {
-
-  })
-
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
       const { table, firstRow, secondRow } = await doInTenant(setup.structures.TENANT_ID, async () => {
@@ -356,6 +434,8 @@ describe("/rows", () => {
         })
         return { table, firstRow, secondRow }
       })
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
 
       // test basic enrichment
       const resBasic = await request
@@ -376,6 +456,8 @@ describe("/rows", () => {
       expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
       expect(resEnriched.body.link[0].name).toBe("Test Contact")
       expect(resEnriched.body.link[0].description).toBe("original description")
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage +2)
     })
   })
 

packages/server/src/automations/steps/queryRows.js

@@ -50,6 +50,7 @@ exports.definition = {
         limit: {
           type: "number",
           title: "Limit",
+          customType: "queryLimit",
         },
       },
       required: ["tableId"],

packages/server/src/constants/index.js

@@ -208,5 +208,10 @@ exports.AutomationErrors = {
   FAILURE_CONDITION: "FAILURE_CONDITION_MET",
 }
 
+exports.LoopStepTypes = {
+  ARRAY: "Array",
+  STRING: "String",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

packages/server/src/integrations/base/sql.ts

@@ -21,6 +21,31 @@ type KnexQuery = Knex.QueryBuilder | Knex
 const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
 const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
 
+function likeKey(client: string, key: string): string {
+  if (!key.includes(" ")) {
+    return key
+  }
+  let start: string, end: string
+  switch (client) {
+    case SqlClients.MY_SQL:
+      start = end = "`"
+      break
+    case SqlClients.ORACLE:
+    case SqlClients.POSTGRES:
+      start = end = '"'
+      break
+    case SqlClients.MS_SQL:
+      start = "["
+      end = "]"
+      break
+    default:
+      throw "Unknown client"
+  }
+  const parts = key.split(".")
+  key = parts.map(part => `${start}${part}${end}`).join(".")
+  return key
+}
+
 function parse(input: any) {
   if (Array.isArray(input)) {
     return JSON.stringify(input)
@@ -125,7 +150,9 @@ class InternalBuilder {
         } else {
           const rawFnc = `${fnc}Raw`
           // @ts-ignore
-          query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`%${value}%`])
+          query = query[rawFnc](`LOWER(${likeKey(this.client, key)}) LIKE ?`, [
+            `%${value}%`,
+          ])
         }
       })
     }

packages/server/src/integrations/mongodb.ts

@@ -4,10 +4,18 @@ import {
   QueryTypes,
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
+import {
+  MongoClient,
+  ObjectID,
+  FilterQuery,
+  UpdateQuery,
+  FindOneAndUpdateOption,
+  UpdateOneOptions,
+  UpdateManyOptions,
+  CommonOptions,
+} from "mongodb"
 
 module MongoDBModule {
-  const { MongoClient } = require("mongodb")
-
   interface MongoDBConfig {
     connectionString: string
     db: string
@@ -76,20 +84,76 @@ module MongoDBModule {
       return this.client.connect()
     }
 
+    createObjectIds(json: any): object {
+      const self = this
+      function interpolateObjectIds(json: any) {
+        for (let field of Object.keys(json)) {
+          if (json[field] instanceof Object) {
+            json[field] = self.createObjectIds(json[field])
+          }
+          if (field === "_id" && typeof json[field] === "string") {
+            const id = json["_id"].match(
+              /(?<=objectid\(['"]).*(?=['"]\))/gi
+            )?.[0]
+            if (id) {
+              json["_id"] = ObjectID.createFromHexString(id)
+            }
+          }
+        }
+        return json
+      }
+
+      if (Array.isArray(json)) {
+        for (let i = 0; i < json.length; i++) {
+          json[i] = interpolateObjectIds(json[i])
+        }
+        return json
+      }
+      return interpolateObjectIds(json)
+    }
+
+    parseQueryParams(params: string, mode: string) {
+      let queryParams = params.split(/(?<=(},)).*{/g)
+      let group1 = queryParams[0]
+      let group2 = queryParams[2]
+      let group3 = queryParams[4]
+      if (group1) {
+        group1 = JSON.parse(group1.replace(/,+$/, ""))
+      }
+      if (group2) {
+        group2 = JSON.parse("{" + group2.replace(/,+$/, ""))
+      }
+      if (group3) {
+        group3 = JSON.parse("{" + group3.replace(/,+$/, ""))
+      }
+      if (mode === "update") {
+        return {
+          filter: group1,
+          update: group2,
+          options: group3 ?? {},
+        }
+      }
+      return {
+        filter: group1,
+        options: group2 ?? {},
+      }
+    }
+
     async create(query: { json: object; extra: { [key: string]: string } }) {
       try {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let json = this.createObjectIds(query.json)
 
         // For mongodb we add an extra actionType to specify
         // which method we want to call on the collection
         switch (query.extra.actionTypes) {
           case "insertOne": {
-            return await collection.insertOne(query.json)
+            return await collection.insertOne(json)
           }
           case "insertMany": {
-            return await collection.insertOne(query.json).toArray()
+            return await collection.insertMany(json)
           }
           default: {
             throw new Error(
@@ -110,22 +174,32 @@ module MongoDBModule {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let json = this.createObjectIds(query.json)
 
         switch (query.extra.actionTypes) {
           case "find": {
-            return await collection.find(query.json).toArray()
+            return await collection.find(json).toArray()
           }
           case "findOne": {
-            return await collection.findOne(query.json)
+            return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            return await collection.findOneAndUpdate(query.json)
+            let findAndUpdateJson = json as {
+              filter: FilterQuery<any>
+              update: UpdateQuery<any>
+              options: FindOneAndUpdateOption<any>
+            }
+            return await collection.findOneAndUpdate(
+              findAndUpdateJson.filter,
+              findAndUpdateJson.update,
+              findAndUpdateJson.options
+            )
           }
           case "count": {
-            return await collection.countDocuments(query.json)
+            return await collection.countDocuments(json)
           }
           case "distinct": {
-            return await collection.distinct(query.json)
+            return await collection.distinct(json)
           }
           default: {
             throw new Error(
@@ -146,13 +220,30 @@ module MongoDBModule {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let queryJson = query.json
+        if (typeof queryJson === "string") {
+          queryJson = this.parseQueryParams(queryJson, "update")
+        }
+        let json = this.createObjectIds(queryJson) as {
+          filter: FilterQuery<any>
+          update: UpdateQuery<any>
+          options: object
+        }
 
         switch (query.extra.actionTypes) {
           case "updateOne": {
-            return await collection.updateOne(query.json)
+            return await collection.updateOne(
+              json.filter,
+              json.update,
+              json.options as UpdateOneOptions
+            )
           }
           case "updateMany": {
-            return await collection.updateMany(query.json).toArray()
+            return await collection.updateMany(
+              json.filter,
+              json.update,
+              json.options as UpdateManyOptions
+            )
           }
           default: {
             throw new Error(
@@ -173,13 +264,21 @@ module MongoDBModule {
         await this.connect()
         const db = this.client.db(this.config.db)
         const collection = db.collection(query.extra.collection)
+        let queryJson = query.json
+        if (typeof queryJson === "string") {
+          queryJson = this.parseQueryParams(queryJson, "delete")
+        }
+        let json = this.createObjectIds(queryJson) as {
+          filter: FilterQuery<any>
+          options: CommonOptions
+        }
 
         switch (query.extra.actionTypes) {
           case "deleteOne": {
-            return await collection.deleteOne(query.json)
+            return await collection.deleteOne(json.filter, json.options)
           }
           case "deleteMany": {
-            return await collection.deleteMany(query.json).toArray()
+            return await collection.deleteMany(json.filter, json.options)
           }
           default: {
             throw new Error(

packages/server/src/integrations/postgres.ts

@@ -147,7 +147,7 @@ module PostgresModule {
       return parts.join(" || ")
     }
 
-    async internalQuery(query: SqlQuery) {
+    async internalQuery(query: SqlQuery, close: boolean = true) {
       const client = this.client
       this.index = 1
       // need to handle a specific issue with json data types in postgres,
@@ -164,10 +164,11 @@ module PostgresModule {
       try {
         return await client.query(query.sql, query.bindings || [])
       } catch (err) {
+        await this.client.end()
         // @ts-ignore
         throw new Error(err)
       } finally {
-        await this.client.end()
+        if (close) await this.client.end()
       }
     }
 
@@ -204,48 +205,54 @@ module PostgresModule {
         }
       } catch (err) {
         tableKeys = {}
-      } finally {
-        await this.client.close()
       }
 
-      const columnsResponse = await this.client.query(this.COLUMNS_SQL)
+      try {
-      const tables: { [key: string]: Table } = {}
+        const columnsResponse = await this.client.query(this.COLUMNS_SQL)
 
-      for (let column of columnsResponse.rows) {
+        const tables: { [key: string]: Table } = {}
-        const tableName: string = column.table_name
-        const columnName: string = column.column_name
 
-        // table key doesn't exist yet
+        for (let column of columnsResponse.rows) {
-        if (!tables[tableName] || !tables[tableName].schema) {
+          const tableName: string = column.table_name
-          tables[tableName] = {
+          const columnName: string = column.column_name
-            _id: buildExternalTableId(datasourceId, tableName),
+
-            primary: tableKeys[tableName] || [],
+          // table key doesn't exist yet
-            name: tableName,
+          if (!tables[tableName] || !tables[tableName].schema) {
-            schema: {},
+            tables[tableName] = {
+              _id: buildExternalTableId(datasourceId, tableName),
+              primary: tableKeys[tableName] || [],
+              name: tableName,
+              schema: {},
+            }
+          }
+
+          const identity = !!(
+            column.identity_generation ||
+            column.identity_start ||
+            column.identity_increment
+          )
+          const hasDefault =
+            typeof column.column_default === "string" &&
+            column.column_default.startsWith("nextval")
+          const isGenerated =
+            column.is_generated && column.is_generated !== "NEVER"
+          const isAuto: boolean = hasDefault || identity || isGenerated
+          tables[tableName].schema[columnName] = {
+            autocolumn: isAuto,
+            name: columnName,
+            ...convertSqlType(column.data_type),
           }
         }
 
-        const identity = !!(
+        const final = finaliseExternalTables(tables, entities)
-          column.identity_generation ||
+        this.tables = final.tables
-          column.identity_start ||
+        this.schemaErrors = final.errors
-          column.identity_increment
+      } catch (err) {
-        )
+        // @ts-ignore
-        const hasDefault =
+        throw new Error(err)
-          typeof column.column_default === "string" &&
+      } finally {
-          column.column_default.startsWith("nextval")
+        await this.client.end()
-        const isGenerated =
-          column.is_generated && column.is_generated !== "NEVER"
-        const isAuto: boolean = hasDefault || identity || isGenerated
-        tables[tableName].schema[columnName] = {
-          autocolumn: isAuto,
-          name: columnName,
-          ...convertSqlType(column.data_type),
-        }
       }
-
-      const final = finaliseExternalTables(tables, entities)
-      this.tables = final.tables
-      this.schemaErrors = final.errors
     }
 
     async create(query: SqlQuery | string) {
@@ -274,8 +281,9 @@ module PostgresModule {
       if (Array.isArray(input)) {
         const responses = []
         for (let query of input) {
-          responses.push(await this.internalQuery(query))
+          responses.push(await this.internalQuery(query, false))
         }
+        await this.client.end()
         return responses
       } else {
         const response = await this.internalQuery(input)

packages/server/src/integrations/tests/mongo.spec.js

@@ -4,19 +4,19 @@ jest.mock("mongodb")
 
 class TestConfiguration {
   constructor(config = {}) {
-    this.integration = new MongoDBIntegration.integration(config) 
+    this.integration = new MongoDBIntegration.integration(config)
   }
 }
 
 function disableConsole() {
-  jest.spyOn(console, 'error');
+  jest.spyOn(console, "error")
-  console.error.mockImplementation(() => {});
+  console.error.mockImplementation(() => {})
 
-  return console.error.mockRestore;
+  return console.error.mockRestore
 }
 
 describe("MongoDB Integration", () => {
-  let config 
+  let config
   let indexName = "Users"
 
   beforeEach(() => {
@@ -25,12 +25,12 @@ describe("MongoDB Integration", () => {
 
   it("calls the create method with the correct params", async () => {
     const body = {
-      name: "Hello"
+      name: "Hello",
     }
     await config.integration.create({
       index: indexName,
       json: body,
-      extra: { collection: 'testCollection', actionTypes: 'insertOne'}
+      extra: { collection: "testCollection", actionTypes: "insertOne" },
     })
     expect(config.integration.client.insertOne).toHaveBeenCalledWith(body)
   })
@@ -38,9 +38,9 @@ describe("MongoDB Integration", () => {
   it("calls the read method with the correct params", async () => {
     const query = {
       json: {
-        address: "test"
+        address: "test",
       },
-      extra: { collection: 'testCollection', actionTypes: 'find'}
+      extra: { collection: "testCollection", actionTypes: "find" },
     }
     const response = await config.integration.read(query)
     expect(config.integration.client.find).toHaveBeenCalledWith(query.json)
@@ -50,30 +50,47 @@ describe("MongoDB Integration", () => {
   it("calls the delete method with the correct params", async () => {
     const query = {
       json: {
-        id: "test"
+        filter: {
+          id: "test",
+        },
+        options: {
+          opt: "option"
+        }
       },
-      extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
+      extra: { collection: "testCollection", actionTypes: "deleteOne" },
     }
     await config.integration.delete(query)
-    expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json)
+    expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json.filter, query.json.options)
   })
 
   it("calls the update method with the correct params", async () => {
     const query = {
       json: {
-        id: "test"
+        filter: {
+          id: "test",
+        },
+        update: {
+          name: "TestName",
+        },
+        options: {
+          upsert: false,
+        },
       },
-      extra: { collection: 'testCollection', actionTypes: 'updateOne'}
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
     }
     await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json)
+    expect(config.integration.client.updateOne).toHaveBeenCalledWith(
+      query.json.filter,
+      query.json.update,
+      query.json.options
+    )
   })
 
   it("throws an error when an invalid query.extra.actionType is passed for each method", async () => {
     const restore = disableConsole()
 
     const query = {
-      extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
+      extra: { collection: "testCollection", actionTypes: "deleteOne" },
     }
 
     let error = null
@@ -85,4 +102,4 @@ describe("MongoDB Integration", () => {
     expect(error).toBeDefined()
     restore()
   })
-})
+})

packages/server/src/threads/automation.js

@@ -8,7 +8,7 @@ const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-const { AutomationErrors } = require("../constants")
+const { AutomationErrors, LoopStepTypes } = require("../constants")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 
@@ -17,6 +17,41 @@ const STOPPED_STATUS = { success: false, status: "STOPPED" }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
+function typecastForLooping(loopStep, input) {
+  if (!input || !input.binding) {
+    return null
+  }
+  const isArray = Array.isArray(input.binding),
+    isString = typeof input.binding === "string"
+  try {
+    switch (loopStep.inputs.option) {
+      case LoopStepTypes.ARRAY:
+        if (isString) {
+          return JSON.parse(input.binding)
+        }
+        break
+      case LoopStepTypes.STRING:
+        if (isArray) {
+          return input.binding.join(",")
+        }
+        break
+    }
+  } catch (err) {
+    throw new Error("Unable to cast to correct type")
+  }
+  return input.binding
+}
+
+function getLoopIterations(loopStep, input) {
+  const binding = typecastForLooping(loopStep, input)
+  if (!loopStep || !binding) {
+    return 1
+  }
+  return Array.isArray(binding)
+    ? binding.length
+    : automationUtils.stringSplit(binding).length
+}
+
 /**
  * The automation orchestrator is a class responsible for executing automations.
  * It handles the context of the automation and makes sure each step gets the correct
@@ -107,7 +142,9 @@ class Orchestrator {
     let loopSteps = []
     for (let step of automation.definition.steps) {
       stepCount++
-      let input
+      let input,
+        iterations = 1,
+        iterationCount = 0
       if (step.stepId === LOOP_STEP_ID) {
         loopStep = step
         loopStepNumber = stepCount
@@ -116,13 +153,9 @@ class Orchestrator {
 
       if (loopStep) {
         input = await processObject(loopStep.inputs, this._context)
+        iterations = getLoopIterations(loopStep, input)
       }
-      let iterations = loopStep
+
-        ? Array.isArray(input.binding)
-          ? input.binding.length
-          : automationUtils.stringSplit(input.binding).length
-        : 1
-      let iterationCount = 0
       for (let index = 0; index < iterations; index++) {
         let originalStepInput = cloneDeep(step.inputs)
 
@@ -132,18 +165,11 @@ class Orchestrator {
             loopStep.inputs,
             cloneDeep(this._context)
           )
-          newInput = automationUtils.cleanInputValues(
-            newInput,
-            loopStep.schema.inputs
-          )
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
-          if (
+          try {
-            (loopStep.inputs.option === "Array" &&
+            newInput.binding = typecastForLooping(loopStep, newInput)
-              !Array.isArray(newInput.binding)) ||
+          } catch (err) {
-            (loopStep.inputs.option === "String" &&
-              typeof newInput.binding !== "string")
-          ) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
               status: AutomationErrors.INCORRECT_TYPE,
               success: false,
@@ -205,21 +231,13 @@ class Orchestrator {
           }
 
           let isFailure = false
-          if (
+          const currentItem = this._context.steps[loopStepNumber]?.currentItem
-            typeof this._context.steps[loopStepNumber]?.currentItem === "object"
+          if (currentItem && typeof currentItem === "object") {
-          ) {
+            isFailure = Object.keys(currentItem).some(value => {
-            isFailure = Object.keys(
+              return currentItem[value] === loopStep.inputs.failure
-              this._context.steps[loopStepNumber].currentItem
-            ).some(value => {
-              return (
-                this._context.steps[loopStepNumber].currentItem[value] ===
-                loopStep.inputs.failure
-              )
             })
           } else {
-            isFailure =
+            isFailure = currentItem && currentItem === loopStep.inputs.failure
-              this._context.steps[loopStepNumber]?.currentItem ===
-              loopStep.inputs.failure
           }
 
           if (isFailure) {

packages/server/yarn.lock

@@ -1014,15 +1014,15 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.148-alpha.1":
+"@budibase/backend-core@1.0.164-alpha.0":
-  version "1.0.148-alpha.1"
+  version "1.0.164-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.148-alpha.1.tgz#56a47d8fe7638ca4094642e4d8e5394f428c1764"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164-alpha.0.tgz#80c7d76de3f643302fcb42b1857500216d405e24"
-  integrity sha512-N5u8P2lv8PieC2Nkb8OnjYODYPz6mi2xN+k3LM0KUcDj+GHwCqlbXSLpYQlSwY5/kMKU6n0yLamKBDLjZabwCA==
+  integrity sha512-mbCJnImaFVF9XuiXbUM6YNj6E0L6Mg/kUAOOMc0SCPFL7c1ugY/6tQVa/6PQPrQW13qqj6EcbOdXlZ5k5kWPQA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
     bcryptjs "^2.4.3"
-    cls-hooked "^4.2.2"
+    emitter-listener "^1.1.2"
     ioredis "^4.27.1"
     jsonwebtoken "^8.5.1"
     koa-passport "^4.1.4"
@@ -1091,12 +1091,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.148-alpha.1":
+"@budibase/pro@1.0.164-alpha.0":
-  version "1.0.148-alpha.1"
+  version "1.0.164-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.148-alpha.1.tgz#67d797adc3f68f34c84bc1fd80e34d013aaf824e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164-alpha.0.tgz#9ab6c3f94ab5c73123590a08e6283ef3fedcff98"
-  integrity sha512-9xLOSBNuvGvAKue2PWpjAEfY79fg6ybdaBR3sol/W64otWJs2zCTQKeGtYDQX5HI/LWJD60xaDopQ1T1sOB/YA==
+  integrity sha512-QBvwcwKRpI43TwlzgzKLC+KAr0Tijls4G2E0tN5sjirMdUYhoeQEzBQ5URtrK8ko5BkZnCVadsA8CYGeUNcmkg==
   dependencies:
-    "@budibase/backend-core" "1.0.148-alpha.1"
+    "@budibase/backend-core" "1.0.164-alpha.0"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":
@@ -3420,13 +3420,6 @@ astral-regex@^1.0.0:
   resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-1.0.0.tgz#6c8c3fb827dd43ee3918f27b82782ab7658a6fd9"
   integrity sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==
 
-async-hook-jl@^1.7.6:
-  version "1.7.6"
-  resolved "https://registry.yarnpkg.com/async-hook-jl/-/async-hook-jl-1.7.6.tgz#4fd25c2f864dbaf279c610d73bf97b1b28595e68"
-  integrity sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==
-  dependencies:
-    stack-chain "^1.3.7"
-
 async-limiter@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
@@ -4222,15 +4215,6 @@ clone-response@1.0.2, clone-response@^1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
-cls-hooked@^4.2.2:
-  version "4.2.2"
-  resolved "https://registry.yarnpkg.com/cls-hooked/-/cls-hooked-4.2.2.tgz#ad2e9a4092680cdaffeb2d3551da0e225eae1908"
-  integrity sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==
-  dependencies:
-    async-hook-jl "^1.7.6"
-    emitter-listener "^1.0.1"
-    semver "^5.4.1"
-
 cluster-key-slot@1.1.0, cluster-key-slot@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
@@ -5068,7 +5052,7 @@ electron-to-chromium@^1.4.17:
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.71.tgz#17056914465da0890ce00351a3b946fd4cd51ff6"
   integrity sha512-Hk61vXXKRb2cd3znPE9F+2pLWdIOmP7GjiTj45y6L3W/lO+hSnUSUhq+6lEaERWBdZOHbk2s3YV5c9xVl3boVw==
 
-emitter-listener@^1.0.1:
+emitter-listener@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8"
   integrity sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==
@@ -11599,7 +11583,7 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.4.1, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
+"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
@@ -11993,11 +11977,6 @@ sshpk@^1.7.0:
     safer-buffer "^2.0.2"
     tweetnacl "~0.14.0"
 
-stack-chain@^1.3.7:
-  version "1.3.7"
-  resolved "https://registry.yarnpkg.com/stack-chain/-/stack-chain-1.3.7.tgz#d192c9ff4ea6a22c94c4dd459171e3f00cea1285"
-  integrity sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=
-
 stack-trace@0.0.x:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/src/helpers/index.js

@@ -13,6 +13,16 @@ const HTML_SWAPS = {
   ">": ">",
 }
 
+function isObject(value) {
+  if (value == null || typeof value !== "object") {
+    return false
+  }
+  return (
+    value.toString() === "[object Object]" ||
+    (value.length > 0 && typeof value[0] === "object")
+  )
+}
+
 const HELPERS = [
   // external helpers
   new Helper(HelperFunctionNames.OBJECT, value => {
@@ -22,11 +32,7 @@ const HELPERS = [
   new Helper(HelperFunctionNames.JS, processJS, false),
   // this help is applied to all statements
   new Helper(HelperFunctionNames.ALL, (value, { __opts }) => {
-    if (
+    if (isObject(value)) {
-      value != null &&
-      typeof value === "object" &&
-      value.toString() === "[object Object]"
-    ) {
       return new SafeString(JSON.stringify(value))
     }
     // null/undefined values produce bad results

packages/string-templates/src/processors/preprocessor.js

@@ -64,9 +64,10 @@ module.exports.processors = [
         return statement
       }
     }
+    const testHelper = possibleHelper.trim().toLowerCase()
     if (
       !noHelpers &&
-      HelperNames().some(option => option.includes(possibleHelper))
+      HelperNames().some(option => testHelper === option.toLowerCase())
     ) {
       insideStatement = `(${insideStatement})`
     }

packages/string-templates/test/basic.spec.js

@@ -106,6 +106,16 @@ describe("Test that the object processing works correctly", () => {
   })
 })
 
+describe("check returning objects", () => {
+  it("should handle an array of objects", async () => {
+    const json = [{a: 1},{a: 2}]
+    const output = await processString("{{ testing }}", {
+      testing: json
+    })
+    expect(output).toEqual(JSON.stringify(json))
+  })
+})
+
 describe("check the utility functions", () => {
   it("should return false for an invalid template string", () => {
     const valid = isValid("{{ table1.thing prop }}")

packages/string-templates/test/escapes.spec.js

@@ -30,6 +30,11 @@ describe("Handling context properties with spaces in their name", () => {
     })
     expect(output).toBe("testcase 1")
   })
+
+  it("should allow the use of a", async () => {
+    const output = await processString("{{ a }}", { a: 1 })
+    expect(output).toEqual("1")
+  })
 })
 
 describe("attempt some complex problems", () => {

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.148-alpha.1",
+  "version": "1.0.164-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -31,9 +31,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.148-alpha.1",
+    "@budibase/backend-core": "^1.0.164-alpha.0",
-    "@budibase/pro": "1.0.148-alpha.1",
+    "@budibase/pro": "1.0.164-alpha.0",
-    "@budibase/string-templates": "^1.0.148-alpha.1",
+    "@budibase/string-templates": "^1.0.164-alpha.0",
     "@koa/router": "^8.0.0",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/yarn.lock

@@ -293,15 +293,15 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.148-alpha.1":
+"@budibase/backend-core@1.0.164-alpha.0":
-  version "1.0.148-alpha.1"
+  version "1.0.164-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.148-alpha.1.tgz#56a47d8fe7638ca4094642e4d8e5394f428c1764"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164-alpha.0.tgz#80c7d76de3f643302fcb42b1857500216d405e24"
-  integrity sha512-N5u8P2lv8PieC2Nkb8OnjYODYPz6mi2xN+k3LM0KUcDj+GHwCqlbXSLpYQlSwY5/kMKU6n0yLamKBDLjZabwCA==
+  integrity sha512-mbCJnImaFVF9XuiXbUM6YNj6E0L6Mg/kUAOOMc0SCPFL7c1ugY/6tQVa/6PQPrQW13qqj6EcbOdXlZ5k5kWPQA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
     bcryptjs "^2.4.3"
-    cls-hooked "^4.2.2"
+    emitter-listener "^1.1.2"
     ioredis "^4.27.1"
     jsonwebtoken "^8.5.1"
     koa-passport "^4.1.4"
@@ -321,12 +321,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.148-alpha.1":
+"@budibase/pro@1.0.164-alpha.0":
-  version "1.0.148-alpha.1"
+  version "1.0.164-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.148-alpha.1.tgz#67d797adc3f68f34c84bc1fd80e34d013aaf824e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164-alpha.0.tgz#9ab6c3f94ab5c73123590a08e6283ef3fedcff98"
-  integrity sha512-9xLOSBNuvGvAKue2PWpjAEfY79fg6ybdaBR3sol/W64otWJs2zCTQKeGtYDQX5HI/LWJD60xaDopQ1T1sOB/YA==
+  integrity sha512-QBvwcwKRpI43TwlzgzKLC+KAr0Tijls4G2E0tN5sjirMdUYhoeQEzBQ5URtrK8ko5BkZnCVadsA8CYGeUNcmkg==
   dependencies:
-    "@budibase/backend-core" "1.0.148-alpha.1"
+    "@budibase/backend-core" "1.0.164-alpha.0"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":
@@ -1290,13 +1290,6 @@ astral-regex@^1.0.0:
   resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-1.0.0.tgz#6c8c3fb827dd43ee3918f27b82782ab7658a6fd9"
   integrity sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==
 
-async-hook-jl@^1.7.6:
-  version "1.7.6"
-  resolved "https://registry.yarnpkg.com/async-hook-jl/-/async-hook-jl-1.7.6.tgz#4fd25c2f864dbaf279c610d73bf97b1b28595e68"
-  integrity sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==
-  dependencies:
-    stack-chain "^1.3.7"
-
 async@~2.1.4:
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc"
@@ -1764,15 +1757,6 @@ clone-response@^1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
-cls-hooked@^4.2.2:
-  version "4.2.2"
-  resolved "https://registry.yarnpkg.com/cls-hooked/-/cls-hooked-4.2.2.tgz#ad2e9a4092680cdaffeb2d3551da0e225eae1908"
-  integrity sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==
-  dependencies:
-    async-hook-jl "^1.7.6"
-    emitter-listener "^1.0.1"
-    semver "^5.4.1"
-
 cluster-key-slot@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
@@ -2258,7 +2242,7 @@ electron-to-chromium@^1.4.17:
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.58.tgz#cd980b08338210b591c25492857a518fe286b1d4"
   integrity sha512-7LXwnKyqcEaMFVXOer+2JPfFs1D+ej7yRRrfZoIH1YlLQZ81OvBNwSCBBLtExVkoMQQgOWwO0FbZVge6U/8rhQ==
 
-emitter-listener@^1.0.1:
+emitter-listener@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8"
   integrity sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==
@@ -5759,7 +5743,7 @@ semver@7.x, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5:
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^5.4.1, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
+semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
@@ -5940,11 +5924,6 @@ sshpk@^1.7.0:
     safer-buffer "^2.0.2"
     tweetnacl "~0.14.0"
 
-stack-chain@^1.3.7:
-  version "1.3.7"
-  resolved "https://registry.yarnpkg.com/stack-chain/-/stack-chain-1.3.7.tgz#d192c9ff4ea6a22c94c4dd459171e3f00cea1285"
-  integrity sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=
-
 stack-utils@^2.0.3:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.5.tgz#d25265fca995154659dbbfba3b49254778d2fdd5"

yarn.lock

@@ -972,6 +972,13 @@
     estree-walker "^1.0.1"
     picomatch "^2.2.2"
 
+"@types/bson@*":
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337"
+  integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg==
+  dependencies:
+    bson "*"
+
 "@types/estree@0.0.39":
   version "0.0.39"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
@@ -982,6 +989,19 @@
   resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c"
   integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==
 
+"@types/mongodb@3.6.3":
+  version "3.6.3"
+  resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.3.tgz#5655af409d9e32d5d5ae9a653abf3e5f9c83eb7a"
+  integrity sha512-6YNqGP1hk5bjUFaim+QoFFuI61WjHiHE1BNeB41TA00Xd2K7zG4lcWyLLq/XtIp36uMavvS5hoAUJ+1u/GcX2Q==
+  dependencies:
+    "@types/bson" "*"
+    "@types/node" "*"
+
+"@types/node@*":
+  version "17.0.33"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.33.tgz#3c1879b276dc63e73030bb91165e62a4509cd506"
+  integrity sha512-miWq2m2FiQZmaHfdZNcbpp9PuXg34W5JZ5CrJ/BaS70VuhoJENBEQybeiYSaPBRNq6KQGnjfEnc/F3PN++D+XQ==
+
 "@types/node@>= 8":
   version "17.0.18"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.18.tgz#3b4fed5cfb58010e3a2be4b6e74615e4847f1074"
@@ -1300,6 +1320,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
+base64-js@^1.3.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
 base@^0.11.1:
   version "0.11.2"
   resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f"
@@ -1361,6 +1386,13 @@ braces@^3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
+bson@*:
+  version "4.6.3"
+  resolved "https://registry.yarnpkg.com/bson/-/bson-4.6.3.tgz#d1a9a0b84b9e84b62390811fc5580f6a8b1d858c"
+  integrity sha512-rAqP5hcUVJhXP2MCSNVsf0oM2OGU1So6A9pVRDYayvJ5+hygXHQApf87wd5NlhPM1J9RJnbqxIG/f8QTzRoQ4A==
+  dependencies:
+    buffer "^5.6.0"
+
 btoa-lite@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337"
@@ -1371,6 +1403,14 @@ buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
   integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
 
+buffer@^5.6.0:
+  version "5.7.1"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
+  integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.1.13"
+
 builtins@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88"
@@ -2969,6 +3009,11 @@ iconv-lite@^0.6.2:
   dependencies:
     safer-buffer ">= 2.1.2 < 3.0.0"
 
+ieee754@^1.1.13:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
+  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
+
 iferr@^0.1.5:
   version "0.1.5"
   resolved "https://registry.yarnpkg.com/iferr/-/iferr-0.1.5.tgz#c60eed69e6d8fdb6b3104a1fcbca1c192dc5b501"
@@ -4663,12 +4708,7 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
-picomatch@^2.2.2:
+picomatch@^2.2.2, picomatch@^2.3.1:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
-  integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
-
-picomatch@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==