Merge branch 'master' into contrib-expose-fetchData-in-SDK

.eslintrc.json

@@ -45,6 +45,16 @@
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error"
       }
+    },
+    {
+      "files": [
+        "packages/builder/**/*",
+        "packages/client/**/*",
+        "packages/frontend-core/**/*"
+      ],
+      "rules": {
+        "no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
+      }
     }
   ],
   "rules": {

README.md

@@ -11,7 +11,7 @@
   The low code platform you'll enjoy using
 </h3>
 <p align="center">
-  Budibase is an open source low-code platform, and the easiest way to build internal apps that improve productivity.
+  Budibase is an open-source low-code platform that saves engineers 100s of hours building forms, portals, and approval apps, securely.
 </p>
 
 <h3 align="center">
@@ -20,7 +20,7 @@
 <br>
 
 <p align="center">
-  <img alt="Budibase design ui" src="https://res.cloudinary.com/daog6scxm/image/upload/v1633524049/ui/design-ui-wide-mobile_gdaveq.jpg">
+  <img alt="Budibase design ui" src="https://res.cloudinary.com/daog6scxm/image/upload/v1680181644/ui/homepage-design-ui_sizp7b.png">
 </p>
 
 <p align="center">
@@ -57,7 +57,7 @@
 ## ✨ Features
 
 ### Build and ship real software 
-Unlike other platforms, with Budibase you build and ship single page applications. Budibase applications have performance baked in and can be designed responsively, providing your users with a great experience.
+Unlike other platforms, with Budibase you build and ship single page applications. Budibase applications have performance baked in and can be designed responsively, providing users with a great experience.
 <br /><br />
 
 ### Open source and extensible
@@ -65,40 +65,36 @@ Budibase is open-source - licensed as GPL v3. This should fill you with confiden
 <br /><br />
 
 ### Load data or start from scratch
-Budibase pulls in data from multiple sources, including MongoDB, CouchDB, PostgreSQL, MySQL, Airtable, S3, DynamoDB, or a REST API. And unlike other platforms, with Budibase you can start from scratch and create business apps with no datasources. [Request new datasources](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
+Budibase pulls data from multiple sources, including MongoDB, CouchDB, PostgreSQL, MySQL, Airtable, S3, DynamoDB, or a REST API. And unlike other platforms, with Budibase you can start from scratch and create business apps with no data sources. [Request new datasources](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
 
 <p align="center">
-  <img alt="Budibase data" src="https://res.cloudinary.com/daog6scxm/image/upload/v1636970242/Out%20of%20beta%20launch/data_n1tlhf.png">
+  <img alt="Budibase data" src="https://res.cloudinary.com/daog6scxm/image/upload/v1680281798/ui/data_klbuna.png">
 </p>
 <br /><br />
 
 ### Design and build apps with powerful pre-made components
 
-Budibase comes out of the box with beautifully designed, powerful components which you can use like building blocks to build your UI. We also expose a lot of your favourite CSS styling options so you can go that extra creative mile. [Request new component](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
+Budibase comes out of the box with beautifully designed, powerful components which you can use like building blocks to build your UI. We also expose many of your favourite CSS styling options so you can go that extra creative mile. [Request new component](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
 
 <p align="center">
-  <img alt="Budibase design" src="https://res.cloudinary.com/daog6scxm/image/upload/v1636970243/Out%20of%20beta%20launch/design-like-a-pro_qhlfeu.gif">
+  <img alt="Budibase design" src="https://res.cloudinary.com/daog6scxm/image/upload/v1675437167/ui/form_2x_mbli8y.png">
 </p>
 <br /><br />
 
-### Automate processes, integrate with other tools, and connect to webhooks
+### Automate processes, integrate with other tools and connect to webhooks
-Save time by automating manual processes and workflows. From connecting to webhooks, to automating emails, simply tell Budibase what to do and let it work for you. You can easily [create new automations for Budibase here](https://github.com/Budibase/automations) or [Request new automation](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
+Save time by automating manual processes and workflows. From connecting to webhooks to automating emails, simply tell Budibase what to do and let it work for you. You can easily [create new automations for Budibase here](https://github.com/Budibase/automations) or [Request new automation](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
-
-<p align="center">
-  <img alt="Budibase automations" src="https://res.cloudinary.com/daog6scxm/image/upload/v1636970486/Out%20of%20beta%20launch/automation_riro7u.png">
-</p>
 <br /><br />
 
 ### Integrate with your favorite tools
 Budibase integrates with a number of popular tools allowing you to build apps that perfectly fit your stack.
 
 <p align="center">
-  <img alt="Budibase integrations" src="https://res.cloudinary.com/daog6scxm/image/upload/v1636970242/Out%20of%20beta%20launch/integrations_kc7dqt.png">
+  <img alt="Budibase integrations" src="https://res.cloudinary.com/daog6scxm/image/upload/v1680195228/ui/automate_fg9z07.png">
 </p>
 <br /><br />
 
-### Admin paradise
+### Deploy with confidence and security
-Budibase is made to scale. With Budibase, you can self-host on your own infrastructure and globally manage users, onboarding, SMTP, apps, groups, theming and more. You can also provide users/groups with an app portal and disseminate user-management to the group manager.
+Budibase is made to scale. With Budibase, you can self-host on your own infrastructure and globally manage users, onboarding, SMTP, apps, groups, theming and more. You can also provide users/groups with an app portal and disseminate user management to the group manager.
 
 - Checkout the promo video: https://youtu.be/xoljVpty_Kw
 
@@ -119,17 +115,14 @@ As with anything that we build in Budibase, our new public API is simple to use,
 #### Docs
 You can learn more about the Budibase API at the following places:
 
-- [General documentation](https://docs.budibase.com/docs/public-api) : Learn how to get your API key, how to use spec, and how to use with Postman
+- [General documentation](https://docs.budibase.com/docs/public-api): Learn how to get your API key, how to use spec, and how to use Postman
 - [Interactive API documentation](https://docs.budibase.com/reference/post_applications) : Learn how to interact with the API
 
-#### Guides
+<br /><br />
-
-- [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/)
 
 ## 🏁 Get started
 
-Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean.
+Deploy Budibase using Docker, Kubernetes, and Digital Ocean on your existing infrastructure. Or use Budibase Cloud if you don't need to self-host and would like to get started quickly.
-Or use Budibase Cloud if you don't need to self-host, and would like to get started quickly.
 
 ### [Get started with self-hosting Budibase](https://docs.budibase.com/docs/hosting-methods)
 
@@ -162,7 +155,7 @@ If you have a question or would like to talk with other Budibase users and join
 
 ## ❗ Code of conduct
 
-Budibase is dedicated to providing a welcoming, diverse, and harrassment-free experience for everyone. We expect everyone in the Budibase community to abide by our [**Code of Conduct**](https://github.com/Budibase/budibase/blob/HEAD/docs/CODE_OF_CONDUCT.md). Please read it.
+Budibase is dedicated to providing everyone a welcoming, diverse, and harassment-free experience. We expect everyone in the Budibase community to abide by our [**Code of Conduct**](https://github.com/Budibase/budibase/blob/HEAD/docs/CODE_OF_CONDUCT.md). Please read it.
 <br />
 
 
@@ -171,16 +164,16 @@ Budibase is dedicated to providing a welcoming, diverse, and harrassment-free ex
 
 ## 🙌 Contributing to Budibase
 
-From opening a bug report to creating a pull request: every contribution is appreciated and welcomed. If you're planning to implement a new feature or change the API please create an issue first. This way we can ensure your work is not in vain.
+From opening a bug report to creating a pull request: every contribution is appreciated and welcomed. If you're planning to implement a new feature or change the API, please create an issue first. This way, we can ensure your work is not in vain.
-Environment setup instructions are available for [Debian](https://github.com/Budibase/budibase/tree/HEAD/docs/DEV-SETUP-DEBIAN.md) and [MacOSX](https://github.com/Budibase/budibase/tree/HEAD/docs/DEV-SETUP-MACOSX.md)
+Environment setup instructions are available [here](https://github.com/Budibase/budibase/tree/HEAD/docs/CONTRIBUTING.md).
 
 ### Not Sure Where to Start?
-A good place to start contributing, is the [First time issues project](https://github.com/Budibase/budibase/projects/22).
+A good place to start contributing is the [First time issues project](https://github.com/Budibase/budibase/projects/22).
 
 ### How the repository is organized
 Budibase is a monorepo managed by lerna. Lerna manages the building and publishing of the budibase packages. At a high level, here are the packages that make up Budibase.
 
-- [packages/builder](https://github.com/Budibase/budibase/tree/HEAD/packages/builder) - contains code for the budibase builder client side svelte application.
+- [packages/builder](https://github.com/Budibase/budibase/tree/HEAD/packages/builder) - contains code for the budibase builder client-side svelte application.
 
 - [packages/client](https://github.com/Budibase/budibase/tree/HEAD/packages/client) - A module that runs in the browser responsible for reading JSON definition and creating living, breathing web apps from it.
 
@@ -193,7 +186,7 @@ For more information, see [CONTRIBUTING.md](https://github.com/Budibase/budibase
 
 ## 📝 License
 
-Budibase is open-source, licensed as [GPL v3](https://www.gnu.org/licenses/gpl-3.0.en.html). The client and component libraries are licensed as [MPL](https://directory.fsf.org/wiki/License:MPL-2.0) - so the apps that you build can be licensed however you like.
+Budibase is open-source, licensed as [GPL v3](https://www.gnu.org/licenses/gpl-3.0.en.html). The client and component libraries are licensed as [MPL](https://directory.fsf.org/wiki/License:MPL-2.0) - so the apps you build can be licensed however you like.
 
 <br /><br />
 

docs/DEV-SETUP-DEBIAN.md

@@ -1,76 +0,0 @@
-## Dev Environment on Debian 11
-
-### Install NVM & Node 14
-
-NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
-
-Install NVM
-
-```
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
-```
-
-Install Node 14
-
-```
-nvm install 14
-```
-
-### Install npm requirements
-
-```
-npm install -g yarn jest lerna
-```
-
-### Install Docker and Docker Compose
-
-```
-apt install docker.io
-pip3 install docker-compose
-```
-
-### Clone the repo
-
-```
-git clone https://github.com/Budibase/budibase.git
-```
-
-### Check Versions
-
-This setup process was tested on Debian 11 (bullseye) with version numbers show below. Your mileage may vary using anything else.
-
-- Docker: 20.10.5
-- Docker-Compose: 1.29.2
-- Node: v14.20.1
-- Yarn: 1.22.19
-- Lerna: 5.1.4
-
-### Build
-
-```
-cd budibase
-yarn setup
-```
-
-The yarn setup command runs several build steps i.e.
-
-```
-node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev
-```
-
-So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
-
-The dev version will be available on port 10000 i.e.
-
-http://127.0.0.1:10000/builder/admin
-
-### File descriptor issues with Vite and Chrome in Linux
-
-If your dev environment stalls forever, with some network requests stuck in flight, it's likely that Chrome is trying to open more file descriptors than your system allows.
-To fix this, apply the following tweaks.
-
-Debian based distros:
-Add `* - nofile 65536` to `/etc/security/limits.conf`.
-
-Arch:
-Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`.

docs/DEV-SETUP-MACOSX.md

@@ -1,84 +0,0 @@
-## Dev Environment on MAC OSX 12 (Monterey)
-
-### Install Homebrew
-
-Install instructions [here](https://brew.sh/)
-
-| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add
-`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install
-through brew.
-
-### Install Node
-
-Budibase requires a recent version of node 14:
-
-```
-brew install node npm
-node -v
-```
-
-### Install npm requirements
-
-```
-npm install -g yarn jest lerna
-```
-
-### Install Docker and Docker Compose
-
-```
-brew install docker docker-compose
-```
-
-### Clone the repo
-
-```
-git clone https://github.com/Budibase/budibase.git
-```
-
-### Check Versions
-
-This setup process was tested on Mac OSX 12 (Monterey) with version numbers shown below. Your mileage may vary using anything else.
-
-- Docker: 20.10.14
-- Docker-Compose: 2.6.0
-- Node: 14.20.1
-- Yarn: 1.22.19
-- Lerna: 5.1.4
-
-### Build
-
-```
-cd budibase
-yarn setup
-```
-
-The yarn setup command runs several build steps i.e.
-
-```
-node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev
-```
-
-So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
-
-The dev version will be available on port 10000 i.e.
-
-http://127.0.0.1:10000/builder/admin
-
-| **NOTE**: If you are working on a M1 Apple Silicon, you will need to uncomment `# platform: linux/amd64` line in
-[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
-
-### Troubleshootings
-
-#### Yarn setup errors
-
-If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11.
-
-#### Node 14.20.1 not supported for arm64
-
-If you are working with M1 or M2 Mac and trying the Node installation via `nvm`, probably you will find the error `curl: (22) The requested URL returned error: 404`.
-
-Version `v14.20.1` is not supported for arm64; in order to use it, you can switch the CPU architecture for this by the following command:
-
-```shell
-arch -x86_64 zsh #Run this before nvm install
-```

docs/DEV-SETUP-WINDOWS.md

@@ -1,92 +0,0 @@
-## Dev Environment on Windows 10/11 (WSL2)
-
-### Install WSL with Ubuntu LTS
-
-Enable WSL 2 on Windows 10/11 for docker support.
-
-```
-wsl --set-default-version 2
-```
-
-Install Ubuntu LTS.
-
-```
-wsl --install Ubuntu
-```
-
-Or follow the instruction here:
-https://learn.microsoft.com/en-us/windows/wsl/install
-
-### Install Docker in windows
-
-Download the installer from docker and install it.
-
-Check this url for more detailed instructions:
-https://docs.docker.com/desktop/install/windows-install/
-
-You should follow the next steps from within the Ubuntu terminal.
-
-### Install NVM & Node 14
-
-NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
-
-Install NVM
-
-```
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
-```
-
-Install Node 14
-
-```
-nvm install 14
-```
-
-### Install npm requirements
-
-```
-npm install -g yarn jest lerna
-```
-
-### Clone the repo
-
-```
-git clone https://github.com/Budibase/budibase.git
-```
-
-### Check Versions
-
-This setup process was tested on Windows 11 with version numbers show below. Your mileage may vary using anything else.
-
-- Docker: 20.10.7
-- Docker-Compose: 2.10.2
-- Node: v14.20.1
-- Yarn: 1.22.19
-- Lerna: 5.5.4
-
-### Build
-
-```
-cd budibase
-yarn setup
-```
-
-The yarn setup command runs several build steps i.e.
-
-```
-node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev
-```
-
-So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
-
-The dev version will be available on port 10000 i.e.
-
-http://127.0.0.1:10000/builder/admin
-
-### Working with the code
-
-Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine.
-
-https://code.visualstudio.com/docs/remote/wsl
-
-Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows.

i18n/README.es.md

@@ -207,8 +207,7 @@ Desde comunicar un bug a solventar un error en el codigo, toda contribucion es a
 implementar una nueva funcionalidad o un realizar un cambio en la API, por favor crea un [nuevo mensaje aqui](https://github.com/Budibase/budibase/issues),
 de esta manera nos encargaremos que tu trabajo no sea en vano.
 
-Aqui tienes instrucciones de como configurar tu entorno Budibase para [Debian](https://github.com/Budibase/budibase/tree/HEAD/docs/DEV-SETUP-DEBIAN.md) 
+Aqui tienes instrucciones de como configurar tu entorno Budibase para [aquí](https://github.com/Budibase/budibase/tree/HEAD/docs/CONTRIBUTING.md).
-y [MacOSX](https://github.com/Budibase/budibase/tree/HEAD/docs/DEV-SETUP-MACOSX.md)
 
 ### No estas seguro por donde empezar?
 Un buen lugar para empezar a contribuir con nosotros es [aqui](https://github.com/Budibase/budibase/projects/22).

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.15.7",
+  "version": "2.16.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -46,7 +46,7 @@
     "kill-accountportal": "kill-port 3001 4003",
     "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal",
     "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
-    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
     "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
     "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
     "dev:all": "yarn run kill-all && lerna run --stream dev",

packages/account-portal

@@ -1 +1 @@
-Subproject commit dd9cec22751405e042ba0fe58e3c05f7223c3723
+Subproject commit 485ec16a9eed48c548a5f1239772139f3319f028

packages/backend-core/package.json

@@ -21,7 +21,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",

packages/backend-core/src/objectStore/cloudfront.ts

@@ -23,7 +23,7 @@ const getCloudfrontSignParams = () => {
   return {
     keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!,
     privateKeyString: getPrivateKey(),
-    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+    expireTime: new Date().getTime() + 1000 * 60 * 60 * 24, // 1 day
   }
 }
 

packages/builder/src/builderStore/componentUtils.js

@@ -92,7 +92,14 @@ export const findAllMatchingComponents = (rootComponent, selector) => {
 }
 
 /**
- * Finds the closes parent component which matches certain criteria
+ * Recurses through the component tree and finds all components.
+ */
+export const findAllComponents = rootComponent => {
+  return findAllMatchingComponents(rootComponent, () => true)
+}
+
+/**
+ * Finds the closest parent component which matches certain criteria
  */
 export const findClosestMatchingComponent = (
   rootComponent,

packages/builder/src/builderStore/dataBinding.js

@@ -1,6 +1,7 @@
 import { cloneDeep } from "lodash/fp"
 import { get } from "svelte/store"
 import {
+  findAllComponents,
   findAllMatchingComponents,
   findComponent,
   findComponentPath,
@@ -102,6 +103,9 @@ export const getAuthBindings = () => {
   return bindings
 }
 
+/**
+ * Gets all bindings for environment variables
+ */
 export const getEnvironmentBindings = () => {
   let envVars = get(environment).variables
   return envVars.map(variable => {
@@ -130,26 +134,22 @@ export const toBindingsArray = (valueMap, prefix, category) => {
     if (!binding) {
       return acc
     }
-
     let config = {
       type: "context",
       runtimeBinding: binding,
       readableBinding: `${prefix}.${binding}`,
       icon: "Brackets",
     }
-
     if (category) {
       config.category = category
     }
-
     acc.push(config)
-
     return acc
   }, [])
 }
 
 /**
- * Utility - coverting a map of readable bindings to runtime
+ * Utility to covert a map of readable bindings to runtime
  */
 export const readableToRuntimeMap = (bindings, ctx) => {
   if (!bindings || !ctx) {
@@ -162,7 +162,7 @@ export const readableToRuntimeMap = (bindings, ctx) => {
 }
 
 /**
- * Utility - coverting a map of runtime bindings to readable
+ * Utility to covert a map of runtime bindings to readable bindings
  */
 export const runtimeToReadableMap = (bindings, ctx) => {
   if (!bindings || !ctx) {
@@ -188,15 +188,23 @@ export const getComponentBindableProperties = (asset, componentId) => {
   if (!def?.context) {
     return []
   }
+  const contexts = Array.isArray(def.context) ? def.context : [def.context]
 
   // Get the bindings for the component
-  return getProviderContextBindings(asset, component)
+  const componentContext = {
+    component,
+    definition: def,
+    contexts,
+  }
+  return generateComponentContextBindings(asset, componentContext)
 }
 
 /**
- * Gets all data provider components above a component.
+ * Gets all component contexts available to a certain component. This handles
+ * both global and local bindings, taking into account a component's position
+ * in the component tree.
  */
-export const getContextProviderComponents = (
+export const getComponentContexts = (
   asset,
   componentId,
   type,
@@ -205,30 +213,55 @@ export const getContextProviderComponents = (
   if (!asset || !componentId) {
     return []
   }
+  let map = {}
 
-  // Get the component tree leading up to this component, ignoring the component
+  // Processes all contexts exposed by a component
-  // itself
+  const processContexts = scope => component => {
-  const path = findComponentPath(asset.props, componentId)
-  if (!options?.includeSelf) {
-    path.pop()
-  }
-
-  // Filter by only data provider components
-  return path.filter(component => {
     const def = store.actions.components.getDefinition(component._component)
     if (!def?.context) {
-      return false
+      return
     }
-
+    if (!map[component._id]) {
-    // If no type specified, return anything that exposes context
+      map[component._id] = {
-    if (!type) {
+        component,
-      return true
+        definition: def,
+        contexts: [],
+      }
     }
-
-    // Otherwise only match components with the specific context type
     const contexts = Array.isArray(def.context) ? def.context : [def.context]
-    return contexts.find(context => context.type === type) != null
+    contexts.forEach(context => {
-  })
+      // Ensure type matches
+      if (type && context.type !== type) {
+        return
+      }
+      // Ensure scope matches
+      let contextScope = context.scope || "global"
+      if (contextScope !== scope) {
+        return
+      }
+      // Ensure the context is compatible with the component's current settings
+      if (!isContextCompatibleWithComponent(context, component)) {
+        return
+      }
+      map[component._id].contexts.push(context)
+    })
+  }
+
+  // Process all global contexts
+  const allComponents = findAllComponents(asset.props)
+  allComponents.forEach(processContexts("global"))
+
+  // Process all local contexts
+  const localComponents = findComponentPath(asset.props, componentId)
+  localComponents.forEach(processContexts("local"))
+
+  // Exclude self if required
+  if (!options?.includeSelf) {
+    delete map[componentId]
+  }
+
+  // Only return components which provide at least 1 matching context
+  return Object.values(map).filter(x => x.contexts.length > 0)
 }
 
 /**
@@ -240,20 +273,19 @@ export const getActionProviders = (
   actionType,
   options = { includeSelf: false }
 ) => {
-  if (!asset || !componentId) {
+  if (!asset) {
     return []
   }
 
-  // Get the component tree leading up to this component, ignoring the component
+  // Get all components
-  // itself
+  const components = findAllComponents(asset.props)
-  const path = findComponentPath(asset.props, componentId)
-  if (!options?.includeSelf) {
-    path.pop()
-  }
 
   // Find matching contexts and generate bindings
   let providers = []
-  path.forEach(component => {
+  components.forEach(component => {
+    if (!options?.includeSelf && component._id === componentId) {
+      return
+    }
     const def = store.actions.components.getDefinition(component._component)
     const actions = (def?.actions || []).map(action => {
       return typeof action === "string" ? { type: action } : action
@@ -317,142 +349,131 @@ export const getDatasourceForProvider = (asset, component) => {
  * Gets all bindable data properties from component data contexts.
  */
 const getContextBindings = (asset, componentId) => {
-  // Extract any components which provide data contexts
+  // Get all available contexts for this component
-  const dataProviders = getContextProviderComponents(asset, componentId)
+  const componentContexts = getComponentContexts(asset, componentId)
 
-  // Generate bindings for all matching components
+  // Generate bindings for each context
-  return getProviderContextBindings(asset, dataProviders)
+  return componentContexts
+    .map(componentContext => {
+      return generateComponentContextBindings(asset, componentContext)
+    })
+    .flat()
 }
 
 /**
- * Gets the context bindings exposed by a set of data provider components.
+ * Generates a set of bindings for a given component context
  */
-const getProviderContextBindings = (asset, dataProviders) => {
+const generateComponentContextBindings = (asset, componentContext) => {
-  if (!asset || !dataProviders) {
+  const { component, definition, contexts } = componentContext
+  if (!component || !definition || !contexts?.length) {
     return []
   }
 
-  // Ensure providers is an array
-  if (!Array.isArray(dataProviders)) {
-    dataProviders = [dataProviders]
-  }
-
   // Create bindings for each data provider
   let bindings = []
-  dataProviders.forEach(component => {
+  contexts.forEach(context => {
-    const def = store.actions.components.getDefinition(component._component)
+    if (!context?.type) {
-    const contexts = Array.isArray(def.context) ? def.context : [def.context]
+      return
+    }
 
-    // Create bindings for each context block provided by this data provider
+    let schema
-    contexts.forEach(context => {
+    let table
-      if (!context?.type) {
+    let readablePrefix
+    let runtimeSuffix = context.suffix
+
+    if (context.type === "form") {
+      // Forms do not need table schemas
+      // Their schemas are built from their component field names
+      schema = buildFormSchema(component, asset)
+      readablePrefix = "Fields"
+    } else if (context.type === "static") {
+      // Static contexts are fully defined by the components
+      schema = {}
+      const values = context.values || []
+      values.forEach(value => {
+        schema[value.key] = {
+          name: value.label,
+          type: value.type || "string",
+        }
+      })
+    } else if (context.type === "schema") {
+      // Schema contexts are generated dynamically depending on their data
+      const datasource = getDatasourceForProvider(asset, component)
+      if (!datasource) {
         return
       }
+      const info = getSchemaForDatasource(asset, datasource)
+      schema = info.schema
+      table = info.table
 
-      let schema
+      // Determine what to prefix bindings with
-      let table
+      if (datasource.type === "jsonarray") {
-      let readablePrefix
+        // For JSON arrays, use the array name as the readable prefix
-      let runtimeSuffix = context.suffix
+        const split = datasource.label.split(".")
-
+        readablePrefix = split[split.length - 1]
-      if (context.type === "form") {
+      } else if (datasource.type === "viewV2") {
-        // Forms do not need table schemas
+        // For views, use the view name
-        // Their schemas are built from their component field names
+        const view = Object.values(table?.views || {}).find(
-        schema = buildFormSchema(component, asset)
+          view => view.id === datasource.id
-        readablePrefix = "Fields"
-      } else if (context.type === "static") {
-        // Static contexts are fully defined by the components
-        schema = {}
-        const values = context.values || []
-        values.forEach(value => {
-          schema[value.key] = {
-            name: value.label,
-            type: value.type || "string",
-          }
-        })
-      } else if (context.type === "schema") {
-        // Schema contexts are generated dynamically depending on their data
-        const datasource = getDatasourceForProvider(asset, component)
-        if (!datasource) {
-          return
-        }
-        const info = getSchemaForDatasource(asset, datasource)
-        schema = info.schema
-        table = info.table
-
-        // Determine what to prefix bindings with
-        if (datasource.type === "jsonarray") {
-          // For JSON arrays, use the array name as the readable prefix
-          const split = datasource.label.split(".")
-          readablePrefix = split[split.length - 1]
-        } else if (datasource.type === "viewV2") {
-          // For views, use the view name
-          const view = Object.values(table?.views || {}).find(
-            view => view.id === datasource.id
-          )
-          readablePrefix = view?.name
-        } else {
-          // Otherwise use the table name
-          readablePrefix = info.table?.name
-        }
-      }
-      if (!schema) {
-        return
-      }
-
-      const keys = Object.keys(schema).sort()
-
-      // Generate safe unique runtime prefix
-      let providerId = component._id
-      if (runtimeSuffix) {
-        providerId += `-${runtimeSuffix}`
-      }
-
-      if (!filterCategoryByContext(component, context)) {
-        return
-      }
-
-      const safeComponentId = makePropSafe(providerId)
-
-      // Create bindable properties for each schema field
-      keys.forEach(key => {
-        const fieldSchema = schema[key]
-
-        // Make safe runtime binding
-        const safeKey = key.split(".").map(makePropSafe).join(".")
-        const runtimeBinding = `${safeComponentId}.${safeKey}`
-
-        // Optionally use a prefix with readable bindings
-        let readableBinding = component._instanceName
-        if (readablePrefix) {
-          readableBinding += `.${readablePrefix}`
-        }
-        readableBinding += `.${fieldSchema.name || key}`
-
-        const bindingCategory = getComponentBindingCategory(
-          component,
-          context,
-          def
         )
+        readablePrefix = view?.name
+      } else {
+        // Otherwise use the table name
+        readablePrefix = info.table?.name
+      }
+    }
+    if (!schema) {
+      return
+    }
 
-        // Create the binding object
+    const keys = Object.keys(schema).sort()
-        bindings.push({
+
-          type: "context",
+    // Generate safe unique runtime prefix
-          runtimeBinding,
+    let providerId = component._id
-          readableBinding,
+    if (runtimeSuffix) {
-          // Field schema and provider are required to construct relationship
+      providerId += `-${runtimeSuffix}`
-          // datasource options, based on bindable properties
+    }
-          fieldSchema,
+    const safeComponentId = makePropSafe(providerId)
-          providerId,
+
-          // Table ID is used by JSON fields to know what table the field is in
+    // Create bindable properties for each schema field
-          tableId: table?._id,
+    keys.forEach(key => {
-          component: component._component,
+      const fieldSchema = schema[key]
-          category: bindingCategory.category,
+
-          icon: bindingCategory.icon,
+      // Make safe runtime binding
-          display: {
+      const safeKey = key.split(".").map(makePropSafe).join(".")
-            name: fieldSchema.name || key,
+      const runtimeBinding = `${safeComponentId}.${safeKey}`
-            type: fieldSchema.type,
+
-          },
+      // Optionally use a prefix with readable bindings
-        })
+      let readableBinding = component._instanceName
+      if (readablePrefix) {
+        readableBinding += `.${readablePrefix}`
+      }
+      readableBinding += `.${fieldSchema.name || key}`
+
+      // Determine which category this binding belongs in
+      const bindingCategory = getComponentBindingCategory(
+        component,
+        context,
+        definition
+      )
+      // Create the binding object
+      bindings.push({
+        type: "context",
+        runtimeBinding,
+        readableBinding: `${readableBinding}`,
+        // Field schema and provider are required to construct relationship
+        // datasource options, based on bindable properties
+        fieldSchema,
+        providerId,
+        // Table ID is used by JSON fields to know what table the field is in
+        tableId: table?._id,
+        component: component._component,
+        category: bindingCategory.category,
+        icon: bindingCategory.icon,
+        display: {
+          name: `${fieldSchema.name || key}`,
+          type: fieldSchema.type,
+        },
       })
     })
   })
@@ -460,25 +481,38 @@ const getProviderContextBindings = (asset, dataProviders) => {
   return bindings
 }
 
-// Exclude a data context based on the component settings
+/**
-const filterCategoryByContext = (component, context) => {
+ * Checks if a certain data context is compatible with a certain instance of a
-  const { _component } = component
+ * configured component.
+ */
+const isContextCompatibleWithComponent = (context, component) => {
+  if (!component) {
+    return false
+  }
+  const { _component, actionType } = component
+  const { type } = context
+
+  // Certain types of form blocks only allow certain contexts
   if (_component.endsWith("formblock")) {
     if (
-      (component.actionType === "Create" && context.type === "schema") ||
+      (actionType === "Create" && type === "schema") ||
-      (component.actionType === "View" && context.type === "form")
+      (actionType === "View" && type === "form")
     ) {
       return false
     }
   }
+
+  // Allow the context by default
   return true
 }
 
 // Enrich binding category information for certain components
 const getComponentBindingCategory = (component, context, def) => {
+  // Default category to component name
   let icon = def.icon
   let category = component._instanceName
 
+  // Form block edge case
   if (component._component.endsWith("formblock")) {
     if (context.type === "form") {
       category = `${component._instanceName} - Fields`
@@ -496,7 +530,7 @@ const getComponentBindingCategory = (component, context, def) => {
 }
 
 /**
- * Gets all bindable properties from the logged in user.
+ * Gets all bindable properties from the logged-in user.
  */
 export const getUserBindings = () => {
   let bindings = []
@@ -566,6 +600,7 @@ const getDeviceBindings = () => {
 
 /**
  * Gets all selected rows bindings for tables in the current asset.
+ * TODO: remove in future because we don't need a separate store for this
  */
 const getSelectedRowsBindings = asset => {
   let bindings = []
@@ -608,6 +643,9 @@ const getSelectedRowsBindings = asset => {
   return bindings
 }
 
+/**
+ * Generates a state binding for a certain key name
+ */
 export const makeStateBinding = key => {
   return {
     type: "context",
@@ -662,6 +700,9 @@ const getUrlBindings = asset => {
   return urlParamBindings.concat([queryParamsBinding])
 }
 
+/**
+ * Generates all bindings for role IDs
+ */
 const getRoleBindings = () => {
   return (get(rolesStore) || []).map(role => {
     return {

packages/builder/src/builderStore/store/frontend.js

@@ -706,10 +706,9 @@ export const getFrontendStore = () => {
           else {
             if (setting.type === "dataProvider") {
               // Validate data provider exists, or else clear it
-              const treeId = parent?._id || component._id
+              const providers = findAllMatchingComponents(
-              const path = findComponentPath(screen?.props, treeId)
+                screen?.props,
-              const providers = path.filter(component =>
+                component => component._component?.endsWith("/dataprovider")
-                component._component?.endsWith("/dataprovider")
               )
               // Validate non-empty values
               const valid = providers?.some(dp => value.includes?.(dp._id))
@@ -731,6 +730,16 @@ export const getFrontendStore = () => {
           return null
         }
 
+        // Find all existing components of this type so that we can give this
+        // component a unique name
+        const screen = get(selectedScreen).props
+        const otherComponents = findAllMatchingComponents(
+          screen,
+          x => x._component === definition.component && x._id !== screen._id
+        )
+        let name = definition.friendlyName || definition.name
+        name = `${name} ${otherComponents.length + 1}`
+
         // Generate basic component structure
         let instance = {
           _id: Helpers.uuid(),
@@ -740,7 +749,7 @@ export const getFrontendStore = () => {
             hover: {},
             active: {},
           },
-          _instanceName: `New ${definition.friendlyName || definition.name}`,
+          _instanceName: name,
           ...presetProps,
         }
 

packages/builder/src/builderStore/websocket.js

@@ -21,7 +21,7 @@ export const createBuilderWebsocket = appId => {
     })
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to builder websocket:", err.message)
+    console.error("Failed to connect to builder websocket:", err.message)
   })
   socket.on("disconnect", () => {
     userStore.actions.reset()

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -373,6 +373,7 @@
   confirmText="Save"
   onConfirm={saveRelationship}
   disabled={!valid}
+  size="L"
 >
   <div class="headings">
     <Detail>Tables</Detail>

packages/builder/src/components/common/CodeEditor/index.js

@@ -312,7 +312,7 @@ export const insertBinding = (view, from, to, text, mode) => {
   } else if (mode.name == "handlebars") {
     parsedInsert = hbInsert(view.state.doc?.toString(), from, to, text)
   } else {
-    console.log("Unsupported")
+    console.warn("Unsupported")
     return
   }
 

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -17,7 +17,7 @@
 </script>
 
 <div class="relationship-container">
-  <div class="relationship-part">
+  <div class="relationship-type">
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipPart1}
@@ -39,7 +39,7 @@
   </div>
 </div>
 <div class="relationship-container">
-  <div class="relationship-part">
+  <div class="relationship-type">
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipPart2}
@@ -79,6 +79,10 @@
   }
 
   .relationship-part {
-    flex-basis: 60%;
+    flex-basis: 70%;
+  }
+
+  .relationship-type {
+    flex-basis: 30%;
   }
 </style>

packages/builder/src/components/common/bindings/BindingPicker.svelte

@@ -47,9 +47,10 @@
     })
   $: filteredHelpers = helpers?.filter(helper => {
     return (
-      !search ||
+      (!search ||
-      helper.label.match(searchRgx) ||
+        helper.label.match(searchRgx) ||
-      helper.description.match(searchRgx)
+        helper.description.match(searchRgx)) &&
+      (mode.name !== "javascript" || helper.allowsJs)
     )
   })
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/utils.js

@@ -1,5 +1,4 @@
-import { getContextProviderComponents } from "builderStore/dataBinding"
+import { getComponentContexts } from "builderStore/dataBinding"
-import { store } from "builderStore"
 import { capitalise } from "helpers"
 
 // Generates bindings for all components that provider "datasource like"
@@ -8,58 +7,49 @@ import { capitalise } from "helpers"
 // Some examples are saving rows or duplicating rows.
 export const getDatasourceLikeProviders = ({ asset, componentId, nested }) => {
   // Get all form context providers
-  const formComponents = getContextProviderComponents(
+  const formComponentContexts = getComponentContexts(
     asset,
     componentId,
     "form",
-    { includeSelf: nested }
+    {
+      includeSelf: nested,
+    }
   )
-
   // Get all schema context providers
-  const schemaComponents = getContextProviderComponents(
+  const schemaComponentContexts = getComponentContexts(
     asset,
     componentId,
     "schema",
-    { includeSelf: nested }
+    {
+      includeSelf: nested,
+    }
   )
 
-  // Generate contexts for all form providers
-  const formContexts = formComponents.map(component => ({
-    component,
-    context: extractComponentContext(component, "form"),
-  }))
-
-  // Generate contexts for all schema providers
-  const schemaContexts = schemaComponents.map(component => ({
-    component,
-    context: extractComponentContext(component, "schema"),
-  }))
-
   // Check for duplicate contexts by the same component. In this case, attempt
   // to label contexts with their suffixes
-  schemaContexts.forEach(schemaContext => {
+  schemaComponentContexts.forEach(schemaContext => {
     // Check if we have a form context for this component
     const id = schemaContext.component._id
-    const existing = formContexts.find(x => x.component._id === id)
+    const existing = formComponentContexts.find(x => x.component._id === id)
     if (existing) {
-      if (existing.context.suffix) {
+      if (existing.contexts[0].suffix) {
-        const suffix = capitalise(existing.context.suffix)
+        const suffix = capitalise(existing.contexts[0].suffix)
         existing.readableSuffix = ` - ${suffix}`
       }
-      if (schemaContext.context.suffix) {
+      if (schemaContext.contexts[0].suffix) {
-        const suffix = capitalise(schemaContext.context.suffix)
+        const suffix = capitalise(schemaContext.contexts[0].suffix)
         schemaContext.readableSuffix = ` - ${suffix}`
       }
     }
   })
 
   // Generate bindings for all contexts
-  const allContexts = formContexts.concat(schemaContexts)
+  const allContexts = formComponentContexts.concat(schemaComponentContexts)
-  return allContexts.map(({ component, context, readableSuffix }) => {
+  return allContexts.map(({ component, contexts, readableSuffix }) => {
     let readableBinding = component._instanceName
     let runtimeBinding = component._id
-    if (context.suffix) {
+    if (contexts[0].suffix) {
-      runtimeBinding += `-${context.suffix}`
+      runtimeBinding += `-${contexts[0].suffix}`
     }
     if (readableSuffix) {
       readableBinding += readableSuffix
@@ -70,13 +60,3 @@ export const getDatasourceLikeProviders = ({ asset, componentId, nested }) => {
     }
   })
 }
-
-// Gets a context definition of a certain type from a component definition
-const extractComponentContext = (component, contextType) => {
-  const def = store.actions.components.getDefinition(component?._component)
-  if (!def) {
-    return null
-  }
-  const contexts = Array.isArray(def.context) ? def.context : [def.context]
-  return contexts.find(context => context?.type === contextType)
-}

packages/builder/src/components/design/settings/controls/DataProviderSelect.svelte

@@ -1,15 +1,16 @@
 <script>
   import { Select } from "@budibase/bbui"
   import { makePropSafe } from "@budibase/string-templates"
-  import { currentAsset, store } from "builderStore"
+  import { currentAsset } from "builderStore"
-  import { findComponentPath } from "builderStore/componentUtils"
+  import { findAllMatchingComponents } from "builderStore/componentUtils"
 
   export let value
 
   const getValue = component => `{{ literal ${makePropSafe(component._id)} }}`
 
-  $: path = findComponentPath($currentAsset?.props, $store.selectedComponentId)
+  $: providers = findAllMatchingComponents($currentAsset?.props, c =>
-  $: providers = path.filter(c => c._component?.endsWith("/dataprovider"))
+    c._component?.endsWith("/dataprovider")
+  )
 </script>
 
 <Select

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -1,6 +1,5 @@
 <script>
   import {
-    getContextProviderComponents,
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
@@ -30,6 +29,7 @@
   import BindingBuilder from "components/integration/QueryBindingBuilder.svelte"
   import IntegrationQueryEditor from "components/integration/index.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
+  import { findAllComponents } from "builderStore/componentUtils"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import DataSourceCategory from "components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte"
   import { API } from "api"
@@ -75,12 +75,13 @@
       ...query,
       type: "query",
     }))
-  $: contextProviders = getContextProviderComponents(
+  $: dataProviders = findAllComponents($currentAsset.props)
-    $currentAsset,
+    .filter(component => {
-    $store.selectedComponentId
+      return (
-  )
+        component._component?.endsWith("/dataprovider") &&
-  $: dataProviders = contextProviders
+        component._id !== $store.selectedComponentId
-    .filter(component => component._component?.endsWith("/dataprovider"))
+      )
+    })
     .map(provider => ({
       label: provider._instanceName,
       name: provider._instanceName,

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -67,7 +67,7 @@
         }))
         navigateStep(target)
       } else {
-        console.log("Could not retrieve step")
+        console.warn("Could not retrieve step")
       }
     } else {
       if (typeof tourStep.onComplete === "function") {

packages/builder/src/components/portal/onboarding/tourHandler.js

@@ -3,11 +3,11 @@ import { get } from "svelte/store"
 
 const registerNode = async (node, tourStepKey) => {
   if (!node) {
-    console.log("Tour Handler - an anchor node is required")
+    console.warn("Tour Handler - an anchor node is required")
   }
 
   if (!get(store).tourKey) {
-    console.log("Tour Handler - No active tour ", tourStepKey, node)
+    console.error("Tour Handler - No active tour ", tourStepKey, node)
     return
   }
 

packages/builder/src/components/portal/onboarding/tours.js

@@ -45,7 +45,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
         onboarding: false,
       }))
     } catch (e) {
-      console.log("Onboarding failed", e)
+      console.error("Onboarding failed", e)
       return false
     }
     return true

packages/builder/src/constants/completions.js

@@ -1,4 +1,4 @@
-import { getManifest } from "@budibase/string-templates"
+import { getManifest, helpersToRemoveForJs } from "@budibase/string-templates"
 
 export function handlebarsCompletions() {
   const manifest = getManifest()
@@ -11,6 +11,9 @@ export function handlebarsCompletions() {
       label: helperName,
       displayText: helperName,
       description: helperConfig.description,
+      allowsJs:
+        !helperConfig.requiresBlock &&
+        !helpersToRemoveForJs.includes(helperName),
     }))
   )
 }

packages/builder/src/helpers/urlStateSync.js

@@ -52,7 +52,7 @@ export const syncURLToState = options => {
   let cachedPage = get(routify.page)
   let previousParamsHash = null
   let debug = false
-  const log = (...params) => debug && console.log(`[${urlParam}]`, ...params)
+  const log = (...params) => debug && console.debug(`[${urlParam}]`, ...params)
 
   // Navigate to a certain URL
   const gotoUrl = (url, params) => {

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -107,7 +107,7 @@
       return
     }
     if (!prodAppId) {
-      console.log("Application id required")
+      console.error("Application id required")
       return
     }
     await usersFetch.update({

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/GeneralPanel.svelte

@@ -66,7 +66,7 @@
     try {
       await store.actions.screens.updateSetting(get(selectedScreen), key, value)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error saving screen settings")
     }
   }

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -71,7 +71,7 @@
       $goto(`./${screenId}`)
       store.actions.screens.select(screenId)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error creating screens")
     }
   }

packages/builder/src/pages/builder/auth/login.svelte

@@ -31,7 +31,7 @@
   async function login() {
     form.validate()
     if (Object.keys(errors).length > 0) {
-      console.log("errors", errors)
+      console.error("errors", errors)
       return
     }
     try {

packages/client/manifest.json

@@ -573,7 +573,6 @@
     "description": "A configurable data list that attaches to your backend tables.",
     "icon": "JourneyData",
     "illegalChildren": ["section"],
-    "requiredAncestors": ["dataprovider"],
     "hasChildren": true,
     "size": {
       "width": 400,
@@ -711,10 +710,12 @@
     ],
     "context": [
       {
-        "type": "schema"
+        "type": "schema",
+        "scope": "local"
       },
       {
         "type": "static",
+        "scope": "local",
         "values": [
           {
             "label": "Row index",
@@ -1564,7 +1565,6 @@
     "name": "Bar Chart",
     "description": "Bar chart",
     "icon": "GraphBarVertical",
-    "requiredAncestors": ["dataprovider"],
     "size": {
       "width": 600,
       "height": 400
@@ -1727,7 +1727,6 @@
       "width": 600,
       "height": 400
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "text",
@@ -1881,7 +1880,6 @@
       "width": 600,
       "height": 400
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "text",
@@ -2047,7 +2045,6 @@
       "width": 600,
       "height": 400
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "text",
@@ -2177,7 +2174,6 @@
       "width": 600,
       "height": 400
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "text",
@@ -2307,7 +2303,6 @@
       "width": 600,
       "height": 400
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "text",
@@ -4087,7 +4082,6 @@
       "width": 400,
       "height": 320
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "dataProvider",
@@ -4643,7 +4637,6 @@
     "name": "Table",
     "icon": "Table",
     "illegalChildren": ["section"],
-    "requiredAncestors": ["dataprovider"],
     "hasChildren": true,
     "showEmptyState": false,
     "size": {
@@ -4734,7 +4727,6 @@
     "name": "Date Range",
     "icon": "Calendar",
     "styles": ["size"],
-    "requiredAncestors": ["dataprovider"],
     "hasChildren": false,
     "size": {
       "width": 200,
@@ -4842,7 +4834,6 @@
       "width": 100,
       "height": 35
     },
-    "requiredAncestors": ["dataprovider"],
     "settings": [
       {
         "type": "dataProvider",
@@ -5617,7 +5608,38 @@
           }
         ]
       }
-    ]
+    ],
+    "context": {
+      "type": "static",
+      "suffix": "provider",
+      "values": [
+        {
+          "label": "Rows",
+          "key": "rows",
+          "type": "array"
+        },
+        {
+          "label": "Extra Info",
+          "key": "info",
+          "type": "string"
+        },
+        {
+          "label": "Rows Length",
+          "key": "rowsLength",
+          "type": "number"
+        },
+        {
+          "label": "Schema",
+          "key": "schema",
+          "type": "object"
+        },
+        {
+          "label": "Page Number",
+          "key": "pageNumber",
+          "type": "number"
+        }
+      ]
+    }
   },
   "cardsblock": {
     "block": true,
@@ -5796,7 +5818,8 @@
     ],
     "context": {
       "type": "schema",
-      "suffix": "repeater"
+      "suffix": "repeater",
+      "scope": "local"
     }
   },
   "repeaterblock": {
@@ -6020,7 +6043,8 @@
       },
       {
         "type": "schema",
-        "suffix": "repeater"
+        "suffix": "repeater",
+        "scope": "local"
       }
     ]
   },
@@ -6166,6 +6190,10 @@
         "type": "form",
         "suffix": "form"
       },
+      {
+        "type": "schema",
+        "suffix": "repeater"
+      },
       {
         "type": "static",
         "suffix": "form",
@@ -6479,9 +6507,27 @@
     ],
     "context": {
       "type": "schema",
-      "suffix": "repeater"
+      "suffix": "repeater",
+      "scope": "local"
     }
   },
+  "grid": {
+    "name": "Grid",
+    "icon": "ViewGrid",
+    "hasChildren": true,
+    "settings": [
+      {
+        "type": "number",
+        "key": "cols",
+        "label": "Columns"
+      },
+      {
+        "type": "number",
+        "key": "rows",
+        "label": "Rows"
+      }
+    ]
+  },
   "gridblock": {
     "name": "Grid Block",
     "icon": "Table",
@@ -6625,7 +6671,8 @@
       }
     ],
     "context": {
-      "type": "schema"
+      "type": "schema",
+      "scope": "local"
     },
     "actions": ["RefreshDatasource"]
   },

packages/client/src/components/Component.svelte

@@ -9,7 +9,7 @@
 </script>
 
 <script>
-  import { getContext, setContext, onMount, onDestroy } from "svelte"
+  import { getContext, setContext, onMount } from "svelte"
   import { writable, get } from "svelte/store"
   import {
     enrichProps,
@@ -30,6 +30,15 @@
   import ScreenPlaceholder from "components/app/ScreenPlaceholder.svelte"
   import ComponentErrorState from "components/error-states/ComponentErrorState.svelte"
   import { BudibasePrefix } from "../stores/components.js"
+  import {
+    decodeJSBinding,
+    findHBSBlocks,
+    isJSBinding,
+  } from "@budibase/string-templates"
+  import {
+    getActionContextKey,
+    getActionDependentContextKeys,
+  } from "../utils/buttonActions.js"
 
   export let instance = {}
   export let isLayout = false
@@ -81,7 +90,6 @@
 
   // Keep track of stringified representations of context and instance
   // to avoid enriching bindings as much as possible
-  let lastContextKey
   let lastInstanceKey
 
   // Visibility flag used by conditional UI
@@ -98,6 +106,13 @@
   // We clear these whenever a new instance is received.
   let ephemeralStyles
 
+  // Single string of all HBS blocks, used to check if we use a certain binding
+  // or not
+  let bindingString = ""
+
+  // List of context keys which we use inside bindings
+  let knownContextKeyMap = {}
+
   // Set up initial state for each new component instance
   $: initialise(instance)
 
@@ -155,9 +170,6 @@
       hasMissingRequiredSettings)
   $: emptyState = empty && showEmptyState
 
-  // Enrich component settings
-  $: enrichComponentSettings($context, settingsDefinitionMap)
-
   // Evaluate conditional UI settings and store any component setting changes
   // which need to be made
   $: evaluateConditions(conditions)
@@ -206,6 +218,7 @@
     errorState,
     parent: id,
     ancestors: [...($component?.ancestors ?? []), instance._component],
+    path: [...($component?.path ?? []), id],
   })
 
   const initialise = (instance, force = false) => {
@@ -214,7 +227,8 @@
     }
 
     // Ensure we're processing a new instance
-    const instanceKey = Helpers.hashString(JSON.stringify(instance))
+    const stringifiedInstance = JSON.stringify(instance)
+    const instanceKey = Helpers.hashString(stringifiedInstance)
     if (instanceKey === lastInstanceKey && !force) {
       return
     } else {
@@ -274,13 +288,54 @@
       return missing
     })
 
+    // When considering bindings we can ignore children, so we remove that
+    // before storing the reference stringified version
+    const noChildren = JSON.stringify({ ...instance, _children: null })
+    const bindings = findHBSBlocks(noChildren).map(binding => {
+      let sanitizedBinding = binding.replace(/\\"/g, '"')
+      if (isJSBinding(sanitizedBinding)) {
+        return decodeJSBinding(sanitizedBinding)
+      } else {
+        return sanitizedBinding
+      }
+    })
+
+    // The known context key map is built up at runtime, as changes to keys are
+    // encountered. We manually seed this to the required action keys as these
+    // are not encountered at runtime and so need computed in advance.
+    knownContextKeyMap = generateActionKeyMap(instance, settingsDefinition)
+    bindingString = bindings.join(" ")
+
     // Run any migrations
     runMigrations(instance, settingsDefinition)
 
     // Force an initial enrichment of the new settings
-    enrichComponentSettings(get(context), settingsDefinitionMap, {
+    enrichComponentSettings(get(context), settingsDefinitionMap)
-      force: true,
+  }
+
+  // Extracts a map of all context keys which are required by action settings
+  // to provide the functions to evaluate at runtime. This needs done manually
+  // as the action definitions themselves do not specify bindings for action
+  // keys, meaning we cannot do this while doing the other normal bindings.
+  const generateActionKeyMap = (instance, settingsDefinition) => {
+    let map = {}
+    settingsDefinition.forEach(setting => {
+      if (setting.type === "event") {
+        instance[setting.key]?.forEach(action => {
+          // We depend on the actual action key
+          const actionKey = getActionContextKey(action)
+          if (actionKey) {
+            map[actionKey] = true
+          }
+
+          // We also depend on any manually declared context keys
+          getActionDependentContextKeys(action)?.forEach(key => {
+            map[key] = true
+          })
+        })
+      }
     })
+    return map
   }
 
   const runMigrations = (instance, settingsDefinition) => {
@@ -381,17 +436,7 @@
   }
 
   // Enriches any string component props using handlebars
-  const enrichComponentSettings = (
+  const enrichComponentSettings = (context, settingsDefinitionMap) => {
-    context,
-    settingsDefinitionMap,
-    options = { force: false }
-  ) => {
-    const contextChanged = context.key !== lastContextKey
-    if (!contextChanged && !options?.force) {
-      return
-    }
-    lastContextKey = context.key
-
     // Record the timestamp so we can reference it after enrichment
     latestUpdateTime = Date.now()
     const enrichmentTime = latestUpdateTime
@@ -506,31 +551,46 @@
     })
   }
 
+  const handleContextChange = key => {
+    // Check if we already know if this key is used
+    let used = knownContextKeyMap[key]
+
+    // If we don't know, check and cache
+    if (used == null) {
+      used = bindingString.indexOf(`[${key}]`) !== -1
+      knownContextKeyMap[key] = used
+    }
+
+    // Enrich settings if we use this key
+    if (used) {
+      enrichComponentSettings($context, settingsDefinitionMap)
+    }
+  }
+
+  // Register an unregister component instance
   onMount(() => {
-    if (
+    if ($appStore.isDevApp) {
-      $appStore.isDevApp &&
+      if (!componentStore.actions.isComponentRegistered(id)) {
-      !componentStore.actions.isComponentRegistered(id)
+        componentStore.actions.registerInstance(id, {
-    ) {
+          component: instance._component,
-      componentStore.actions.registerInstance(id, {
+          getSettings: () => cachedSettings,
-        component: instance._component,
+          getRawSettings: () => ({ ...staticSettings, ...dynamicSettings }),
-        getSettings: () => cachedSettings,
+          getDataContext: () => get(context),
-        getRawSettings: () => ({ ...staticSettings, ...dynamicSettings }),
+          reload: () => initialise(instance, true),
-        getDataContext: () => get(context),
+          setEphemeralStyles: styles => (ephemeralStyles = styles),
-        reload: () => initialise(instance, true),
+          state: store,
-        setEphemeralStyles: styles => (ephemeralStyles = styles),
+        })
-        state: store,
+      }
-      })
+      return () => {
+        if (componentStore.actions.isComponentRegistered(id)) {
+          componentStore.actions.unregisterInstance(id)
+        }
+      }
     }
   })
 
-  onDestroy(() => {
+  // Observe changes to context
-    if (
+  onMount(() => context.actions.observeChanges(handleContextChange))
-      $appStore.isDevApp &&
-      componentStore.actions.isComponentRegistered(id)
-    ) {
-      componentStore.actions.unregisterInstance(id)
-    }
-  })
 </script>
 
 {#if constructor && initialSettings && (visible || inSelectedPath) && !builderHidden}

packages/client/src/components/app/DataProvider.svelte

@@ -71,7 +71,7 @@
     datasource: dataSource || {},
     schema,
     rowsLength: $fetch.rows.length,
-
+    pageNumber: $fetch.pageNumber + 1,
     // Undocumented properties. These aren't supposed to be used in builder
     // bindings, but are used internally by other components
     id: $component?.id,

packages/client/src/components/app/EmptyPlaceholder.svelte

@@ -1,6 +1,5 @@
 <script>
   import { getContext } from "svelte"
-  import { Icon } from "@budibase/bbui"
 
   const component = getContext("component")
   const { builderStore, componentStore } = getContext("sdk")
@@ -10,15 +9,7 @@
 
 {#if $builderStore.inBuilder}
   <div class="component-placeholder">
-    <Icon name="Help" color="var(--spectrum-global-color-blue-600)" />
+    {$component.name || definition?.name || "Component"}
-    <span
-      class="spectrum-Link"
-      on:click={() => {
-        builderStore.actions.requestAddComponent()
-      }}
-    >
-      Add components inside your {definition?.name || $component.type}
-    </span>
   </div>
 {/if}
 
@@ -32,14 +23,4 @@
     font-size: var(--font-size-s);
     gap: var(--spacing-s);
   }
-
-  /* Common styles for all error states to use */
-  .component-placeholder :global(mark) {
-    background-color: var(--spectrum-global-color-gray-400);
-    padding: 0 4px;
-    border-radius: 2px;
-  }
-  .component-placeholder :global(.spectrum-Link) {
-    cursor: pointer;
-  }
 </style>

packages/client/src/components/app/GridBlock.svelte

@@ -19,7 +19,36 @@
   export let onRowClick = null
   export let buttons = null
 
-  // parses columns to fix older formats
+  const context = getContext("context")
+  const component = getContext("component")
+  const {
+    styleable,
+    API,
+    builderStore,
+    notificationStore,
+    enrichButtonActions,
+    ActionTypes,
+    createContextStore,
+    Provider,
+  } = getContext("sdk")
+
+  let grid
+
+  $: columnWhitelist = parsedColumns
+    ?.filter(col => col.active)
+    ?.map(col => col.field)
+  $: schemaOverrides = getSchemaOverrides(parsedColumns)
+  $: enrichedButtons = enrichButtons(buttons)
+  $: parsedColumns = getParsedColumns(columns)
+  $: actions = [
+    {
+      type: ActionTypes.RefreshDatasource,
+      callback: () => grid?.getContext()?.rows.actions.refreshData(),
+      metadata: { dataSource: table },
+    },
+  ]
+
+  // Parses columns to fix older formats
   const getParsedColumns = columns => {
     // If the first element has an active key all elements should be in the new format
     if (columns?.length && columns[0]?.active !== undefined) {
@@ -33,28 +62,6 @@
     }))
   }
 
-  $: parsedColumns = getParsedColumns(columns)
-
-  const context = getContext("context")
-  const component = getContext("component")
-  const {
-    styleable,
-    API,
-    builderStore,
-    notificationStore,
-    enrichButtonActions,
-    ActionTypes,
-    createContextStore,
-  } = getContext("sdk")
-
-  let grid
-
-  $: columnWhitelist = parsedColumns
-    ?.filter(col => col.active)
-    ?.map(col => col.field)
-  $: schemaOverrides = getSchemaOverrides(parsedColumns)
-  $: enrichedButtons = enrichButtons(buttons)
-
   const getSchemaOverrides = columns => {
     let overrides = {}
     columns?.forEach(column => {
@@ -78,11 +85,6 @@
         const id = get(component).id
         const gridContext = createContextStore(context)
         gridContext.actions.provideData(id, row)
-        gridContext.actions.provideAction(
-          id,
-          ActionTypes.RefreshDatasource,
-          () => grid?.getContext()?.rows.actions.refreshData()
-        )
         const fn = enrichButtonActions(settings.onClick, get(gridContext))
         return await fn?.({ row })
       },
@@ -94,29 +96,31 @@
   use:styleable={$component.styles}
   class:in-builder={$builderStore.inBuilder}
 >
-  <Grid
+  <Provider {actions}>
-    bind:this={grid}
+    <Grid
-    datasource={table}
+      bind:this={grid}
-    {API}
+      datasource={table}
-    {stripeRows}
+      {API}
-    {initialFilter}
+      {stripeRows}
-    {initialSortColumn}
+      {initialFilter}
-    {initialSortOrder}
+      {initialSortColumn}
-    {fixedRowHeight}
+      {initialSortOrder}
-    {columnWhitelist}
+      {fixedRowHeight}
-    {schemaOverrides}
+      {columnWhitelist}
-    canAddRows={allowAddRows}
+      {schemaOverrides}
-    canEditRows={allowEditRows}
+      canAddRows={allowAddRows}
-    canDeleteRows={allowDeleteRows}
+      canEditRows={allowEditRows}
-    canEditColumns={false}
+      canDeleteRows={allowDeleteRows}
-    canExpandRows={false}
+      canEditColumns={false}
-    canSaveSchema={false}
+      canExpandRows={false}
-    showControls={false}
+      canSaveSchema={false}
-    notifySuccess={notificationStore.actions.success}
+      showControls={false}
-    notifyError={notificationStore.actions.error}
+      notifySuccess={notificationStore.actions.success}
-    buttons={enrichedButtons}
+      notifyError={notificationStore.actions.error}
-    on:rowclick={e => onRowClick?.({ row: e.detail })}
+      buttons={enrichedButtons}
-  />
+      on:rowclick={e => onRowClick?.({ row: e.detail })}
+    />
+  </Provider>
 </div>
 
 <style>

packages/client/src/components/app/Repeater.svelte

@@ -2,6 +2,7 @@
   import { getContext } from "svelte"
   import Placeholder from "./Placeholder.svelte"
   import Container from "./Container.svelte"
+  import { ContextScopes } from "constants"
 
   export let dataProvider
   export let noRowsMessage
@@ -9,6 +10,7 @@
   export let hAlign
   export let vAlign
   export let gap
+  export let scope = ContextScopes.Local
 
   const { Provider } = getContext("sdk")
   const component = getContext("component")
@@ -22,7 +24,7 @@
     <Placeholder />
   {:else if rows.length > 0}
     {#each rows as row, index}
-      <Provider data={{ ...row, index }}>
+      <Provider data={{ ...row, index }} {scope}>
         <slot />
       </Provider>
     {/each}

packages/client/src/components/app/blocks/MultiStepFormblock.svelte

@@ -1,5 +1,6 @@
 <script>
   import BlockComponent from "components/BlockComponent.svelte"
+  import { Helpers } from "@budibase/bbui"
   import { getContext, setContext } from "svelte"
   import { builderStore } from "stores"
   import { Utils } from "@budibase/frontend-core"
@@ -41,7 +42,7 @@
   let schema
 
   $: fetchSchema(dataSource)
-  $: enrichedSteps = enrichSteps(steps, schema, $component.id)
+  $: enrichedSteps = enrichSteps(steps, schema, $component.id, $currentStep)
   $: updateCurrentStep(enrichedSteps, $builderStore, $component)
 
   const updateCurrentStep = (steps, builderStore, component) => {
@@ -115,6 +116,7 @@
         dataSource,
       })
       return {
+        _stepId: Helpers.uuid(),
         fields: getDefaultFields(fields || [], schema),
         title: title ?? defaultProps.title,
         desc,
@@ -142,7 +144,7 @@
       },
     }}
   >
-    {#each enrichedSteps as step, stepIdx}
+    {#each enrichedSteps as step, stepIdx (step._stepId)}
       <BlockComponent
         type="formstep"
         props={{ step: stepIdx + 1, _instanceName: `Step ${stepIdx + 1}` }}
@@ -186,12 +188,13 @@
             </BlockComponent>
           </BlockComponent>
           <BlockComponent type="text" props={{ text: step.desc }} order={1} />
+
           <BlockComponent type="container" order={2}>
             <div
               class="form-block fields"
               class:mobile={$context.device.mobile}
             >
-              {#each step.fields as field, fieldIdx (`${field.field || field.name}_${stepIdx}_${fieldIdx}`)}
+              {#each step.fields as field, fieldIdx (`${field.field || field.name}_${fieldIdx}`)}
                 {#if getComponentForField(field)}
                   <BlockComponent
                     type={getComponentForField(field)}

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -231,6 +231,7 @@
           paginate,
           limit: rowCount,
         }}
+        context="provider"
         order={1}
       >
         <BlockComponent

packages/client/src/components/app/blocks/form/FormBlockWrapper.svelte

@@ -10,6 +10,7 @@
   export let noRowsMessage
 
   const component = getContext("component")
+  const { ContextScopes } = getContext("sdk")
 
   $: providerId = `${$component.id}-provider`
   $: dataProvider = `{{ literal ${safe(providerId)} }}`
@@ -55,6 +56,7 @@
           noRowsMessage: noRowsMessage || "We couldn't find a row to display",
           direction: "column",
           hAlign: "center",
+          scope: ContextScopes.Global,
         }}
       >
         <slot />

packages/client/src/components/app/embedded-map/EmbeddedMap.svelte

@@ -307,7 +307,7 @@
       // Reset view
       resetView()
     } catch (e) {
-      console.log("There was a problem with the map", e)
+      console.error("There was a problem with the map", e)
     }
   }
 

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -61,7 +61,7 @@
           resolve({ initialised: true })
         })
         .catch(err => {
-          console.log("There was a problem scanning the image", err)
+          console.error("There was a problem scanning the image", err)
           resolve({ initialised: false })
         })
     })

packages/client/src/components/app/forms/Form.svelte

@@ -21,6 +21,7 @@
   export let editAutoColumns = false
 
   const context = getContext("context")
+  const component = getContext("component")
   const { API, fetchDatasourceSchema } = getContext("sdk")
 
   const getInitialFormStep = () => {
@@ -38,28 +39,47 @@
 
   $: fetchSchema(dataSource)
   $: schemaKey = generateSchemaKey(schema)
-  $: initialValues = getInitialValues(actionType, dataSource, $context)
+  $: initialValues = getInitialValues(
+    actionType,
+    dataSource,
+    $component.path,
+    $context
+  )
   $: resetKey = Helpers.hashString(
     schemaKey + JSON.stringify(initialValues) + disabled + readonly
   )
 
   // Returns the closes data context which isn't a built in context
-  const getInitialValues = (type, dataSource, context) => {
+  const getInitialValues = (type, dataSource, path, context) => {
     // Only inherit values for update forms
     if (type !== "Update") {
       return {}
     }
     // Only inherit values for forms targeting internal tables
-    if (!dataSource?.tableId) {
+    const dsType = dataSource?.type
+    if (dsType !== "table" && dsType !== "viewV2") {
       return {}
     }
-    // Don't inherit values representing built in contexts
+    // Look up the component tree and find something that is provided by an
-    if (["user", "url"].includes(context.closestComponentId)) {
+    // ancestor that matches our datasource. This is for backwards compatibility
-      return {}
+    // as previously we could use the "closest" context.
+    for (let id of path.reverse().slice(1)) {
+      // Check for matching view datasource
+      if (
+        dataSource.type === "viewV2" &&
+        context[id]?._viewId === dataSource.id
+      ) {
+        return context[id]
+      }
+      // Check for matching table datasource
+      if (
+        dataSource.type === "table" &&
+        context[id]?.tableId === dataSource.tableId
+      ) {
+        return context[id]
+      }
     }
-    // Always inherit the closest datasource
+    return {}
-    const closestContext = context[`${context.closestComponentId}`] || {}
-    return closestContext || {}
   }
 
   // Fetches the form schema from this form's dataSource

packages/client/src/components/context/Provider.svelte

@@ -1,21 +1,24 @@
 <script>
   import { getContext, setContext, onDestroy } from "svelte"
   import { dataSourceStore, createContextStore } from "stores"
-  import { ActionTypes } from "constants"
+  import { ActionTypes, ContextScopes } from "constants"
   import { generate } from "shortid"
 
   export let data
   export let actions
   export let key
+  export let scope = ContextScopes.Global
 
-  // Clone and create new data context for this component tree
+  let context = getContext("context")
-  const context = getContext("context")
   const component = getContext("component")
-  const newContext = createContextStore(context)
-  setContext("context", newContext)
-
   const providerKey = key || $component.id
 
+  // Create a new layer of context if we are only locally scoped
+  if (scope === ContextScopes.Local) {
+    context = createContextStore(context)
+    setContext("context", context)
+  }
+
   // Generate a permanent unique ID for this component and use it to register
   // any datasource actions
   const instanceId = generate()
@@ -30,7 +33,7 @@
   const provideData = newData => {
     const dataKey = JSON.stringify(newData)
     if (dataKey !== lastDataKey) {
-      newContext.actions.provideData(providerKey, newData)
+      context.actions.provideData(providerKey, newData, scope)
       lastDataKey = dataKey
     }
   }
@@ -40,7 +43,7 @@
     if (actionsKey !== lastActionsKey) {
       lastActionsKey = actionsKey
       newActions?.forEach(({ type, callback, metadata }) => {
-        newContext.actions.provideAction(providerKey, type, callback)
+        context.actions.provideAction(providerKey, type, callback, scope)
 
         // Register any "refresh datasource" actions with a singleton store
         // so we can easily refresh data at all levels for any datasource

packages/client/src/constants.js

@@ -12,5 +12,10 @@ export const ActionTypes = {
   ScrollTo: "ScrollTo",
 }
 
+export const ContextScopes = {
+  Local: "local",
+  Global: "global",
+}
+
 export const DNDPlaceholderID = "dnd-placeholder"
 export const ScreenslotType = "screenslot"

packages/client/src/sdk.js

@@ -23,7 +23,7 @@ import { getAction } from "utils/getAction"
 import Provider from "components/context/Provider.svelte"
 import Block from "components/Block.svelte"
 import BlockComponent from "components/BlockComponent.svelte"
-import { ActionTypes } from "./constants"
+import { ActionTypes, ContextScopes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
@@ -57,6 +57,7 @@ export default {
   fetchDatasourceSchema,
   fetchData,
   LuceneUtils,
+  ContextScopes,
   getAPIKey,
   enrichButtonActions,
   processStringSync,

packages/client/src/stores/context.js

@@ -1,59 +1,98 @@
 import { writable, derived } from "svelte/store"
-import { Helpers } from "@budibase/bbui"
+import { ContextScopes } from "constants"
 
-export const createContextStore = oldContext => {
+export const createContextStore = parentContext => {
-  const newContext = writable({})
+  const context = writable({})
-  const contexts = oldContext ? [oldContext, newContext] : [newContext]
+  let observers = []
+
+  // Derive the total context state at this point in the tree
+  const contexts = parentContext ? [parentContext, context] : [context]
   const totalContext = derived(contexts, $contexts => {
-    // The key is the serialized representation of context
+    return $contexts.reduce((total, context) => ({ ...total, ...context }), {})
-    let key = ""
-    for (let i = 0; i < $contexts.length - 1; i++) {
-      key += $contexts[i].key
-    }
-    key = Helpers.hashString(
-      key + JSON.stringify($contexts[$contexts.length - 1])
-    )
-
-    // Reduce global state
-    const reducer = (total, context) => ({ ...total, ...context })
-    const context = $contexts.reduce(reducer, {})
-
-    return {
-      ...context,
-      key,
-    }
   })
 
-  // Adds a data context layer to the tree
+  // Subscribe to updates in the parent context, so that we can proxy on any
-  const provideData = (providerId, data) => {
+  // change messages to our own subscribers
-    if (!providerId || data === undefined) {
+  if (parentContext) {
-      return
+    parentContext.actions.observeChanges(key => {
-    }
+      broadcastChange(key)
-    newContext.update(state => {
-      state[providerId] = data
-
-      // Keep track of the closest component ID so we can later hydrate a "data" prop.
-      // This is only required for legacy bindings that used "data" rather than a
-      // component ID.
-      state.closestComponentId = providerId
-
-      return state
     })
   }
 
-  // Adds an action context layer to the tree
+  // Provide some data in context
-  const provideAction = (providerId, actionType, callback) => {
+  const provideData = (providerId, data, scope = ContextScopes.Global) => {
+    if (!providerId || data === undefined) {
+      return
+    }
+
+    // Proxy message up the chain if we have a parent and are providing global
+    // context
+    if (scope === ContextScopes.Global && parentContext) {
+      parentContext.actions.provideData(providerId, data, scope)
+    }
+
+    // Otherwise this is either the context root, or we're providing a local
+    // context override, so we need to update the local context instead
+    else {
+      context.update(state => {
+        state[providerId] = data
+        return state
+      })
+      broadcastChange(providerId)
+    }
+  }
+
+  // Provides some action in context
+  const provideAction = (
+    providerId,
+    actionType,
+    callback,
+    scope = ContextScopes.Global
+  ) => {
     if (!providerId || !actionType) {
       return
     }
-    newContext.update(state => {
+
-      state[`${providerId}_${actionType}`] = callback
+    // Proxy message up the chain if we have a parent and are providing global
-      return state
+    // context
-    })
+    if (scope === ContextScopes.Global && parentContext) {
+      parentContext.actions.provideAction(
+        providerId,
+        actionType,
+        callback,
+        scope
+      )
+    }
+
+    // Otherwise this is either the context root, or we're providing a local
+    // context override, so we need to update the local context instead
+    else {
+      const key = `${providerId}_${actionType}`
+      context.update(state => {
+        state[key] = callback
+        return state
+      })
+      broadcastChange(key)
+    }
+  }
+
+  const observeChanges = callback => {
+    observers.push(callback)
+    return () => {
+      observers = observers.filter(cb => cb !== callback)
+    }
+  }
+
+  const broadcastChange = key => {
+    observers.forEach(cb => cb(key))
   }
 
   return {
     subscribe: totalContext.subscribe,
-    actions: { provideData, provideAction },
+    actions: {
+      provideData,
+      provideAction,
+      observeChanges,
+    },
   }
 }

packages/client/src/stores/org.js

@@ -14,7 +14,7 @@ const createOrgStore = () => {
       const settingsConfigDoc = await API.getTenantConfig(tenantId)
       set({ logoUrl: settingsConfigDoc.config.logoUrl })
     } catch (e) {
-      console.log("Could not init org ", e)
+      console.error("Could not init org ", e)
     }
   }
 

packages/client/src/utils/buttonActions.js

@@ -17,6 +17,54 @@ import { ActionTypes } from "constants"
 import { enrichDataBindings } from "./enrichDataBinding"
 import { Helpers } from "@budibase/bbui"
 
+// Default action handler, which extracts an action from context that was
+// provided by another component and executes it with all action parameters
+const contextActionHandler = async (action, context) => {
+  const key = getActionContextKey(action)
+  const fn = context[key]
+  if (fn) {
+    return await fn(action.parameters)
+  }
+}
+
+// Generates the context key, which is the key that this action depends on in
+// context to provide the function it will run. This is broken out as a util
+// because we reuse this inside the core Component.svelte file to determine
+// what the required action context keys are for all action settings.
+export const getActionContextKey = action => {
+  const type = action?.["##eventHandlerType"]
+  const key = (componentId, type) => `${componentId}_${type}`
+  switch (type) {
+    case "Scroll To Field":
+      return key(action.parameters.componentId, ActionTypes.ScrollTo)
+    case "Update Field Value":
+      return key(action.parameters.componentId, ActionTypes.UpdateFieldValue)
+    case "Validate Form":
+      return key(action.parameters.componentId, ActionTypes.ValidateForm)
+    case "Refresh Data Provider":
+      return key(action.parameters.componentId, ActionTypes.RefreshDatasource)
+    case "Clear Form":
+      return key(action.parameters.componentId, ActionTypes.ClearForm)
+    case "Change Form Step":
+      return key(action.parameters.componentId, ActionTypes.ChangeFormStep)
+    default:
+      return null
+  }
+}
+
+// If button actions depend on context, they must declare which keys they need
+export const getActionDependentContextKeys = action => {
+  const type = action?.["##eventHandlerType"]
+  switch (type) {
+    case "Save Row":
+    case "Duplicate Row":
+      if (action.parameters?.providerId) {
+        return [action.parameters.providerId]
+      }
+  }
+  return []
+}
+
 const saveRowHandler = async (action, context) => {
   const { fields, providerId, tableId, notificationOverride } =
     action.parameters
@@ -32,20 +80,21 @@ const saveRowHandler = async (action, context) => {
     }
   }
   if (tableId) {
-    payload.tableId = tableId
+    if (tableId.startsWith("view")) {
+      payload._viewId = tableId
+    } else {
+      payload.tableId = tableId
+    }
   }
   try {
     const row = await API.saveRow(payload)
-
     if (!notificationOverride) {
       notificationStore.actions.success("Row saved")
     }
-
     // Refresh related datasources
     await dataSourceStore.actions.invalidateDataSource(tableId, {
       invalidateRelationships: true,
     })
-
     return { row }
   } catch (error) {
     // Abort next actions
@@ -64,7 +113,11 @@ const duplicateRowHandler = async (action, context) => {
       }
     }
     if (tableId) {
-      payload.tableId = tableId
+      if (tableId.startsWith("view")) {
+        payload._viewId = tableId
+      } else {
+        payload.tableId = tableId
+      }
     }
     delete payload._id
     delete payload._rev
@@ -73,12 +126,10 @@ const duplicateRowHandler = async (action, context) => {
       if (!notificationOverride) {
         notificationStore.actions.success("Row saved")
       }
-
       // Refresh related datasources
       await dataSourceStore.actions.invalidateDataSource(tableId, {
         invalidateRelationships: true,
       })
-
       return { row }
     } catch (error) {
       // Abort next actions
@@ -190,17 +241,6 @@ const navigationHandler = action => {
   routeStore.actions.navigate(url, peek, externalNewTab)
 }
 
-const scrollHandler = async (action, context) => {
-  return await executeActionHandler(
-    context,
-    action.parameters.componentId,
-    ActionTypes.ScrollTo,
-    {
-      field: action.parameters.field,
-    }
-  )
-}
-
 const queryExecutionHandler = async action => {
   const { datasourceId, queryId, queryParams, notificationOverride } =
     action.parameters
@@ -236,47 +276,6 @@ const queryExecutionHandler = async action => {
   }
 }
 
-const executeActionHandler = async (
-  context,
-  componentId,
-  actionType,
-  params
-) => {
-  const fn = context[`${componentId}_${actionType}`]
-  if (fn) {
-    return await fn(params)
-  }
-}
-
-const updateFieldValueHandler = async (action, context) => {
-  return await executeActionHandler(
-    context,
-    action.parameters.componentId,
-    ActionTypes.UpdateFieldValue,
-    {
-      type: action.parameters.type,
-      field: action.parameters.field,
-      value: action.parameters.value,
-    }
-  )
-}
-
-const validateFormHandler = async (action, context) => {
-  return await executeActionHandler(
-    context,
-    action.parameters.componentId,
-    ActionTypes.ValidateForm
-  )
-}
-
-const refreshDataProviderHandler = async (action, context) => {
-  return await executeActionHandler(
-    context,
-    action.parameters.componentId,
-    ActionTypes.RefreshDatasource
-  )
-}
-
 const logoutHandler = async action => {
   await authStore.actions.logOut()
   let redirectUrl = "/builder/auth/login"
@@ -293,23 +292,6 @@ const logoutHandler = async action => {
   }
 }
 
-const clearFormHandler = async (action, context) => {
-  return await executeActionHandler(
-    context,
-    action.parameters.componentId,
-    ActionTypes.ClearForm
-  )
-}
-
-const changeFormStepHandler = async (action, context) => {
-  return await executeActionHandler(
-    context,
-    action.parameters.componentId,
-    ActionTypes.ChangeFormStep,
-    action.parameters
-  )
-}
-
 const closeScreenModalHandler = action => {
   let url
   if (action?.parameters) {
@@ -417,16 +399,10 @@ const handlerMap = {
   ["Duplicate Row"]: duplicateRowHandler,
   ["Delete Row"]: deleteRowHandler,
   ["Navigate To"]: navigationHandler,
-  ["Scroll To Field"]: scrollHandler,
   ["Execute Query"]: queryExecutionHandler,
   ["Trigger Automation"]: triggerAutomationHandler,
-  ["Validate Form"]: validateFormHandler,
-  ["Update Field Value"]: updateFieldValueHandler,
-  ["Refresh Data Provider"]: refreshDataProviderHandler,
   ["Log Out"]: logoutHandler,
-  ["Clear Form"]: clearFormHandler,
   ["Close Screen Modal"]: closeScreenModalHandler,
-  ["Change Form Step"]: changeFormStepHandler,
   ["Update State"]: updateStateHandler,
   ["Upload File to S3"]: s3UploadHandler,
   ["Export Data"]: exportDataHandler,
@@ -461,7 +437,12 @@ export const enrichButtonActions = (actions, context) => {
     return actions
   }
 
-  const handlers = actions.map(def => handlerMap[def["##eventHandlerType"]])
+  // Get handlers for each action. If no bespoke handler is configured, fall
+  // back to simply executing this action from context.
+  const handlers = actions.map(def => {
+    return handlerMap[def["##eventHandlerType"]] || contextActionHandler
+  })
+
   return async eventContext => {
     // Button context is built up as actions are executed.
     // Inherit any previous button context which may have come from actions

packages/client/src/utils/componentProps.js

@@ -23,16 +23,6 @@ export const propsAreSame = (a, b) => {
  * Data bindings are enriched, and button actions are enriched.
  */
 export const enrichProps = (props, context, settingsDefinitionMap) => {
-  // Create context of all bindings and data contexts
-  // Duplicate the closest context as "data" which the builder requires
-  const totalContext = {
-    ...context,
-
-    // This is only required for legacy bindings that used "data" rather than a
-    // component ID.
-    data: context[context.closestComponentId],
-  }
-
   // We want to exclude any button actions from enrichment at this stage.
   // Extract top level button action settings.
   let normalProps = { ...props }
@@ -49,13 +39,13 @@ export const enrichProps = (props, context, settingsDefinitionMap) => {
   let rawConditions = normalProps._conditions
 
   // Enrich all props except button actions
-  let enrichedProps = enrichDataBindings(normalProps, totalContext)
+  let enrichedProps = enrichDataBindings(normalProps, context)
 
   // Enrich button actions.
   // Actions are enriched into a function at this stage, but actual data
   // binding enrichment is done dynamically at runtime.
   Object.keys(actionProps).forEach(prop => {
-    enrichedProps[prop] = enrichButtonActions(actionProps[prop], totalContext)
+    enrichedProps[prop] = enrichButtonActions(actionProps[prop], context)
   })
 
   // Conditions
@@ -66,7 +56,7 @@ export const enrichProps = (props, context, settingsDefinitionMap) => {
         // action
         condition.settingValue = enrichButtonActions(
           rawConditions[idx].settingValue,
-          totalContext
+          context
         )
 
         // Since we can't compare functions, we need to assume that conditions

packages/frontend-core/src/api/rows.js

@@ -19,11 +19,12 @@ export const buildRowEndpoints = API => ({
    * @param suppressErrors whether or not to suppress error notifications
    */
   saveRow: async (row, suppressErrors = false) => {
-    if (!row?.tableId) {
+    const resourceId = row?._viewId || row?.tableId
+    if (!resourceId) {
       return
     }
     return await API.post({
-      url: `/api/${row._viewId || row.tableId}/rows`,
+      url: `/api/${resourceId}/rows`,
       body: row,
       suppressErrors,
     })
@@ -35,11 +36,12 @@ export const buildRowEndpoints = API => ({
    * @param suppressErrors whether or not to suppress error notifications
    */
   patchRow: async (row, suppressErrors = false) => {
-    if (!row?.tableId && !row?._viewId) {
+    const resourceId = row?._viewId || row?.tableId
+    if (!resourceId) {
       return
     }
     return await API.patch({
-      url: `/api/${row._viewId || row.tableId}/rows`,
+      url: `/api/${resourceId}/rows`,
       body: row,
       suppressErrors,
     })

packages/frontend-core/src/components/grid/lib/websocket.js

@@ -29,7 +29,7 @@ export const createGridWebsocket = context => {
     connectToDatasource(get(datasource))
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to grid websocket:", err.message)
+    console.error("Failed to connect to grid websocket:", err.message)
   })
 
   // User events

packages/server/scripts/test.sh

@@ -5,10 +5,10 @@ if [[ -n $CI ]]
 then
   # Running in ci, where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
+  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
-  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
+  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --maxWorkers=2 --forceExit"
+  echo "jest --coverage --maxWorkers=2 --forceExit $@"
-  jest --coverage --maxWorkers=2 --forceExit
+  jest --coverage --maxWorkers=2 --forceExit $@
 fi

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -42,7 +42,7 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
 
   beforeAll(async () => {
     await config.init()

packages/server/src/api/routes/public/tests/compare.spec.ts

@@ -12,7 +12,7 @@ let apiKey: string, table: Table, app: App, makeRequest: any
 
 beforeAll(async () => {
   app = await config.init()
-  table = await config.updateTable()
+  table = await config.upsertTable()
   apiKey = await config.generateApiKey()
   makeRequest = generateMakeRequest(apiKey)
 })
@@ -69,7 +69,7 @@ describe("check the applications endpoints", () => {
 describe("check the tables endpoints", () => {
   it("should allow retrieving tables through search", async () => {
     await config.createApp("new app 1")
-    table = await config.updateTable()
+    table = await config.upsertTable()
     const res = await makeRequest("post", "/tables/search")
     expect(res).toSatisfyApiSpec()
   })
@@ -108,7 +108,7 @@ describe("check the tables endpoints", () => {
 describe("check the rows endpoints", () => {
   let row: Row
   it("should allow retrieving rows through search", async () => {
-    table = await config.updateTable()
+    table = await config.upsertTable()
     const res = await makeRequest("post", `/tables/${table._id}/rows/search`, {
       query: {},
     })

packages/server/src/api/routes/tests/query.seq.spec.ts

@@ -1,5 +1,4 @@
-const tk = require("timekeeper")
+import tk from "timekeeper"
-tk.freeze(Date.now())
 
 // Mock out postgres for this
 jest.mock("pg")
@@ -17,16 +16,24 @@ jest.mock("@budibase/backend-core", () => {
     },
   }
 })
-const setup = require("./utilities")
+import * as setup from "./utilities"
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+import { checkBuilderEndpoint } from "./utilities/TestFunctions"
-const { checkCacheForDynamicVariable } = require("../../../threads/utils")
+import { checkCacheForDynamicVariable } from "../../../threads/utils"
+
 const { basicQuery, basicDatasource } = setup.structures
-const { events, db: dbCore } = require("@budibase/backend-core")
+import { events, db as dbCore } from "@budibase/backend-core"
+import { Datasource, Query, SourceName } from "@budibase/types"
+
+tk.freeze(Date.now())
+
+const mockIsProdAppID = dbCore.isProdAppID as jest.MockedFunction<
+  typeof dbCore.isProdAppID
+>
 
 describe("/queries", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let datasource, query
+  let datasource: Datasource & Required<Pick<Datasource, "_id">>, query: Query
 
   afterAll(setup.afterAll)
 
@@ -40,18 +47,7 @@ describe("/queries", () => {
     await setupTest()
   })
 
-  async function createInvalidIntegration() {
+  const createQuery = async (query: Query) => {
-    const datasource = await config.createDatasource({
-      datasource: {
-        ...basicDatasource().datasource,
-        source: "INVALID_INTEGRATION",
-      },
-    })
-    const query = await config.createQuery()
-    return { datasource, query }
-  }
-
-  const createQuery = async query => {
     return request
       .post(`/api/queries`)
       .send(query)
@@ -67,7 +63,7 @@ describe("/queries", () => {
       jest.clearAllMocks()
       const res = await createQuery(query)
 
-      expect(res.res.statusMessage).toEqual(
+      expect((res as any).res.statusMessage).toEqual(
         `Query ${query.name} saved successfully.`
       )
       expect(res.body).toEqual({
@@ -92,7 +88,7 @@ describe("/queries", () => {
       query._rev = res.body._rev
       await createQuery(query)
 
-      expect(res.res.statusMessage).toEqual(
+      expect((res as any).res.statusMessage).toEqual(
         `Query ${query.name} saved successfully.`
       )
       expect(res.body).toEqual({
@@ -168,8 +164,8 @@ describe("/queries", () => {
 
     it("should remove sensitive info for prod apps", async () => {
       // Mock isProdAppID to pretend we are using a prod app
-      dbCore.isProdAppID.mockClear()
+      mockIsProdAppID.mockClear()
-      dbCore.isProdAppID.mockImplementation(() => true)
+      mockIsProdAppID.mockImplementation(() => true)
 
       const query = await config.createQuery()
       const res = await request
@@ -184,7 +180,7 @@ describe("/queries", () => {
 
       // Reset isProdAppID mock
       expect(dbCore.isProdAppID).toHaveBeenCalledTimes(1)
-      dbCore.isProdAppID.mockImplementation(() => false)
+      mockIsProdAppID.mockImplementation(() => false)
     })
   })
 
@@ -211,10 +207,11 @@ describe("/queries", () => {
     })
 
     it("should apply authorization to endpoint", async () => {
+      const query = await config.createQuery()
       await checkBuilderEndpoint({
         config,
         method: "DELETE",
-        url: `/api/queries/${config._id}/${config._rev}`,
+        url: `/api/queries/${query._id}/${query._rev}`,
       })
     })
   })
@@ -272,20 +269,21 @@ describe("/queries", () => {
     })
 
     it("should fail with invalid integration type", async () => {
-      let error
+      const response = await config.api.datasource.create(
-      try {
+        {
-        await createInvalidIntegration()
+          ...basicDatasource().datasource,
-      } catch (err) {
+          source: "INVALID_INTEGRATION" as SourceName,
-        error = err
+        },
-      }
+        { expectStatus: 500, rawResponse: true }
-      expect(error).toBeDefined()
+      )
-      expect(error.message).toBe("No datasource implementation found.")
+
+      expect(response.body.message).toBe("No datasource implementation found.")
     })
   })
 
   describe("variables", () => {
-    async function preview(datasource, fields) {
+    async function preview(datasource: Datasource, fields: any) {
-      return config.previewQuery(request, config, datasource, fields)
+      return config.previewQuery(request, config, datasource, fields, undefined)
     }
 
     it("should work with static variables", async () => {
@@ -370,11 +368,19 @@ describe("/queries", () => {
   })
 
   describe("Current User Request Mapping", () => {
-    async function previewGet(datasource, fields, params) {
+    async function previewGet(
+      datasource: Datasource,
+      fields: any,
+      params: any
+    ) {
       return config.previewQuery(request, config, datasource, fields, params)
     }
 
-    async function previewPost(datasource, fields, params) {
+    async function previewPost(
+      datasource: Datasource,
+      fields: any,
+      params: any
+    ) {
       return config.previewQuery(
         request,
         config,
@@ -394,14 +400,18 @@ describe("/queries", () => {
           emailHdr: "{{[user].[email]}}",
         },
       })
-      const res = await previewGet(datasource, {
+      const res = await previewGet(
-        path: "www.google.com",
+        datasource,
-        queryString: "email={{[user].[email]}}",
+        {
-        headers: {
+          path: "www.google.com",
-          queryHdr: "{{[user].[firstName]}}",
+          queryString: "email={{[user].[email]}}",
-          secondHdr: "1234",
+          headers: {
+            queryHdr: "{{[user].[firstName]}}",
+            secondHdr: "1234",
+          },
         },
-      })
+        undefined
+      )
 
       const parsedRequest = JSON.parse(res.body.extra.raw)
       expect(parsedRequest.opts.headers).toEqual({

packages/server/src/api/routes/tests/row.spec.ts

@@ -581,7 +581,7 @@ describe.each([
         tableId: InternalTable.USER_METADATA,
       }
 
-      let table = await config.api.table.create({
+      let table = await config.api.table.save({
         name: "TestTable",
         type: "table",
         sourceType: TableSourceType.INTERNAL,
@@ -1690,7 +1690,7 @@ describe.each([
           tableConfig.sourceType = TableSourceType.EXTERNAL
         }
       }
-      const table = await config.api.table.create({
+      const table = await config.api.table.save({
         ...tableConfig,
         schema: {
           ...tableConfig.schema,

packages/server/src/api/routes/tests/table.spec.ts

@@ -438,7 +438,7 @@ describe("/tables", () => {
     })
 
     it("should successfully migrate a one-to-many user relationship to a user column", async () => {
-      const table = await config.api.table.create({
+      const table = await config.api.table.save({
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
@@ -496,7 +496,7 @@ describe("/tables", () => {
       // We found a bug just after releasing this feature where if the row was created from the
       // users table, not the table linking to it, the migration would succeed but lose the data.
       // This happened because the order of the documents in the link was reversed.
-      const table = await config.api.table.create({
+      const table = await config.api.table.save({
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
@@ -554,7 +554,7 @@ describe("/tables", () => {
     })
 
     it("should successfully migrate a many-to-many user relationship to a users column", async () => {
-      const table = await config.api.table.create({
+      const table = await config.api.table.save({
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
@@ -611,7 +611,7 @@ describe("/tables", () => {
     })
 
     it("should successfully migrate a many-to-one user relationship to a users column", async () => {
-      const table = await config.api.table.create({
+      const table = await config.api.table.save({
         name: "table",
         type: "table",
         sourceId: INTERNAL_TABLE_SOURCE_ID,
@@ -670,7 +670,7 @@ describe("/tables", () => {
     describe("unhappy paths", () => {
       let table: Table
       beforeAll(async () => {
-        table = await config.api.table.create({
+        table = await config.api.table.save({
           name: "table",
           type: "table",
           sourceId: INTERNAL_TABLE_SOURCE_ID,

packages/server/src/automations/automationUtils.ts

@@ -5,7 +5,7 @@ import {
 } from "@budibase/string-templates"
 import sdk from "../sdk"
 import { Row } from "@budibase/types"
-import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
+import { LoopInput, LoopStep, LoopStepType } from "../definitions/automations"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -128,23 +128,28 @@ export function substituteLoopStep(hbsString: string, substitute: string) {
 }
 
 export function stringSplit(value: string | string[]) {
-  if (value == null || Array.isArray(value)) {
+  if (value == null) {
-    return value || []
+    return []
   }
-  if (value.split("\n").length > 1) {
+  if (Array.isArray(value)) {
-    value = value.split("\n")
+    return value
-  } else {
-    value = value.split(",")
   }
-  return value
+  if (typeof value !== "string") {
+    throw new Error(`Unable to split value of type ${typeof value}: ${value}`)
+  }
+  const splitOnNewLine = value.split("\n")
+  if (splitOnNewLine.length > 1) {
+    return splitOnNewLine
+  }
+  return value.split(",")
 }
 
-export function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+export function typecastForLooping(input: LoopInput) {
   if (!input || !input.binding) {
     return null
   }
   try {
-    switch (loopStep.inputs.option) {
+    switch (input.option) {
       case LoopStepType.ARRAY:
         if (typeof input.binding === "string") {
           return JSON.parse(input.binding)

packages/server/src/automations/tests/loop.spec.ts

@@ -3,11 +3,13 @@ import * as triggers from "../triggers"
 import { loopAutomation } from "../../tests/utilities/structures"
 import { context } from "@budibase/backend-core"
 import * as setup from "./utilities"
+import { Row, Table } from "@budibase/types"
+import { LoopInput, LoopStepType } from "../../definitions/automations"
 
 describe("Attempt to run a basic loop automation", () => {
   let config = setup.getConfig(),
-    table: any,
+    table: Table,
-    row: any
+    row: Row
 
   beforeEach(async () => {
     await automation.init()
@@ -18,12 +20,12 @@ describe("Attempt to run a basic loop automation", () => {
 
   afterAll(setup.afterAll)
 
-  async function runLoop(loopOpts?: any) {
+  async function runLoop(loopOpts?: LoopInput) {
     const appId = config.getAppId()
     return await context.doInAppContext(appId, async () => {
       const params = { fields: { appId } }
       return await triggers.externalTrigger(
-        loopAutomation(table._id, loopOpts),
+        loopAutomation(table._id!, loopOpts),
         params,
         { getResponses: true }
       )
@@ -37,9 +39,17 @@ describe("Attempt to run a basic loop automation", () => {
 
   it("test a loop with a string", async () => {
     const resp = await runLoop({
-      type: "String",
+      option: LoopStepType.STRING,
       binding: "a,b,c",
     })
     expect(resp.steps[2].outputs.iterations).toBe(3)
   })
+
+  it("test a loop with a binding that returns an integer", async () => {
+    const resp = await runLoop({
+      option: LoopStepType.ARRAY,
+      binding: "{{ 1 }}",
+    })
+    expect(resp.steps[2].outputs.iterations).toBe(1)
+  })
 })

packages/server/src/automations/tests/updateRow.spec.ts

@@ -67,7 +67,7 @@ describe("test the update row action", () => {
       tableId: InternalTable.USER_METADATA,
     }
 
-    let table = await config.api.table.create({
+    let table = await config.api.table.save({
       name: uuid.v4(),
       type: "table",
       sourceType: TableSourceType.INTERNAL,
@@ -120,7 +120,7 @@ describe("test the update row action", () => {
       tableId: InternalTable.USER_METADATA,
     }
 
-    let table = await config.api.table.create({
+    let table = await config.api.table.save({
       name: uuid.v4(),
       type: "table",
       sourceType: TableSourceType.INTERNAL,

packages/server/src/automations/triggers.ts

@@ -9,7 +9,7 @@ import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"
-import { executeSynchronously } from "../threads/automation"
+import { executeInThread } from "../threads/automation"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -117,8 +117,7 @@ export async function externalTrigger(
       appId: context.getAppId(),
       automation,
     }
-    const job = { data } as AutomationJob
+    return executeInThread({ data } as AutomationJob)
-    return executeSynchronously(job)
   } else {
     return automationQueue.add(data, JOB_OPTS)
   }

packages/server/src/automations/unitTests/automationUtils.spec.ts

@@ -1,10 +1,15 @@
-const automationUtils = require("../automationUtils")
+import { LoopStep, LoopStepType } from "../../definitions/automations"
+import {
+  typecastForLooping,
+  cleanInputValues,
+  substituteLoopStep,
+} from "../automationUtils"
 
 describe("automationUtils", () => {
   describe("substituteLoopStep", () => {
     it("should allow multiple loop binding substitutes", () => {
       expect(
-        automationUtils.substituteLoopStep(
+        substituteLoopStep(
           `{{ loop.currentItem._id }} {{ loop.currentItem._id }} {{ loop.currentItem._id }}`,
           "step.2"
         )
@@ -15,7 +20,7 @@ describe("automationUtils", () => {
 
     it("should handle not subsituting outside of curly braces", () => {
       expect(
-        automationUtils.substituteLoopStep(
+        substituteLoopStep(
           `loop {{ loop.currentItem._id }}loop loop{{ loop.currentItem._id }}loop`,
           "step.2"
         )
@@ -28,37 +33,20 @@ describe("automationUtils", () => {
   describe("typeCastForLooping", () => {
     it("should parse to correct type", () => {
       expect(
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: [1, 2, 3] })
-          { inputs: { option: "Array" } },
-          { binding: [1, 2, 3] }
-        )
       ).toEqual([1, 2, 3])
       expect(
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: "[1,2,3]" })
-          { inputs: { option: "Array" } },
-          { binding: "[1, 2, 3]" }
-        )
       ).toEqual([1, 2, 3])
       expect(
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.STRING, binding: [1, 2, 3] })
-          { inputs: { option: "String" } },
-          { binding: [1, 2, 3] }
-        )
       ).toEqual("1,2,3")
     })
     it("should handle null values", () => {
       // expect it to handle where the binding is null
-      expect(
+      expect(typecastForLooping({ option: LoopStepType.ARRAY })).toEqual(null)
-        automationUtils.typecastForLooping(
-          { inputs: { option: "Array" } },
-          { binding: null }
-        )
-      ).toEqual(null)
       expect(() =>
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: "test" })
-          { inputs: { option: "Array" } },
-          { binding: "test" }
-        )
       ).toThrow()
     })
   })
@@ -80,7 +68,7 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        automationUtils.cleanInputValues(
+        cleanInputValues(
           {
             row: {
               relationship: `[{"_id": "ro_ta_users_us_3"}]`,
@@ -113,7 +101,7 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        automationUtils.cleanInputValues(
+        cleanInputValues(
           {
             row: {
               relationship: `ro_ta_users_us_3`,

packages/server/src/db/tests/linkController.spec.ts

@@ -324,7 +324,7 @@ describe("test the link controller", () => {
       name: "link",
       autocolumn: true,
     }
-    await config.updateTable(table)
+    await config.upsertTable(table)
   })
 
   it("should be able to remove a linked field from a table, even if the linked table does not exist", async () => {

packages/server/src/definitions/automations.ts

@@ -6,14 +6,14 @@ export enum LoopStepType {
 }
 
 export interface LoopStep extends AutomationStep {
-  inputs: {
+  inputs: LoopInput
-    option: LoopStepType
-    [key: string]: any
-  }
 }
 
 export interface LoopInput {
-  binding: string[] | string
+  option: LoopStepType
+  binding?: string[] | string | number[]
+  iterations?: string
+  failure?: any
 }
 
 export interface TriggerOutput {

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -5,7 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
-  let config = new TestConfig(false)
+  const config = new TestConfig()
 
   beforeEach(async () => {
     await config.init()

packages/server/src/sdk/app/rows/tests/internal.spec.ts

@@ -8,10 +8,10 @@ import {
   FieldType,
   Table,
   AutoFieldSubType,
+  AutoColumnFieldMetadata,
 } from "@budibase/types"
 
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-import { cache } from "@budibase/backend-core"
 
 tk.freeze(Date.now())
 
@@ -213,8 +213,10 @@ describe("sdk >> rows >> internal", () => {
       )
 
       const persistedTable = await config.getTable(table._id)
-      expect((table as any).schema.id.lastID).toBe(0)
+      expect((table.schema.id as AutoColumnFieldMetadata).lastID).toBe(0)
-      expect(persistedTable.schema.id.lastID).toBe(20)
+      expect((persistedTable.schema.id as AutoColumnFieldMetadata).lastID).toBe(
+        20
+      )
     })
   })
 })

packages/server/src/sdk/tests/tables.spec.ts

@@ -9,7 +9,7 @@ describe("tables", () => {
 
   beforeAll(async () => {
     await config.init()
-    table = await config.api.table.create(basicTable())
+    table = await config.api.table.save(basicTable())
   })
 
   describe("getTables", () => {

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -27,7 +27,18 @@ import {
   sessions,
   tenancy,
 } from "@budibase/backend-core"
-import * as controllers from "./controllers"
+import {
+  app as appController,
+  deploy as deployController,
+  role as roleController,
+  automation as automationController,
+  webhook as webhookController,
+  query as queryController,
+  screen as screenController,
+  layout as layoutController,
+  view as viewController,
+} from "./controllers"
+
 import { cleanup } from "../../utilities/fileSystem"
 import newid from "../../db/newid"
 import { generateUserMetadataID } from "../../db/utils"
@@ -44,13 +55,14 @@ import {
   RelationshipFieldMetadata,
   RelationshipType,
   Row,
-  SearchFilters,
+  SearchParams,
   SourceName,
   Table,
   TableSourceType,
   User,
   UserRoles,
   View,
+  WithRequired,
 } from "@budibase/types"
 
 import API from "./api"
@@ -543,11 +555,7 @@ class TestConfiguration {
     // clear any old app
     this.appId = null
     this.app = await context.doInTenant(this.tenantId!, async () => {
-      const app = await this._req(
+      const app = await this._req({ name: appName }, null, appController.create)
-        { name: appName },
-        null,
-        controllers.app.create
-      )
       this.appId = app.appId!
       return app
     })
@@ -563,7 +571,7 @@ class TestConfiguration {
   }
 
   async publish() {
-    await this._req(null, null, controllers.deploy.publishApp)
+    await this._req(null, null, deployController.publishApp)
     // @ts-ignore
     const prodAppId = this.getAppId().replace("_dev", "")
     this.prodAppId = prodAppId
@@ -578,7 +586,7 @@ class TestConfiguration {
     const response = await this._req(
       null,
       { appId: this.appId },
-      controllers.app.unpublish
+      appController.unpublish
     )
     this.prodAppId = null
     this.prodApp = null
@@ -587,14 +595,16 @@ class TestConfiguration {
 
   // TABLE
 
-  async updateTable(
+  async upsertTable(
     config?: TableToBuild,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
-    config.sourceType = config.sourceType || TableSourceType.INTERNAL
+    const response = await this.api.table.save({
-    config.sourceId = config.sourceId || INTERNAL_TABLE_SOURCE_ID
+      ...config,
-    const response = await this._req(config, null, controllers.table.save)
+      sourceType: config.sourceType || TableSourceType.INTERNAL,
+      sourceId: config.sourceId || INTERNAL_TABLE_SOURCE_ID,
+    })
     if (!skipReassigning) {
       this.table = response
     }
@@ -612,7 +622,7 @@ class TestConfiguration {
     if (!config.sourceId) {
       config.sourceId = INTERNAL_TABLE_SOURCE_ID
     }
-    return this.updateTable(config, options)
+    return this.upsertTable(config, options)
   }
 
   async createExternalTable(
@@ -627,12 +637,12 @@ class TestConfiguration {
       config.sourceId = this.datasource._id
       config.sourceType = TableSourceType.EXTERNAL
     }
-    return this.updateTable(config, options)
+    return this.upsertTable(config, options)
   }
 
   async getTable(tableId?: string) {
     tableId = tableId || this.table!._id!
-    return this._req(null, { tableId }, controllers.table.find)
+    return this.api.table.get(tableId)
   }
 
   async createLinkedTable(
@@ -680,37 +690,35 @@ class TestConfiguration {
     if (!this.table) {
       throw "Test requires table to be configured."
     }
-    const tableId = (config && config.tableId) || this.table._id
+    const tableId = (config && config.tableId) || this.table._id!
     config = config || basicRow(tableId!)
-    return this._req(config, { tableId }, controllers.row.save)
+    return this.api.row.save(tableId, config)
   }
 
   async getRow(tableId: string, rowId: string): Promise<Row> {
-    return this._req(null, { tableId, rowId }, controllers.row.find)
+    const res = await this.api.row.get(tableId, rowId)
+    return res.body
   }
 
   async getRows(tableId: string) {
     if (!tableId && this.table) {
       tableId = this.table._id!
     }
-    return this._req(null, { tableId }, controllers.row.fetch)
+    return this.api.row.fetch(tableId)
   }
 
-  async searchRows(tableId: string, searchParams: SearchFilters = {}) {
+  async searchRows(tableId: string, searchParams?: SearchParams) {
     if (!tableId && this.table) {
       tableId = this.table._id!
     }
-    const body = {
+    return this.api.row.search(tableId, searchParams)
-      query: searchParams,
-    }
-    return this._req(body, { tableId }, controllers.row.search)
   }
 
   // ROLE
 
   async createRole(config?: any) {
     config = config || basicRole()
-    return this._req(config, null, controllers.role.save)
+    return this._req(config, null, roleController.save)
   }
 
   // VIEW
@@ -723,7 +731,7 @@ class TestConfiguration {
       tableId: this.table!._id,
       name: generator.guid(),
     }
-    return this._req(view, null, controllers.view.v1.save)
+    return this._req(view, null, viewController.v1.save)
   }
 
   async createView(
@@ -753,13 +761,13 @@ class TestConfiguration {
       delete config._rev
     }
     this.automation = (
-      await this._req(config, null, controllers.automation.create)
+      await this._req(config, null, automationController.create)
     ).automation
     return this.automation
   }
 
   async getAllAutomations() {
-    return this._req(null, null, controllers.automation.fetch)
+    return this._req(null, null, automationController.fetch)
   }
 
   async deleteAutomation(automation?: any) {
@@ -770,7 +778,7 @@ class TestConfiguration {
     return this._req(
       null,
       { id: automation._id, rev: automation._rev },
-      controllers.automation.destroy
+      automationController.destroy
     )
   }
 
@@ -779,28 +787,27 @@ class TestConfiguration {
       throw "Must create an automation before creating webhook."
     }
     config = config || basicWebhook(this.automation._id)
-    return (await this._req(config, null, controllers.webhook.save)).webhook
+
+    return (await this._req(config, null, webhookController.save)).webhook
   }
 
   // DATASOURCE
 
   async createDatasource(config?: {
     datasource: Datasource
-  }): Promise<Datasource> {
+  }): Promise<WithRequired<Datasource, "_id">> {
     config = config || basicDatasource()
-    const response = await this._req(config, null, controllers.datasource.save)
+    const response = await this.api.datasource.create(config.datasource)
-    this.datasource = response.datasource
+    this.datasource = response
-    return this.datasource!
+    return { ...this.datasource, _id: this.datasource!._id! }
   }
 
-  async updateDatasource(datasource: Datasource): Promise<Datasource> {
+  async updateDatasource(
-    const response = await this._req(
+    datasource: Datasource
-      datasource,
+  ): Promise<WithRequired<Datasource, "_id">> {
-      { datasourceId: datasource._id },
+    const response = await this.api.datasource.update(datasource)
-      controllers.datasource.update
+    this.datasource = response
-    )
+    return { ...this.datasource, _id: this.datasource!._id! }
-    this.datasource = response.datasource
-    return this.datasource!
   }
 
   async restDatasource(cfg?: any) {
@@ -815,6 +822,7 @@ class TestConfiguration {
 
   async dynamicVariableDatasource() {
     let datasource = await this.restDatasource()
+
     const basedOnQuery = await this.createQuery({
       ...basicQuery(datasource._id!),
       fields: {
@@ -886,21 +894,21 @@ class TestConfiguration {
       throw "No datasource created for query."
     }
     config = config || basicQuery(this.datasource!._id!)
-    return this._req(config, null, controllers.query.save)
+    return this._req(config, null, queryController.save)
   }
 
   // SCREEN
 
   async createScreen(config?: any) {
     config = config || basicScreen()
-    return this._req(config, null, controllers.screen.save)
+    return this._req(config, null, screenController.save)
   }
 
   // LAYOUT
 
   async createLayout(config?: any) {
     config = config || basicLayout()
-    return await this._req(config, null, controllers.layout.save)
+    return await this._req(config, null, layoutController.save)
   }
 }
 

packages/server/src/tests/utilities/api/datasource.ts

@@ -2,20 +2,23 @@ import {
   CreateDatasourceRequest,
   Datasource,
   VerifyDatasourceRequest,
-  VerifyDatasourceResponse,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
+import supertest from "supertest"
 
 export class DatasourceAPI extends TestAPI {
   constructor(config: TestConfiguration) {
     super(config)
   }
 
-  create = async (
+  create = async <B extends boolean = false>(
     config: Datasource,
-    { expectStatus } = { expectStatus: 200 }
+    {
-  ): Promise<Datasource> => {
+      expectStatus,
+      rawResponse,
+    }: { expectStatus?: number; rawResponse?: B } = {}
+  ): Promise<B extends false ? Datasource : supertest.Response> => {
     const body: CreateDatasourceRequest = {
       datasource: config,
       tablesFilter: [],
@@ -25,8 +28,11 @@ export class DatasourceAPI extends TestAPI {
       .send(body)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(expectStatus)
+      .expect(expectStatus || 200)
-    return result.body.datasource as Datasource
+    if (rawResponse) {
+      return result as any
+    }
+    return result.body.datasource
   }
 
   update = async (

packages/server/src/tests/utilities/api/row.ts

@@ -7,6 +7,7 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   SearchRowResponse,
+  SearchParams,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -154,10 +155,12 @@ export class RowAPI extends TestAPI {
 
   search = async (
     sourceId: string,
+    params?: SearchParams,
     { expectStatus } = { expectStatus: 200 }
   ): Promise<SearchRowResponse> => {
     const request = this.request
       .post(`/api/${sourceId}/search`)
+      .send(params)
       .set(this.config.defaultHeaders())
       .expect(expectStatus)
 

packages/server/src/tests/utilities/api/table.ts

@@ -13,7 +13,7 @@ export class TableAPI extends TestAPI {
     super(config)
   }
 
-  create = async (
+  save = async (
     data: SaveTableRequest,
     { expectStatus } = { expectStatus: 200 }
   ): Promise<SaveTableResponse> => {

packages/server/src/tests/utilities/structures.ts

@@ -21,8 +21,9 @@ import {
   Table,
   INTERNAL_TABLE_SOURCE_ID,
   TableSourceType,
-  AutomationIOType,
+  Query,
 } from "@budibase/types"
+import { LoopInput, LoopStepType } from "../../definitions/automations"
 
 const { BUILTIN_ROLE_IDS } = roles
 
@@ -204,10 +205,13 @@ export function serverLogAutomation(appId?: string): Automation {
   }
 }
 
-export function loopAutomation(tableId: string, loopOpts?: any): Automation {
+export function loopAutomation(
+  tableId: string,
+  loopOpts?: LoopInput
+): Automation {
   if (!loopOpts) {
     loopOpts = {
-      option: "Array",
+      option: LoopStepType.ARRAY,
       binding: "{{ steps.1.rows }}",
     }
   }
@@ -360,7 +364,7 @@ export function basicDatasource(): { datasource: Datasource } {
   }
 }
 
-export function basicQuery(datasourceId: string) {
+export function basicQuery(datasourceId: string): Query {
   return {
     datasourceId: datasourceId,
     name: "New Query",
@@ -368,6 +372,8 @@ export function basicQuery(datasourceId: string) {
     fields: {},
     schema: {},
     queryVerb: "read",
+    transformer: null,
+    readable: true,
   }
 }
 

packages/server/src/threads/automation.ts

@@ -43,22 +43,19 @@ const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
 function getLoopIterations(loopStep: LoopStep) {
-  let binding = loopStep.inputs.binding
+  const binding = loopStep.inputs.binding
   if (!binding) {
     return 0
   }
-  const isString = typeof binding === "string"
   try {
-    if (isString) {
+    const json = typeof binding === "string" ? JSON.parse(binding) : binding
-      binding = JSON.parse(binding)
+    if (Array.isArray(json)) {
+      return json.length
     }
   } catch (err) {
     // ignore error - wasn't able to parse
   }
-  if (Array.isArray(binding)) {
+  if (typeof binding === "string") {
-    return binding.length
-  }
-  if (isString) {
     return automationUtils.stringSplit(binding).length
   }
   return 0
@@ -256,7 +253,7 @@ class Orchestrator {
         this._context.env = await sdkUtils.getEnvironmentVariables()
         let automation = this._automation
         let stopped = false
-        let loopStep: AutomationStep | undefined = undefined
+        let loopStep: LoopStep | undefined = undefined
 
         let stepCount = 0
         let loopStepNumber: any = undefined
@@ -311,7 +308,7 @@ class Orchestrator {
 
             stepCount++
             if (step.stepId === LOOP_STEP_ID) {
-              loopStep = step
+              loopStep = step as LoopStep
               loopStepNumber = stepCount
               continue
             }
@@ -331,7 +328,6 @@ class Orchestrator {
                 }
                 try {
                   loopStep.inputs.binding = automationUtils.typecastForLooping(
-                    loopStep as LoopStep,
                     loopStep.inputs as LoopInput
                   )
                 } catch (err) {
@@ -348,7 +344,7 @@ class Orchestrator {
                   loopStep = undefined
                   break
                 }
-                let item = []
+                let item: any[] = []
                 if (
                   typeof loopStep.inputs.binding === "string" &&
                   loopStep.inputs.option === "String"
@@ -399,7 +395,8 @@ class Orchestrator {
 
                 if (
                   index === env.AUTOMATION_MAX_ITERATIONS ||
-                  index === parseInt(loopStep.inputs.iterations)
+                  (loopStep.inputs.iterations &&
+                    index === parseInt(loopStep.inputs.iterations))
                 ) {
                   this.updateContextAndOutput(
                     loopStepNumber,
@@ -615,7 +612,7 @@ export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
   })
 }
 
-export function executeSynchronously(job: Job) {
+export async function executeInThread(job: Job<AutomationData>) {
   const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
@@ -627,10 +624,10 @@ export function executeSynchronously(job: Job) {
     }, job.data.event.timeout || 12000)
   })
 
-  return context.doInAppContext(appId, async () => {
+  return await context.doInAppContext(appId, async () => {
     const envVars = await sdkUtils.getEnvironmentVariables()
     // put into automation thread for whole context
-    return context.doInEnvironmentContext(envVars, async () => {
+    return await context.doInEnvironmentContext(envVars, async () => {
       const automationOrchestrator = new Orchestrator(job)
       return await Promise.race([
         automationOrchestrator.execute(),

packages/string-templates/package.json

@@ -25,7 +25,7 @@
     "manifest": "node ./scripts/gen-collection-info.js"
   },
   "dependencies": {
-    "@budibase/handlebars-helpers": "^0.13.0",
+    "@budibase/handlebars-helpers": "^0.13.1",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "lodash.clonedeep": "^4.5.0",

packages/string-templates/scripts/gen-collection-info.js

@@ -10,8 +10,8 @@ const marked = require("marked")
  * https://github.com/budibase/handlebars-helpers
  */
 const { join } = require("path")
+const path = require("path")
 
-const DIRECTORY = join(__dirname, "..", "..", "..")
 const COLLECTIONS = [
   "math",
   "array",
@@ -115,6 +115,8 @@ function getCommentInfo(file, func) {
     docs.example = docs.example.replace("product", "multiply")
   }
   docs.description = blocks[0].trim()
+  docs.acceptsBlock = docs.tags.some(el => el.title === "block")
+  docs.acceptsInline = docs.tags.some(el => el.title === "inline")
   return docs
 }
 
@@ -127,7 +129,7 @@ function run() {
   const foundNames = []
   for (let collection of COLLECTIONS) {
     const collectionFile = fs.readFileSync(
-      `${DIRECTORY}/node_modules/${HELPER_LIBRARY}/lib/${collection}.js`,
+      `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`,
       "utf8"
     )
     const collectionInfo = {}
@@ -159,6 +161,7 @@ function run() {
         numArgs: args.length,
         example: jsDocInfo.example || undefined,
         description: jsDocInfo.description,
+        requiresBlock: jsDocInfo.acceptsBlock && !jsDocInfo.acceptsInline,
       })
     }
     outputJSON[collection] = collectionInfo

packages/string-templates/src/conversion/index.js

@@ -1,4 +1,4 @@
-const { getHelperList } = require("../helpers")
+const { getJsHelperList } = require("../helpers")
 
 function getLayers(fullBlock) {
   let layers = []
@@ -109,7 +109,7 @@ module.exports.convertHBSBlock = (block, blockNumber) => {
   const layers = getLayers(block)
 
   let value = null
-  const list = getHelperList()
+  const list = getJsHelperList()
   for (let layer of layers) {
     const parts = splitBySpace(layer)
     if (value || parts.length > 1 || list[parts[0]]) {

packages/string-templates/src/helpers/date.js

@@ -115,7 +115,7 @@ module.exports.duration = (str, pattern, format) => {
   setLocale(config.str, config.pattern)
 
   const duration = dayjs.duration(config.str, config.pattern)
-  if (!isOptions(format)) {
+  if (format && !isOptions(format)) {
     return duration.format(format)
   } else {
     return duration.humanize()

packages/string-templates/src/helpers/index.js

@@ -7,7 +7,7 @@ const {
   HelperFunctionBuiltin,
   LITERAL_MARKER,
 } = require("./constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 const HTML_SWAPS = {
   "<": "&lt;",
@@ -97,4 +97,4 @@ module.exports.unregisterAll = handlebars => {
   externalHandlebars.unregisterAll(handlebars)
 }
 
-module.exports.getHelperList = getHelperList
+module.exports.getJsHelperList = getJsHelperList

packages/string-templates/src/helpers/javascript.js

@@ -1,7 +1,7 @@
 const { atob } = require("../utilities")
 const cloneDeep = require("lodash.clonedeep")
 const { LITERAL_MARKER } = require("../helpers/constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.cjs or index.mjs).
@@ -49,7 +49,7 @@ module.exports.processJS = (handlebars, context) => {
     // app context.
     const sandboxContext = {
       $: path => getContextValue(path, cloneDeep(context)),
-      helpers: getHelperList(),
+      helpers: getJsHelperList(),
     }
 
     // Create a sandbox with our context and run the JS

packages/string-templates/src/helpers/list.js

@@ -3,7 +3,10 @@ const helperList = require("@budibase/handlebars-helpers")
 
 let helpers = undefined
 
-module.exports.getHelperList = () => {
+const helpersToRemoveForJs = ["sortBy"]
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs
+
+module.exports.getJsHelperList = () => {
   if (helpers) {
     return helpers
   }
@@ -15,12 +18,17 @@ module.exports.getHelperList = () => {
   }
   for (let collection of constructed) {
     for (let [key, func] of Object.entries(collection)) {
-      helpers[key] = func
+      // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
+      helpers[key] = (...props) => func(...props, {})
     }
   }
   for (let key of Object.keys(externalHandlebars.addedHelpers)) {
     helpers[key] = externalHandlebars.addedHelpers[key]
   }
+
+  for (const toRemove of helpersToRemoveForJs) {
+    delete helpers[toRemove]
+  }
   Object.freeze(helpers)
   return helpers
 }

packages/string-templates/src/index.cjs

@@ -20,6 +20,7 @@ module.exports.findHBSBlocks = templates.findHBSBlocks
 module.exports.convertToJS = templates.convertToJS
 module.exports.setJSRunner = templates.setJSRunner
 module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (!process.env.NO_JS) {
   const { VM } = require("vm2")

packages/string-templates/src/index.js

@@ -10,6 +10,7 @@ const {
 } = require("./utilities")
 const { convertHBSBlock } = require("./conversion")
 const javascript = require("./helpers/javascript")
+const { helpersToRemoveForJs } = require("./helpers/list")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -394,3 +395,4 @@ module.exports.convertToJS = hbs => {
 }
 
 module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs

packages/string-templates/src/index.mjs

@@ -21,6 +21,7 @@ export const findHBSBlocks = templates.findHBSBlocks
 export const convertToJS = templates.convertToJS
 export const setJSRunner = templates.setJSRunner
 export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+export const helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (process && !process.env.NO_JS) {
   /**

packages/string-templates/test/manifest.spec.js

@@ -16,21 +16,55 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
 })
 
 const fs = require("fs")
-const { processString } = require("../src/index.cjs")
+const {
+  processString,
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
+const { getJsHelperList } = require("../src/helpers")
+
 tk.freeze("2021-01-21T12:00:00")
 
+const processJS = (js, context) => {
+  return processStringSync(encodeJSBinding(js), context)
+}
+
 const manifest = JSON.parse(
   fs.readFileSync(require.resolve("../manifest.json"), "utf8")
 )
 
 const collections = Object.keys(manifest)
 const examples = collections.reduce((acc, collection) => {
-  const functions = Object.keys(manifest[collection]).filter(
+  const functions = Object.entries(manifest[collection])
-    fnc => manifest[collection][fnc].example
+    .filter(([_, details]) => details.example)
-  )
+    .map(([name, details]) => {
-  if (functions.length) {
+      const example = details.example
+      let [hbs, js] = example.split("->").map(x => x.trim())
+      if (!js) {
+        // The function has no return value
+        return
+      }
+
+      // Trim 's
+      js = js.replace(/^\'|\'$/g, "")
+      if ((parsedExpected = tryParseJson(js))) {
+        if (Array.isArray(parsedExpected)) {
+          if (typeof parsedExpected[0] === "object") {
+            js = JSON.stringify(parsedExpected)
+          } else {
+            js = parsedExpected.join(",")
+          }
+        }
+      }
+      const requiresHbsBody = details.requiresBlock
+      return [name, { hbs, js, requiresHbsBody }]
+    })
+    .filter(x => !!x)
+
+  if (Object.keys(functions).length) {
     acc[collection] = functions
   }
   return acc
@@ -55,11 +89,7 @@ function tryParseJson(str) {
 describe("manifest", () => {
   describe("examples are valid", () => {
     describe.each(Object.keys(examples))("%s", collection => {
-      it.each(examples[collection])("%s", async func => {
+      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
-        const example = manifest[collection][func].example
-
-        let [hbs, js] = example.split("->").map(x => x.trim())
-
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -71,23 +101,40 @@ describe("manifest", () => {
           context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
         })
 
-        if (js === undefined) {
+        let result = await processString(hbs, context)
-          // The function has no return value
+        result = result.replace(/ /g, " ")
-          return
+        expect(result).toEqual(js)
+      })
+    })
+  })
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(examples).reduce((acc, v) => {
+      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      it.each(
+        jsExamples[collection].filter(
+          ([_, { requiresHbsBody }]) => !requiresHbsBody
+        )
+      )("%s", async (_, { hbs, js }) => {
+        const context = {
+          double: i => i * 2,
+          isString: x => typeof x === "string",
         }
 
-        let result = await processString(hbs, context)
+        const arrays = hbs.match(/\[[^/\]]+\]/)
-        // Trim 's
+        arrays?.forEach((arrayString, i) => {
-        js = js.replace(/^\'|\'$/g, "")
+          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-        if ((parsedExpected = tryParseJson(js))) {
+          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-          if (Array.isArray(parsedExpected)) {
+        })
-            if (typeof parsedExpected[0] === "object") {
+
-              js = JSON.stringify(parsedExpected)
+        let convertedJs = convertToJS(hbs)
-            } else {
+
-              js = parsedExpected.join(",")
+        let result = processJS(convertedJs, context)
-            }
-          }
-        }
         result = result.replace(/ /g, " ")
         expect(result).toEqual(js)
       })

packages/types/package.json

@@ -15,7 +15,7 @@
   },
   "jest": {},
   "devDependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@types/koa": "2.13.4",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",

packages/types/src/sdk/events/auth.ts

@@ -1,7 +1,12 @@
 import { BaseEvent } from "./event"
 import { ConfigType } from "../../documents"
 
-export type LoginSource = "local" | "google" | "oidc" | "google-internal"
+export type LoginSource =
+  | "local"
+  | "google"
+  | "microsoft"
+  | "oidc"
+  | "google-internal"
 export type SSOType = ConfigType.OIDC | ConfigType.GOOGLE
 
 export interface LoginEvent extends BaseEvent {

packages/types/src/shared/typeUtils.ts

@@ -7,3 +7,5 @@ export type ISO8601 = string
 export type RequiredKeys<T> = {
   [K in keyof Required<T>]: T[K]
 }
+
+export type WithRequired<T, K extends keyof T> = T & Required<Pick<T, K>>

yarn.lock

@@ -2031,10 +2031,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/handlebars-helpers@^0.13.0":
+"@budibase/handlebars-helpers@^0.13.1":
-  version "0.13.0"
+  version "0.13.1"
-  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.0.tgz#224333d14e3900b7dacf48286af1e624a9fd62ea"
+  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.1.tgz#d02e73c0df8305cd675e70dc37f8427eb0842080"
-  integrity sha512-g8+sFrMNxsIDnK+MmdUICTVGr6ReUFtnPp9hJX0VZwz1pN3Ynolpk/Qbu6rEWAvoU1sEqY1mXr9uo/+kEfeGbQ==
+  integrity sha512-v4RbXhr3igvK3i2pj5cNltu/4NMxdPIzcUt/o0RoInhesNH1VSLRdweSFr6/Y34fsCR5jHZ6vltdcz2RgrTKgw==
   dependencies:
     get-object "^0.2.0"
     get-value "^3.0.1"
@@ -2050,10 +2050,10 @@
     to-gfm-code-block "^0.1.1"
     uuid "^9.0.1"
 
-"@budibase/nano@10.1.4":
+"@budibase/nano@10.1.5":
-  version "10.1.4"
+  version "10.1.5"
-  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.4.tgz#5c2670d0b4c12d736ddd6581c57d47c0aa45efad"
+  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.5.tgz#eeaded7bfc707ecabf8fde604425b865a90c06ec"
-  integrity sha512-J+IVaAljGideDvJss/AUxXA1599HEIUJo5c0LLlmc1KMA3GZWZjyX+w2fxAw3qF7hqFvX+qAStQgdcD3+/GPMA==
+  integrity sha512-q1eKIsYKo+iK17zsJYd3VBl+5ufQMPpHYLec0wVsid8wnJVrTQk7RNpBlBUn/EDgXM7t8XNNHlERqHu+CxJu8Q==
   dependencies:
     "@types/tough-cookie" "^4.0.2"
     axios "^1.1.3"