Merge branch 'develop' of github.com:Budibase/budibase into global-bindings

.eslintignore

@@ -7,7 +7,4 @@ packages/worker/coverage
 packages/backend-core/coverage
 packages/server/client
 packages/builder/.routify
-packages/builder/cypress/support/queryLevelTransformerFunction.js
-packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
-packages/builder/cypress/reports
 packages/sdk/sdk

.github/workflows/README.md

@@ -1,4 +1,3 @@
-
 # Budibase CI Pipelines
 
 Welcome to the budibase CI pipelines directory. This document details what each of the CI pipelines are for, and come common combinations.
@@ -6,27 +5,34 @@ Welcome to the budibase CI pipelines directory. This document details what each
 ## All CI Pipelines
 
 ### Note
-- When running workflow dispatch jobs, ensure you always run them off the `master` branch. It defaults to `develop`, so double check before running any jobs. The exception to this case is the `deploy-release` job which requires the develop branch. 
+
+- When running workflow dispatch jobs, ensure you always run them off the `master` branch. It defaults to `develop`, so double check before running any jobs. The exception to this case is the `deploy-release` job which requires the develop branch.
 
 ### Standard CI Build Job (budibase_ci.yml)
+
 Triggers:
+
 - PR or push to develop
 - PR or push to master
 
-The standard CI Build job is what runs when you raise a PR to develop or master. 
+The standard CI Build job is what runs when you raise a PR to develop or master.
+
 - Installs all dependencies,
-- builds the project 
+- builds the project
 - run the unit tests
 - Generate test coverage metrics with codecov
-- Run the cypress tests
+- Run the integration tests
 
 ### Release Develop Job (release-develop.yml)
+
 Triggers:
+
 - Push to develop
 
-The job responsible for building, tagging and pushing docker images out to the test and release environments. 
+The job responsible for building, tagging and pushing docker images out to the test and release environments.
+
 - Installs all dependencies
-- builds the project 
+- builds the project
 - run the unit tests
 - publish the budibase JS packages under a prerelease tag to NPM
 - build, tag and push docker images under the `develop` tag to docker hub
@@ -34,23 +40,29 @@ The job responsible for building, tagging and pushing docker images out to the t
 These images will then be pulled by the test and release environments, updating the latest automatically. Discord notifications are sent to the #infra channel when this occurs.
 
 ### Release Job (release.yml)
+
 Triggers:
+
 - Push to master
 
 This job is responsible for building and pushing the latest code to NPM and docker hub, so that it can be deployed.
+
 - Installs all dependencies
-- builds the project 
+- builds the project
 - run the unit tests
 - publish the budibase JS packages under a release tag to NPM (always incremented by patch versions)
 - build, tag and push docker images under the `v.x.x.x` (the tag of the NPM release) tag to docker hub
 
 ### Release Selfhost Job (release-selfhost.yml)
+
 Triggers:
+
 - Manual Workflow Dispatch Trigger
 
-This job is responsible for delivering the latest version of budibase to those that are self-hosting. 
+This job is responsible for delivering the latest version of budibase to those that are self-hosting.
 
 This job relies on the release job to have run first, so the latest image is pushed to dockerhub. This job then will pull the latest version from `lerna.json` and try to find an image in dockerhub corresponding to that version. For example, if the version in `lerna.json` is `1.0.0`:
+
 - Pull the images for all budibase services tagged `v1.0.0` from dockerhub
 - Tag these images as `latest`
 - Push them back to dockerhub. This now means anyone who pulls `latest` (self hosters using docker-compose) will get the latest version.
@@ -58,53 +70,61 @@ This job relies on the release job to have run first, so the latest image is pus
 - Perform a github release with the latest version. You can see previous releases here (https://github.com/Budibase/budibase/releases)
 
 ### Deploy Release (deploy-release.yml)
+
 Triggers:
+
 - Manual Workflow Dispatch Trigger
 
 This job is responsible for deploying to our release, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. After kicking off this job, the following will occur:
 
-- Checks out the release branch 
+- Checks out the release branch
 - Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration
 - Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off
-- Configures AWS Credentials  
+- Configures AWS Credentials
 - Deploys the helm chart in the budibase repo to our preproduction EKS cluster, injecting the `values.yaml` we pulled from budibase-infra
 - Fires off a discord webhook in the #infra channel to show that the deployment completely successfully.
 
 ### Deploy Preprod (deploy-preprod.yml)
+
 Triggers:
+
 - Manual Workflow Dispatch Trigger
 
 This job is responsible for deploying to our preprod, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. After kicking off this job, the following will occur:
 
-- Checks out the master branch 
+- Checks out the master branch
 - Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration
 - Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off
-- Configures AWS Credentials  
+- Configures AWS Credentials
 - Deploys the helm chart in the budibase repo to our preprod EKS cluster, injecting the `values.yaml` we pulled from budibase-infra
 - Fires off a discord webhook in the #infra channel to show that the deployment completely successfully.
 
 ### Deploy Production (deploy-cloud.yml)
+
 Triggers:
+
 - Manual Workflow Dispatch Trigger
 
 This job is responsible for deploying to our production, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. You can also manually enter a version number for this job, so you can perform rollbacks or upgrade to a specific version. After kicking off this job, the following will occur:
 
-- Checks out the master branch 
+- Checks out the master branch
 - Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration
 - Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off
-- Configures AWS Credentials  
+- Configures AWS Credentials
 - Deploys the helm chart in the budibase repo to our production EKS cluster, injecting the `values.yaml` we pulled from budibase-infra
 - Fires off a discord webhook in the #infra channel to show that the deployment completely successfully.
 
 ## Common Workflows
 
 ### Deploy Changes to Production (Release)
+
 - Merge `develop` into `master`
 - Wait for budibase CI job and release job to run
 - Run cloud deploy job
 - Run release selfhost job
 
 ### Deploy Changes to Production (Hotfix)
+
 - Branch off `master`
 - Perform your hotfix
 - Merge back into `master`
@@ -113,79 +133,7 @@ This job is responsible for deploying to our production, cloud kubernetes enviro
 - Run release selfhost job
 
 ### Rollback A Bad Cloud Deployment
+
 - Kick off cloud deploy job
 - Ensure you are running off master
 - Enter the version number of the last known good version of budibase. For example `1.0.0`
-
-## Pro
-
-| **NOTE**: When developing for both pro / budibase repositories, your branch names need to match, or else the correct pro doesn't get run within your CI job.
-
-### Installing Pro
-
-The pro package is always installed from source in our CI jobs. 
-
-This is done to prevent pro needing to be published prior to CI runs in budiabse. This is required for two reasons:
-- To reduce developer need to manually bump versions, i.e: 
-  - release pro, bump pro dep in budibase, now ci can run successfully
-- The cyclic dependency on backend-core, i.e:
-  - pro depends on backend-core
-  - server depends on pro
-  - backend-core lives in the monorepo, so it can't be released independently to be used in pro
-  - therefore the only option is to pull pro from source and release it as a part of the monorepo release, as if it were a mono package
-
-The install is performed using the same steps as local development, via the `yarn bootstrap` command, see the [Contributing Guide#Pro](../../docs/CONTRIBUTING.md#pro)
-
-The branch to install pro from can vary depending on ref of the commit that triggered the budibase CI job. This is done to enable branches which have changes in both the monorepo and the pro repo to have their CI pass successfully. 
-
-This is done using the [pro/install.sh](../../scripts/pro/install.sh) script. The script will:
-- Clone pro to it's default branch (`develop`)
-- Check if the clone worked, on forked versions of budibase this will fail due to no access
-  - This is fine as the `yarn` command will install the version from NPM
-  - Community PRs should never touch pro so this will always work
-- Checkout the `BRANCH` argument, if this fails fallback to `BASE_BRANCH`
-  - This enables the more complex case of a feature branch being merged to another feature branch, e.g. 
-    - I am working on a branch `epic/stonks` which exists on budibase and pro. 
-    - I want to merge a change to this branch in budibase from `feature/stonks-ui`, which only exists in budibase
-    - The base branch ensures that `epic/stonks` in pro will still be checked out for the CI run, rather than falling back to `develop`
-- Run `yarn setup` to build and install dependencies
-  - `yarn` 
-  - `yarn bootstrap`
-  - `yarn build`
-    - The will build .ts files, and also update the `main` and `types` of `package.json` to point to `dist` rather than src
-    - The build command will only ever work in CI, it is prevented in local dev
-
-#### `BRANCH` and `BASE_BRANCH` arguments
-These arguments are supplied by the various budibase build and release pipelines
-- `budibase_ci` 
-  - `BRANCH: ${{ github.event.pull_request.head.ref }}` -> The branch being merged
-  - `BASE_BRANCH: ${{ github.event.pull_request.base.ref}}` -> The base branch
-- `release-develop`
-  - `BRANCH: develop` -> always use the `develop` branch in pro
-- `release`
-  - `BRANCH: master` -> always use the `master` branch in pro 
-
-
-### Releasing Pro
-After budibase dependencies have been released we will release the new version of pro to match the release version of budibase dependencies. This is to ensure that we are always keeping the version of `backend-core` in sync in the pro package and in budibase packages. Without this we could run into scenarios where different versions are being used when installed via `yarn` inside the docker images, creating very difficult to debug cases.
-
-Pro is released using the [pro/release.sh](../../scripts/pro/release.sh) script. The script will:
-- Inspect the `VERSION` from the `lerna.json` file in budibase
-- Determine whether to use the `latest` or `develop` tag based on the command argument
-- Go to pro directory
-  - install npm creds
-  - update the version of `backend-core` to be `VERSION`, the version just released by lerna
-  - publish to npm. Uses a `lerna publish` command, pro itself is a mono repo. 
-    - force the version to be the same as `VERSION` to keep pro and budibase in sync
-  - reverts the changes to `main` and `types` in `package.json` that were made by the build step, to point back to source
-  - commit & push: `Prep next development iteration`
-- Go to budibase
-  - Update to the new version of pro in `server` and `worker` so the latest pro version is used in the docker builds
-  - commit & push: `Update pro version to $VERSION`
-
-
-#### `COMMAND` argument
-This argument is supplied by the existing `release` and `release:develop` budibase commands, which invoke the pro release
-- `release` will supply no command and default to use `latest`
-- `release:develop` will supply `develop` 
-

.github/workflows/budibase_ci.yml

@@ -18,6 +18,8 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  NX_BASE_BRANCH: origin/${{ github.base_ref }}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
 
 jobs:
   lint:
@@ -25,20 +27,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn lint
 
   build:
@@ -46,45 +48,66 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
+
       # Run build all the projects
-      - run: yarn build
+      - name: Build
+        run: |
+          yarn build
       # Check the types of the projects built via esbuild
-      - run: yarn check:types
+      - name: Check types
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn check:types
+          fi
 
   test-libraries:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+      - name: Test
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+          fi
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -96,21 +119,31 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
+      - name: Test worker and server
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/worker --scope=@budibase/server
+          fi
+
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -119,42 +152,50 @@ jobs:
 
   test-pro:
     runs-on: ubuntu-latest
-    if: github.repository == 'Budibase/budibase'
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/pro
+      - name: Test
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/pro
+          fi
 
   integration-test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client
+      - name: Build packages
+        run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core
       - name: Run tests
         run: |
           cd qa-core
@@ -166,14 +207,14 @@ jobs:
 
   check-pro-submodule:
     runs-on: ubuntu-latest
-    if: github.repository == 'Budibase/budibase'
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 0
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
 
       - name: Check pro commit
         id: get_pro_commits
@@ -211,4 +252,4 @@ jobs:
               process.exit(1);
             } else {
               console.log('All good, the submodule had been merged and setup correctly!')
-            }
+            }

.github/workflows/deploy-featurebranch.yml

@@ -0,0 +1,19 @@
+name: deploy-featurebranch
+
+on:
+  pull_request:
+    branches:
+      - develop
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+            BRANCH: ${{ github.head_ref }}
+        with:
+          repository: budibase/budibase-deploys
+          event: featurebranch-qa-deploy
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-develop.yml

@@ -44,7 +44,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - run: yarn install --frozen-lockfile
       - name: Update versions

.github/workflows/release-master.yml

@@ -36,7 +36,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - run: yarn install --frozen-lockfile
       - name: Update versions
@@ -60,9 +60,9 @@ jobs:
       - name: "Get Current tag"
         id: currenttag
         run: |
-          version=v$(./scripts/getCurrentVersion.sh)
+          version=$(./scripts/getCurrentVersion.sh)
-          echo 'Using tag $version'
+          echo "Using tag $version"
-          echo "::set-output name=tag::$resversionult"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
 
       - name: Build/release Docker images
         run: |
@@ -71,7 +71,7 @@ jobs:
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.tag }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

.github/workflows/release-selfhost.yml

@@ -28,10 +28,10 @@ jobs:
             exit 1
           fi
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - name: Get the latest budibase release version
         id: version
@@ -67,7 +67,6 @@ jobs:
       - name: Bootstrap and build (CLI)
         run: |
           yarn
-          yarn bootstrap
           yarn build
 
       - name: Build OpenAPI spec

.github/workflows/release-singleimage.yml

@@ -1,4 +1,4 @@
-name: release-singleimage
+name: Deploy Budibase Single Container Image to DockerHub
 
 on:
   workflow_dispatch:
@@ -8,13 +8,20 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   REGISTRY_URL: registry.hub.docker.com
 jobs:
-  build-amd64:
+  build:
-    name: "build-amd64"
+    name: "build"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 
@@ -27,12 +34,14 @@ jobs:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
+
       - name: Fail if tag is not in master
         run: |
           if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
             echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
             exit 1
           fi
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
@@ -68,139 +77,9 @@ jobs:
         with:
           context: .
           push: true
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
-          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
+          tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile
-
-      - name: Tag and release Budibase Azure App Service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64
-          build-args: TARGETBUILD=aas
-          tags: budibase/budibase-aas,budibase/budibase-aas:v${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
-
-  build-arm64:
-    name: "build-arm64"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build:docker:pre
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-      - name: Tag and release Budibase service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/arm64
-          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
-
-  build-aas:
-    name: "build-aas"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build:docker:pre
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:

.gitignore

@@ -97,8 +97,6 @@ typings/
 
 bin/
 hosting/.generated*
-packages/builder/cypress.env.json
-packages/builder/cypress/reports
 stats.html
 
 

.nvmrc

@@ -1 +1 @@
-v14.20.1
+v18.17.0

.prettierignore

@@ -9,6 +9,4 @@ packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify
-packages/builder/cypress/support/queryLevelTransformerFunction.js
-packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
 packages/sdk/sdk

.tool-versions

@@ -1,3 +1,3 @@
-nodejs 14.21.3
+nodejs 18.17.0
 python 3.10.0
-yarn 1.22.19
+yarn 1.22.19

.vscode/launch.json

@@ -1,42 +1,31 @@
+
 {
-    // Use IntelliSense to learn about possible attributes.
+  // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
+  // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
+  "version": "0.2.0",
-    "configurations": [
+  "configurations": [
-        {
+    {
-            "name": "Budibase Server",
+      "name": "Budibase Server",
-            "type": "node",
+      "type": "node",
-            "request": "launch",
+      "request": "launch",
-            "runtimeArgs": [
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
-                "--nolazy",
+      "args": ["${workspaceFolder}/packages/server/src/index.ts"],
-                "-r",
+      "cwd": "${workspaceFolder}/packages/server"
-                "ts-node/register/transpile-only"
+    },
-            ],
+    {
-            "args": [
+      "name": "Budibase Worker",
-                "${workspaceFolder}/packages/server/src/index.ts"
+      "type": "node",
-            ],
+      "request": "launch",
-            "cwd": "${workspaceFolder}/packages/server"
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
-        },
+      "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
-        {
+      "cwd": "${workspaceFolder}/packages/worker"
-            "name": "Budibase Worker",
+    }
-            "type": "node",
+  ],
-            "request": "launch",
+  "compounds": [
-            "runtimeArgs": [
+    {
-                "--nolazy",
+      "name": "Start Budibase",
-                "-r",
+      "configurations": ["Budibase Server", "Budibase Worker"]
-                "ts-node/register/transpile-only"
+    }
-            ],
+  ]
-            "args": [
+}
-                "${workspaceFolder}/packages/worker/src/index.ts"
-            ],
-            "cwd": "${workspaceFolder}/packages/worker"
-        },
-    ],
-    "compounds": [
-        {
-          "name": "Start Budibase",
-          "configurations": ["Budibase Server", "Budibase Worker"]
-        }
-    ]
-}

charts/budibase/templates/app-service-deployment.yaml

@@ -120,6 +120,8 @@ spec:
         {{ end }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.apps.logLevel | quote }}
         - name: REDIS_PASSWORD

charts/budibase/templates/secrets.yaml

@@ -1,4 +1,5 @@
-{{- if .Values.globals.createSecrets -}}
+{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (include "budibase.fullname" .) }}
+{{- if .Values.globals.createSecrets }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -10,8 +11,15 @@ metadata:
     heritage: "{{ .Release.Service }}"
 type: Opaque
 data:
+  {{- if $existingSecret }}
+  internalApiKey: {{ index $existingSecret.data "internalApiKey" }}
+  jwtSecret: {{ index $existingSecret.data "jwtSecret" }}
+  objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" }}
+  objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" }}
+  {{- else }}
   internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }}
   jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }}
   objectStoreAccess: {{ template "budibase.defaultsecret" .Values.services.objectStore.accessKey }}
   objectStoreSecret: {{ template "budibase.defaultsecret" .Values.services.objectStore.secretKey }}
-{{- end -}}
+  {{- end }}
+{{- end }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -116,6 +116,8 @@ spec:
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.worker.logLevel | quote }}
         - name: REDIS_PASSWORD

charts/budibase/values.yaml

@@ -82,6 +82,7 @@ globals:
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  offlineMode: "0" # set to 1 to enable offline mode
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""
@@ -136,7 +137,6 @@ services:
         path: /health
         port: 10000
         scheme: HTTP
-      enabled: true
       periodSeconds: 3
       failureThreshold: 1
     livenessProbe:
@@ -169,7 +169,6 @@ services:
         path: /health
         port: 4002
         scheme: HTTP
-      enabled: true
       periodSeconds: 3
       failureThreshold: 1
     livenessProbe:
@@ -203,7 +202,6 @@ services:
         path: /health
         port: 4003
         scheme: HTTP
-      enabled: true
       periodSeconds: 3
       failureThreshold: 1
     livenessProbe:
@@ -410,14 +408,12 @@ couchdb:
   ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
   # FOR COUCHDB
   livenessProbe:
-    enabled: true
     failureThreshold: 3
     initialDelaySeconds: 0
     periodSeconds: 10
     successThreshold: 1
     timeoutSeconds: 1
   readinessProbe:
-    enabled: true
     failureThreshold: 3
     initialDelaySeconds: 0
     periodSeconds: 10

docs/CONTRIBUTING.md

@@ -90,7 +90,7 @@ Component libraries are collections of components as well as the definition of t
 
 #### 1. Prerequisites
 
-- NodeJS version `14.x.x`
+- NodeJS version `18.x.x`
 - Python version `3.x`
 
 ### Using asdf (recommended)
@@ -264,16 +264,14 @@ Sometimes, things go wrong. This can be due to incompatible updates on the budib
 
 ### Running tests
 
-#### End-to-end Tests
+#### Unit Tests
 
-Budibase uses Cypress to run a number of E2E tests. To run the tests execute the following command in the root folder:
+Budibase uses Jest to run a number of tests. To run the tests execute the following command in the root folder:
 
 ```
-yarn test:e2e
+yarn test
 ```
 
-Or if you are in the builder you can run `yarn cy:test`.
-
 ### Other Useful Information
 
 - The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).

docs/DEV-SETUP-DEBIAN.md

@@ -55,7 +55,7 @@ yarn setup
 The yarn setup command runs several build steps i.e.
 
 ```
-node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
+node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev
 ```
 
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.

docs/DEV-SETUP-MACOSX.md

@@ -55,7 +55,7 @@ yarn setup
 The yarn setup command runs several build steps i.e.
 
 ```
-node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
+node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev
 ```
 
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.

docs/DEV-SETUP-WINDOWS.md

@@ -74,7 +74,7 @@ yarn setup
 The yarn setup command runs several build steps i.e.
 
 ```
-node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
+node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev
 ```
 
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.

hosting/couchdb/Dockerfile

@@ -5,11 +5,11 @@ ENV COUCHDB_PASSWORD admin
 EXPOSE 5984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
-    wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \
+    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo apt-key add - && \
     apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
     apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
-    apt-add-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \
+    apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \
-    apt-get update && apt-get install -y --no-install-recommends adoptopenjdk-8-hotspot && \
+    apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
     rm -rf /var/lib/apt/lists/
 
 # setup clouseau

hosting/docker-compose.test.yaml

@@ -1,47 +0,0 @@
-version: "3"
-
-# optional ports are specified throughout for more advanced use cases.
-
-services:
-  minio-service:
-    restart: on-failure
-    # Last version that supports the "fs" backend
-    image: minio/minio:RELEASE.2022-10-24T18-35-07Z
-    ports:
-      - "9000"
-      - "9001"
-    environment:
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
-    command: server /data --console-address ":9001"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-
-  couchdb-service:
-    # platform: linux/amd64
-    restart: on-failure
-    image: budibase/couchdb
-    environment:
-      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
-      - COUCHDB_USER=${COUCH_DB_USER}
-    ports:
-      - "5984"
-      - "4369"
-      - "9100"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5984/_up"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-
-  redis-service:
-    restart: on-failure
-    image: redis
-    command: redis-server --requirepass ${REDIS_PASSWORD}
-    ports:
-      - "6379"
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]

hosting/docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
+      OFFLINE_MODE: ${OFFLINE_MODE}
     depends_on:
       - worker-service
       - redis-service
@@ -54,6 +55,7 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
+      OFFLINE_MODE: ${OFFLINE_MODE}
     depends_on:
       - redis-service
       - minio-service

hosting/single/Dockerfile

@@ -1,7 +1,7 @@
-FROM node:14-slim as build
+FROM node:18-slim as build
 
 # install node-gyp dependencies
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python3
 
 # add pin script
 WORKDIR /

hosting/single/README.md

@@ -58,7 +58,6 @@ Node setup:
 ```
 node ./hosting/scripts/setup.js
 yarn
-yarn bootstrap
 yarn build
 ```
 #### Build Image

hosting/tests/README.md

@@ -47,7 +47,6 @@ Node setup:
 ```
 node ./hosting/scripts/setup.js
 yarn
-yarn bootstrap
 yarn build
 ```
 #### Build Image

jestTestcontainersConfigGenerator.js

@@ -1,9 +1,16 @@
 module.exports = () => {
   return {
-    dockerCompose: {
+    couchdb: {
-      composeFilePath: "../../hosting",
+      image: "budibase/couchdb",
-      composeFile: "docker-compose.test.yaml",
+      ports: [5984],
-      startupTimeout: 10000,
+      env: {
-    },
+        COUCHDB_PASSWORD: "budibase",
+        COUCHDB_USER: "budibase",
+      },
+      wait: {
+        type: "ports",
+        timeout: 20000,
+      }
+    }
   }
 }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.32-alpha.4",
+  "version": "2.10.3-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -9,7 +9,6 @@
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
-    "eslint-plugin-cypress": "^2.11.3",
     "husky": "^8.0.3",
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
@@ -33,25 +32,22 @@
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
-    "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
+    "build": "lerna run build --stream",
-    "build": "yarn nx run-many -t=build",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "check:types": "lerna run check:types",
-    "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
-    "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
     "release:develop": "yarn release --dist-tag develop",
-    "restore": "yarn run clean && yarn run bootstrap && yarn run build",
+    "restore": "yarn run clean && yarn && yarn run build",
     "nuke": "yarn run nuke:packages && yarn run nuke:docker",
     "nuke:packages": "yarn run restore",
     "nuke:docker": "lerna run --stream dev:stack:nuke",
-    "clean": "lerna clean",
+    "clean": "lerna clean -y",
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna run --stream dev:builder",
+    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev:builder",
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
@@ -93,9 +89,8 @@
     "mode:account": "yarn mode:cloud && yarn env:account:enable",
     "security:audit": "node scripts/audit.js",
     "postinstall": "husky install",
-    "dep:clean": "yarn clean -y && yarn bootstrap",
+    "submodules:load": "git submodule init && git submodule update && yarn",
-    "submodules:load": "git submodule init && git submodule update && yarn && yarn bootstrap",
+    "submodules:unload": "git submodule deinit --all && yarn"
-    "submodules:unload": "git submodule deinit --all && yarn && yarn bootstrap"
   },
   "workspaces": {
     "packages": [
@@ -109,7 +104,7 @@
     "@budibase/types": "0.0.0"
   },
   "engines": {
-    "node": ">=14.0.0 <15.0.0"
+    "node": ">=18.0.0 <19.0.0"
   },
   "dependencies": {}
 }

packages/backend-core/.npmignore

@@ -0,0 +1,6 @@
+*
+!dist/**/*
+dist/tsconfig.build.tsbuildinfo
+!package.json
+!src/**
+!tests/**

packages/backend-core/package.json

@@ -2,10 +2,10 @@
   "name": "@budibase/backend-core",
   "version": "0.0.0",
   "description": "Budibase backend core libraries used in server and worker",
-  "main": "dist/src/index.js",
+  "main": "dist/index.js",
   "types": "dist/src/index.d.ts",
   "exports": {
-    ".": "./dist/src/index.js",
+    ".": "./dist/index.js",
     "./tests": "./dist/tests/index.js",
     "./*": "./dist/*.js"
   },
@@ -14,7 +14,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
@@ -68,8 +68,8 @@
     "@types/jest": "29.5.3",
     "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
-    "@types/node": "14.18.20",
+    "@types/node": "18.17.0",
-    "@types/node-fetch": "2.6.1",
+    "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",
     "@types/semver": "7.3.7",
@@ -88,5 +88,20 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

packages/backend-core/plugins.ts

@@ -1 +0,0 @@
-export * from "./src/plugin"

packages/backend-core/scripts/build.js

@@ -0,0 +1,5 @@
+#!/usr/bin/node
+const coreBuild = require("../../../scripts/build")
+
+coreBuild("./src/plugin/index.ts", "./dist/plugins.js")
+coreBuild("./src/index.ts", "./dist/index.js")

packages/backend-core/src/cache/appMetadata.ts

@@ -55,7 +55,7 @@ export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
         throw err
       }
     }
-    // needed for cypress/some scenarios where the caching happens
+    // needed for some scenarios where the caching happens
     // so quickly the requests can get slightly out of sync
     // might store its invalid just before it stores its valid
     if (isInvalid(metadata)) {

packages/backend-core/src/cache/user.ts

@@ -4,6 +4,8 @@ import * as context from "../context"
 import * as platform from "../platform"
 import env from "../environment"
 import * as accounts from "../accounts"
+import { UserDB } from "../users"
+import { sdk } from "@budibase/shared-core"
 
 const EXPIRY_SECONDS = 3600
 
@@ -60,6 +62,18 @@ export async function getUser(
     // make sure the tenant ID is always correct/set
     user.tenantId = tenantId
   }
+  // if has groups, could have builder permissions granted by a group
+  if (user.userGroups && !sdk.users.isGlobalBuilder(user)) {
+    await context.doInTenant(tenantId, async () => {
+      const appIds = await UserDB.getGroupBuilderAppIds(user)
+      if (appIds.length) {
+        const existing = user.builder?.apps || []
+        user.builder = {
+          apps: [...new Set(existing.concat(appIds))],
+        }
+      }
+    })
+  }
   return user
 }
 

packages/backend-core/src/constants/misc.ts

@@ -22,6 +22,8 @@ export enum Header {
   TENANT_ID = "x-budibase-tenant-id",
   VERIFICATION_CODE = "x-budibase-verification-code",
   RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
+  RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
+  RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
   CORRELATION_ID = "x-budibase-correlation-id",

packages/backend-core/src/db/db.ts

@@ -11,7 +11,11 @@ export function getDB(dbName?: string, opts?: any): Database {
 // we have to use a callback for this so that we can close
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
-export async function doWithDB(dbName: string, cb: any, opts = {}) {
+export async function doWithDB<T>(
+  dbName: string,
+  cb: (db: Database) => Promise<T>,
+  opts = {}
+) {
   const db = getDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
   // async operations have been completed

packages/backend-core/src/db/lucene.ts

@@ -1,7 +1,6 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
-import { SearchFilters, Row } from "@budibase/types"
+import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
-import { createUserIndex } from "./searchIndexes/searchIndexes"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
@@ -65,6 +64,7 @@ export class QueryBuilder<T> {
     this.#index = index
     this.#query = {
       allOr: false,
+      onEmptyFilter: EmptyFilterOption.RETURN_ALL,
       string: {},
       fuzzy: {},
       range: {},
@@ -218,6 +218,10 @@ export class QueryBuilder<T> {
     this.#query.allOr = true
   }
 
+  setOnEmptyFilter(value: EmptyFilterOption) {
+    this.#query.onEmptyFilter = value
+  }
+
   handleSpaces(input: string) {
     if (this.#noEscaping) {
       return input
@@ -289,8 +293,9 @@ export class QueryBuilder<T> {
     const builder = this
     let allOr = this.#query && this.#query.allOr
     let query = allOr ? "" : "*:*"
+    let allFiltersEmpty = true
     const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
-    let tableId
+    let tableId: string = ""
     if (this.#query.equal!.tableId) {
       tableId = this.#query.equal!.tableId
       delete this.#query.equal!.tableId
@@ -305,7 +310,7 @@ export class QueryBuilder<T> {
     }
 
     const contains = (key: string, value: any, mode = "AND") => {
-      if (Array.isArray(value) && value.length === 0) {
+      if (!value || (Array.isArray(value) && value.length === 0)) {
         return null
       }
       if (!Array.isArray(value)) {
@@ -384,6 +389,12 @@ export class QueryBuilder<T> {
           built += ` ${mode} `
         }
         built += expression
+        if (
+          (typeof value !== "string" && value != null) ||
+          (typeof value === "string" && value !== tableId && value !== "")
+        ) {
+          allFiltersEmpty = false
+        }
       }
       if (opts?.returnBuilt) {
         return built
@@ -463,6 +474,13 @@ export class QueryBuilder<T> {
       allOr = false
       build({ tableId }, equal)
     }
+    if (allFiltersEmpty) {
+      if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
+        return ""
+      } else if (this.#query?.allOr) {
+        return query.replace("()", "(*:*)")
+      }
+    }
     return query
   }
 

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -1,6 +1,6 @@
 import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
-import { Database } from "@budibase/types"
+import { Database, EmptyFilterOption } from "@budibase/types"
 import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
 
 const INDEX_NAME = "main"
@@ -156,6 +156,76 @@ describe("lucene", () => {
       expect(resp.rows.length).toBe(2)
     })
 
+    describe("empty filters behaviour", () => {
+      it("should return all rows by default", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(3)
+      })
+
+      it("should return all rows when onEmptyFilter is ALL", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
+        builder.setAllOr()
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(3)
+      })
+
+      it("should return no rows when onEmptyFilter is NONE", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(0)
+      })
+
+      it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
+        builder.addEqual("property", "")
+        builder.addEqual("number", 1)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(1)
+      })
+    })
+
     describe("skip", () => {
       const skipDbName = `db-${newid()}`
       let docs: {

packages/backend-core/src/environment.ts

@@ -2,15 +2,15 @@ import { existsSync, readFileSync } from "fs"
 import { ServiceType } from "@budibase/types"
 
 function isTest() {
-  return isCypress() || isJest()
+  return isJest()
 }
 
 function isJest() {
-  return !!(process.env.NODE_ENV === "jest" || process.env.JEST_WORKER_ID)
+  return (
-}
+    process.env.NODE_ENV === "jest" ||
-
+    (process.env.JEST_WORKER_ID != null &&
-function isCypress() {
+      process.env.JEST_WORKER_ID !== "null")
-  return process.env.NODE_ENV === "cypress"
+  )
 }
 
 function isDev() {

packages/backend-core/src/features/index.ts

@@ -1,5 +1,6 @@
 import env from "../environment"
 import * as context from "../context"
+export * from "./installation"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.

packages/backend-core/src/features/installation.ts

@@ -0,0 +1,17 @@
+export function processFeatureEnvVar<T>(
+  fullList: string[],
+  featureList?: string
+) {
+  let list
+  if (!featureList) {
+    list = fullList
+  } else {
+    list = featureList.split(",")
+  }
+  for (let feature of list) {
+    if (!fullList.includes(feature)) {
+      throw new Error(`Feature: ${feature} is not an allowed option`)
+    }
+  }
+  return list as unknown as T[]
+}

packages/backend-core/src/index.ts

@@ -6,7 +6,8 @@ export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
 export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as featureFlags from "./featureFlags"
+export * as featureFlags from "./features"
+export * as features from "./features/installation"
 export * as sessions from "./security/sessions"
 export * as platform from "./platform"
 export * as auth from "./auth"

packages/backend-core/src/middleware/builderOnly.ts

@@ -5,11 +5,12 @@ import env from "../environment"
 
 export default async (ctx: UserCtx, next: any) => {
   const appId = getAppId()
-  const builderFn = env.isWorker()
+  const builderFn =
-    ? hasBuilderPermissions
+    env.isWorker() || !appId
-    : env.isApps()
+      ? hasBuilderPermissions
-    ? isBuilder
+      : env.isApps()
-    : undefined
+      ? isBuilder
+      : undefined
   if (!builderFn) {
     throw new Error("Service name unknown - middleware inactive.")
   }

packages/backend-core/src/middleware/builderOrAdmin.ts

@@ -5,11 +5,12 @@ import env from "../environment"
 
 export default async (ctx: UserCtx, next: any) => {
   const appId = getAppId()
-  const builderFn = env.isWorker()
+  const builderFn =
-    ? hasBuilderPermissions
+    env.isWorker() || !appId
-    : env.isApps()
+      ? hasBuilderPermissions
-    ? isBuilder
+      : env.isApps()
-    : undefined
+      ? isBuilder
+      : undefined
   if (!builderFn) {
     throw new Error("Service name unknown - middleware inactive.")
   }

packages/backend-core/src/security/permissions.ts

@@ -78,7 +78,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -87,8 +86,8 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
+      new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
     ],
   },
   POWER: {
@@ -98,8 +97,8 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.USER, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
+      new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
     ],
   },
   ADMIN: {
@@ -109,9 +108,9 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
       new Permission(PermissionType.USER, PermissionLevel.ADMIN),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
-      new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
+      new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
     ],
   },
 }

packages/backend-core/src/security/roles.ts

@@ -253,7 +253,7 @@ export function checkForRoleResourceArray(
  * Given an app ID this will retrieve all of the roles that are currently within that app.
  * @return {Promise<object[]>} An array of the role objects that were found.
  */
-export async function getAllRoles(appId?: string) {
+export async function getAllRoles(appId?: string): Promise<RoleDoc[]> {
   if (appId) {
     return doWithDB(appId, internal)
   } else {
@@ -312,37 +312,6 @@ export async function getAllRoles(appId?: string) {
   }
 }
 
-/**
- * This retrieves the required role for a resource
- * @param permLevel The level of request
- * @param resourceId The resource being requested
- * @param subResourceId The sub resource being requested
- * @return {Promise<{permissions}|Object>} returns the permissions required to access.
- */
-export async function getRequiredResourceRole(
-  permLevel: string,
-  { resourceId, subResourceId }: { resourceId?: string; subResourceId?: string }
-) {
-  const roles = await getAllRoles()
-  let main = [],
-    sub = []
-  for (let role of roles) {
-    // no permissions, ignore it
-    if (!role.permissions) {
-      continue
-    }
-    const mainRes = resourceId ? role.permissions[resourceId] : undefined
-    const subRes = subResourceId ? role.permissions[subResourceId] : undefined
-    if (mainRes && mainRes.indexOf(permLevel) !== -1) {
-      main.push(role._id)
-    } else if (subRes && subRes.indexOf(permLevel) !== -1) {
-      sub.push(role._id)
-    }
-  }
-  // for now just return the IDs
-  return main.concat(sub)
-}
-
 export class AccessController {
   userHierarchies: { [key: string]: string[] }
   constructor() {
@@ -411,8 +380,8 @@ export function getDBRoleID(roleName: string) {
 export function getExternalRoleID(roleId: string, version?: string) {
   // for built-in roles we want to remove the DB role ID element (role_)
   if (
-    (roleId.startsWith(DocumentType.ROLE) && isBuiltin(roleId)) ||
+    roleId.startsWith(DocumentType.ROLE) &&
-    version === RoleIDVersion.NAME
+    (isBuiltin(roleId) || version === RoleIDVersion.NAME)
   ) {
     return roleId.split(`${DocumentType.ROLE}${SEPARATOR}`)[1]
   }

packages/backend-core/src/users/db.ts

@@ -1,30 +1,32 @@
 import env from "../environment"
 import * as eventHelpers from "./events"
 import * as accounts from "../accounts"
+import * as accountSdk from "../accounts"
 import * as cache from "../cache"
-import { getIdentity, getTenantId, getGlobalDB } from "../context"
+import { getGlobalDB, getIdentity, getTenantId } from "../context"
 import * as dbUtils from "../db"
 import { EmailUnavailableError, HTTPError } from "../errors"
 import * as platform from "../platform"
 import * as sessions from "../security/sessions"
 import * as usersCore from "./users"
 import {
+  Account,
   AllDocsResponse,
   BulkUserCreated,
   BulkUserDeleted,
+  isSSOAccount,
+  isSSOUser,
   RowResponse,
   SaveUserOpts,
   User,
-  Account,
-  isSSOUser,
-  isSSOAccount,
   UserStatus,
+  UserGroup,
+  ContextUser,
 } from "@budibase/types"
-import * as accountSdk from "../accounts"
 import {
-  validateUniqueUser,
   getAccountHolderFromUserIds,
   isAdmin,
+  validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
@@ -32,8 +34,14 @@ import { hash } from "../utils"
 type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
+type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
+type GroupBuildersFn = (user: User) => Promise<string[]>
 type QuotaFns = { addUsers: QuotaUpdateFn; removeUsers: QuotaUpdateFn }
-type GroupFns = { addUsers: GroupUpdateFn }
+type GroupFns = {
+  addUsers: GroupUpdateFn
+  getBulk: GroupGetFn
+  getGroupBuilderAppIds: GroupBuildersFn
+}
 type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
 
 const bulkDeleteProcessing = async (dbUser: User) => {
@@ -179,6 +187,14 @@ export class UserDB {
     return user
   }
 
+  static async bulkGet(userIds: string[]) {
+    return await usersCore.bulkGetGlobalUsersById(userIds)
+  }
+
+  static async bulkUpdate(users: User[]) {
+    return await usersCore.bulkUpdateGlobalUsers(users)
+  }
+
   static async save(user: User, opts: SaveUserOpts = {}): Promise<User> {
     // default booleans to true
     if (opts.hashPassword == null) {
@@ -457,4 +473,12 @@ export class UserDB {
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })
   }
+
+  static async getGroups(groupIds: string[]) {
+    return await this.groups.getBulk(groupIds)
+  }
+
+  static async getGroupBuilderAppIds(user: User) {
+    return await this.groups.getGroupBuilderAppIds(user)
+  }
 }

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -86,6 +86,10 @@ export const useAuditLogs = () => {
   return useFeature(Feature.AUDIT_LOGS)
 }
 
+export const usePublicApiUserRoles = () => {
+  return useFeature(Feature.USER_ROLE_PUBLIC_API)
+}
+
 export const useScimIntegration = () => {
   return useFeature(Feature.SCIM)
 }
@@ -98,6 +102,10 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
+export const useViewPermissions = () => {
+  return useFeature(Feature.VIEW_PERMISSIONS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -32,8 +32,8 @@ function getTestContainerSettings(
 ): string | null {
   const entry = Object.entries(global).find(
     ([k]) =>
-      k.includes(`_${serverName.toUpperCase()}`) &&
+      k.includes(`${serverName.toUpperCase()}`) &&
-      k.includes(`_${key.toUpperCase()}__`)
+      k.includes(`${key.toUpperCase()}`)
   )
   if (!entry) {
     return null
@@ -67,27 +67,14 @@ function getContainerInfo(containerName: string, port: number) {
 }
 
 function getCouchConfig() {
-  return getContainerInfo("couchdb-service", 5984)
+  return getContainerInfo("couchdb", 5984)
-}
-
-function getMinioConfig() {
-  return getContainerInfo("minio-service", 9000)
-}
-
-function getRedisConfig() {
-  return getContainerInfo("redis-service", 6379)
 }
 
 export function setupEnv(...envs: any[]) {
-  const couch = getCouchConfig(),
+  const couch = getCouchConfig()
-    minio = getCouchConfig(),
-    redis = getRedisConfig()
   const configs = [
     { key: "COUCH_DB_PORT", value: couch.port },
     { key: "COUCH_DB_URL", value: couch.url },
-    { key: "MINIO_PORT", value: minio.port },
-    { key: "MINIO_URL", value: minio.url },
-    { key: "REDIS_URL", value: redis.url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tsconfig.build.json

@@ -12,7 +12,11 @@
     "declaration": true,
     "types": ["node", "jest"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/shared-core": ["../shared-core/src"]
+    }
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [

packages/backend-core/tsconfig.json

@@ -1,12 +1,4 @@
 {
   "extends": "./tsconfig.build.json",
-  "compilerOptions": {
-    "composite": true,
-    "baseUrl": ".",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/shared-core": ["../shared-core/src"]
-    }
-  },
   "exclude": ["node_modules", "dist"]
 }

packages/bbui/package.json

@@ -98,8 +98,7 @@
           {
             "projects": [
               "@budibase/string-templates",
-              "@budibase/shared-core",
+              "@budibase/shared-core"
-              "@budibase/types"
             ],
             "target": "build"
           }

packages/bbui/src/Actions/position_dropdown.js

@@ -17,6 +17,8 @@ export default function positionDropdown(element, opts) {
       maxWidth,
       useAnchorWidth,
       offset = 5,
+      customUpdate,
+      offsetBelow,
     } = opts
     if (!anchor) {
       return
@@ -33,33 +35,41 @@ export default function positionDropdown(element, opts) {
       top: null,
     }
 
-    // Determine vertical styles
+    if (typeof customUpdate === "function") {
-    if (align === "right-outside") {
+      styles = customUpdate(anchorBounds, elementBounds, styles)
-      styles.top = anchorBounds.top
-    } else if (window.innerHeight - anchorBounds.bottom < 100) {
-      styles.top = anchorBounds.top - elementBounds.height - offset
-      styles.maxHeight = maxHeight || 240
     } else {
-      styles.top = anchorBounds.bottom + offset
+      // Determine vertical styles
-      styles.maxHeight =
+      if (align === "right-outside") {
-        maxHeight || window.innerHeight - anchorBounds.bottom - 20
+        styles.top = anchorBounds.top
-    }
+      } else if (
+        window.innerHeight - anchorBounds.bottom <
+        (maxHeight || 100)
+      ) {
+        styles.top = anchorBounds.top - elementBounds.height - offset
+        styles.maxHeight = maxHeight || 240
+      } else {
+        styles.top = anchorBounds.bottom + (offsetBelow || offset)
+        styles.maxHeight =
+          maxHeight || window.innerHeight - anchorBounds.bottom - 20
+      }
 
-    // Determine horizontal styles
+      // Determine horizontal styles
-    if (!maxWidth && useAnchorWidth) {
+      if (!maxWidth && useAnchorWidth) {
-      styles.maxWidth = anchorBounds.width
+        styles.maxWidth = anchorBounds.width
-    }
+      }
-    if (useAnchorWidth) {
+      if (useAnchorWidth) {
-      styles.minWidth = anchorBounds.width
+        styles.minWidth = anchorBounds.width
-    }
+      }
-    if (align === "right") {
+      if (align === "right") {
-      styles.left = anchorBounds.left + anchorBounds.width - elementBounds.width
+        styles.left =
-    } else if (align === "right-outside") {
+          anchorBounds.left + anchorBounds.width - elementBounds.width
-      styles.left = anchorBounds.right + offset
+      } else if (align === "right-outside") {
-    } else if (align === "left-outside") {
+        styles.left = anchorBounds.right + offset
-      styles.left = anchorBounds.left - elementBounds.width - offset
+      } else if (align === "left-outside") {
-    } else {
+        styles.left = anchorBounds.left - elementBounds.width - offset
-      styles.left = anchorBounds.left
+      } else {
+        styles.left = anchorBounds.left
+      }
     }
 
     // Apply styles

packages/bbui/src/ColorPicker/ColorPicker.svelte

@@ -1,8 +1,8 @@
 <script>
+  import Popover from "../Popover/Popover.svelte"
+  import Layout from "../Layout/Layout.svelte"
   import { createEventDispatcher } from "svelte"
   import "@spectrum-css/popover/dist/index-vars.css"
-  import clickOutside from "../Actions/click_outside"
-  import { fly } from "svelte/transition"
   import Icon from "../Icon/Icon.svelte"
   import Input from "../Form/Input.svelte"
   import { capitalise } from "../helpers"
@@ -10,9 +10,11 @@
   export let value
   export let size = "M"
   export let spectrumTheme
-  export let alignRight = false
+  export let offset
+  export let align
 
-  let open = false
+  let dropdown
+  let preview
 
   $: customValue = getCustomValue(value)
   $: checkColor = getCheckColor(value)
@@ -82,7 +84,7 @@
 
   const onChange = value => {
     dispatch("change", value)
-    open = false
+    dropdown.hide()
   }
 
   const getCustomValue = value => {
@@ -119,30 +121,25 @@
 
     return "var(--spectrum-global-color-static-gray-900)"
   }
-
-  const handleOutsideClick = event => {
-    if (open) {
-      event.stopPropagation()
-      open = false
-    }
-  }
 </script>
 
-<div class="container">
+<div
-  <div class="preview size--{size || 'M'}" on:click={() => (open = true)}>
+  bind:this={preview}
-    <div
+  class="preview size--{size || 'M'}"
-      class="fill {spectrumTheme || ''}"
+  on:click={() => {
-      style={value ? `background: ${value};` : ""}
+    dropdown.toggle()
-      class:placeholder={!value}
+  }}
-    />
+>
-  </div>
+  <div
-  {#if open}
+    class="fill {spectrumTheme || ''}"
-    <div
+    style={value ? `background: ${value};` : ""}
-      use:clickOutside={handleOutsideClick}
+    class:placeholder={!value}
-      transition:fly|local={{ y: -20, duration: 200 }}
+  />
-      class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
+</div>
-      class:spectrum-Popover--align-right={alignRight}
+
-    >
+<Popover bind:this={dropdown} anchor={preview} maxHeight={320} {offset} {align}>
+  <Layout paddingX="XL" paddingY="L">
+    <div class="container">
       {#each categories as category}
         <div class="category">
           <div class="heading">{category.label}</div>
@@ -187,8 +184,8 @@
         </div>
       </div>
     </div>
-  {/if}
+  </Layout>
-</div>
+</Popover>
 
 <style>
   .container {
@@ -248,20 +245,6 @@
     width: 48px;
     height: 48px;
   }
-  .spectrum-Popover {
-    width: 210px;
-    z-index: 999;
-    top: 100%;
-    padding: var(--spacing-l) var(--spacing-xl);
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-    gap: var(--spacing-xl);
-  }
-  .spectrum-Popover--align-right {
-    right: 0;
-  }
   .colors {
     display: grid;
     grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr 1fr 1fr;
@@ -297,7 +280,11 @@
   .category--custom .heading {
     margin-bottom: var(--spacing-xs);
   }
-
+  .container {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-xl);
+  }
   .spectrum-wrapper {
     background-color: transparent;
   }

packages/bbui/src/DetailSummary/DetailSummary.svelte

@@ -44,7 +44,9 @@
     align-items: stretch;
     border-bottom: var(--border-light);
   }
-
+  .property-group-container:last-child {
+    border-bottom: 0px;
+  }
   .property-group-name {
     cursor: pointer;
     display: flex;

packages/bbui/src/Drawer/Drawer.svelte

@@ -4,6 +4,8 @@
   import Body from "../Typography/Body.svelte"
   import Heading from "../Typography/Heading.svelte"
   import { setContext } from "svelte"
+  import { createEventDispatcher } from "svelte"
+  import { generate } from "shortid"
 
   export let title
   export let fillWidth
@@ -11,13 +13,17 @@
   export let width = "calc(100% - 626px)"
   export let headless = false
 
+  const dispatch = createEventDispatcher()
+
   let visible = false
+  let drawerId = generate()
 
   export function show() {
     if (visible) {
       return
     }
     visible = true
+    dispatch("drawerShow", drawerId)
   }
 
   export function hide() {
@@ -25,6 +31,7 @@
       return
     }
     visible = false
+    dispatch("drawerHide", drawerId)
   }
 
   setContext("drawer-actions", {

packages/bbui/src/FancyForm/FancySelect.svelte

@@ -2,8 +2,9 @@
   import { createEventDispatcher } from "svelte"
   import FancyField from "./FancyField.svelte"
   import Icon from "../Icon/Icon.svelte"
-  import Popover from "../Popover/Popover.svelte"
   import FancyFieldLabel from "./FancyFieldLabel.svelte"
+  import StatusLight from "../StatusLight/StatusLight.svelte"
+  import Picker from "../Form/Core/Picker.svelte"
 
   export let label
   export let value
@@ -11,18 +12,30 @@
   export let error = null
   export let validate = null
   export let options = []
+  export let isOptionEnabled = () => true
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
-
+  export let getOptionSubtitle = option => extractProperty(option, "subtitle")
+  export let getOptionColour = () => null
   const dispatch = createEventDispatcher()
 
   let open = false
-  let popover
   let wrapper
 
   $: placeholder = !value
   $: selectedLabel = getSelectedLabel(value)
+  $: fieldColour = getFieldAttribute(getOptionColour, value, options)
 
+  const getFieldAttribute = (getAttribute, value, options) => {
+    // Wait for options to load if there is a value but no options
+    if (!options?.length) {
+      return ""
+    }
+    const index = options.findIndex(
+      (option, idx) => getOptionValue(option, idx) === value
+    )
+    return index !== -1 ? getAttribute(options[index], index) : null
+  }
   const extractProperty = (value, property) => {
     if (value && typeof value === "object") {
       return value[property]
@@ -64,46 +77,45 @@
     <FancyFieldLabel {placeholder}>{label}</FancyFieldLabel>
   {/if}
 
+  {#if fieldColour}
+    <span class="align">
+      <StatusLight square color={fieldColour} />
+    </span>
+  {/if}
+
   <div class="value" class:placeholder>
     {selectedLabel || ""}
   </div>
 
-  <div class="arrow">
+  <div class="align arrow-alignment">
     <Icon name="ChevronDown" />
   </div>
 </FancyField>
 
-<Popover
+<div id="picker-wrapper">
-  anchor={wrapper}
+  <Picker
-  align="left"
+    customAnchor={wrapper}
-  portalTarget={document.documentElement}
+    onlyPopover={true}
-  bind:this={popover}
+    bind:open
-  {open}
+    {error}
-  on:close={() => (open = false)}
+    {disabled}
-  useAnchorWidth={true}
+    {options}
-  maxWidth={null}
+    {getOptionLabel}
->
+    {getOptionValue}
-  <div class="popover-content">
+    {getOptionSubtitle}
-    {#if options.length}
+    {getOptionColour}
-      {#each options as option, idx}
+    {isOptionEnabled}
-        <div
+    isPlaceholder={value == null || value === ""}
-          class="popover-option"
+    placeholderOption={placeholder === false ? null : placeholder}
-          tabindex="0"
+    onSelectOption={onChange}
-          on:click={() => onChange(getOptionValue(option, idx))}
+    isOptionSelected={option => option === value}
-        >
+  />
-          <span class="option-text">
+</div>
-            {getOptionLabel(option, idx)}
-          </span>
-          {#if value === getOptionValue(option, idx)}
-            <Icon name="Checkmark" />
-          {/if}
-        </div>
-      {/each}
-    {/if}
-  </div>
-</Popover>
 
 <style>
+  #picker-wrapper :global(.spectrum-Picker) {
+    display: none;
+  }
   .value {
     display: block;
     flex: 1 1 auto;
@@ -118,30 +130,23 @@
     width: 0;
     transform: translateY(9px);
   }
+
+  .align {
+    display: block;
+    font-size: 15px;
+    line-height: 17px;
+    color: var(--spectrum-global-color-gray-900);
+    transition: transform 130ms ease-out, opacity 130ms ease-out;
+    transform: translateY(9px);
+  }
+
+  .arrow-alignment {
+    transform: translateY(-2px);
+  }
   .value.placeholder {
     transform: translateY(0);
     opacity: 0;
     pointer-events: none;
     margin-top: 0;
   }
-  .popover-content {
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-    padding: 7px 0;
-  }
-  .popover-option {
-    display: flex;
-    flex-direction: row;
-    justify-content: space-between;
-    align-items: center;
-    padding: 7px 16px;
-    transition: background 130ms ease-out;
-    font-size: 15px;
-  }
-  .popover-option:hover {
-    background: var(--spectrum-global-color-gray-200);
-    cursor: pointer;
-  }
 </style>

packages/bbui/src/Form/Core/Combobox.svelte

@@ -2,8 +2,8 @@
   import "@spectrum-css/inputgroup/dist/index-vars.css"
   import "@spectrum-css/popover/dist/index-vars.css"
   import "@spectrum-css/menu/dist/index-vars.css"
-  import { fly } from "svelte/transition"
   import { createEventDispatcher } from "svelte"
+  import clickOutside from "../../Actions/click_outside"
 
   export let value = null
   export let id = null
@@ -80,10 +80,11 @@
     </svg>
   </button>
   {#if open}
-    <div class="overlay" on:mousedown|self={() => (open = false)} />
     <div
-      transition:fly|local={{ y: -20, duration: 200 }}
       class="spectrum-Popover spectrum-Popover--bottom is-open"
+      use:clickOutside={() => {
+        open = false
+      }}
     >
       <ul class="spectrum-Menu" role="listbox">
         {#if options && Array.isArray(options)}
@@ -125,14 +126,6 @@
   .spectrum-Textfield-input {
     width: 0;
   }
-  .overlay {
-    position: fixed;
-    top: 0;
-    left: 0;
-    width: 100vw;
-    height: 100vh;
-    z-index: 999;
-  }
   .spectrum-Popover {
     max-height: 240px;
     width: 100%;

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -17,6 +17,9 @@
   export let fetchTerm = null
   export let useFetch = false
   export let customPopoverHeight
+  export let customPopoverOffsetBelow
+  export let customPopoverMaxHeight
+  export let open = false
 
   const dispatch = createEventDispatcher()
 
@@ -88,6 +91,7 @@
   isPlaceholder={!arrayValue.length}
   {autocomplete}
   bind:fetchTerm
+  bind:open
   {useFetch}
   {isOptionSelected}
   {getOptionLabel}
@@ -96,4 +100,6 @@
   {sort}
   {autoWidth}
   {customPopoverHeight}
+  {customPopoverOffsetBelow}
+  {customPopoverMaxHeight}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -8,6 +8,8 @@
   import Icon from "../../Icon/Icon.svelte"
   import StatusLight from "../../StatusLight/StatusLight.svelte"
   import Popover from "../../Popover/Popover.svelte"
+  import Tags from "../../Tags/Tags.svelte"
+  import Tag from "../../Tags/Tag.svelte"
 
   export let id = null
   export let disabled = false
@@ -26,6 +28,7 @@
   export let getOptionIcon = () => null
   export let useOptionIconImage = false
   export let getOptionColour = () => null
+  export let getOptionSubtitle = () => null
   export let open = false
   export let readonly = false
   export let quiet = false
@@ -35,9 +38,11 @@
   export let fetchTerm = null
   export let useFetch = false
   export let customPopoverHeight
+  export let customPopoverOffsetBelow
+  export let customPopoverMaxHeight
   export let align = "left"
   export let footer = null
-
+  export let customAnchor = null
   const dispatch = createEventDispatcher()
 
   let searchTerm = null
@@ -139,16 +144,17 @@
     <use xlink:href="#spectrum-css-icon-Chevron100" />
   </svg>
 </button>
-
 <Popover
-  anchor={button}
+  anchor={customAnchor ? customAnchor : button}
   align={align || "left"}
   bind:this={popover}
   {open}
   on:close={() => (open = false)}
   useAnchorWidth={!autoWidth}
   maxWidth={autoWidth ? 400 : null}
+  maxHeight={customPopoverMaxHeight}
   customHeight={customPopoverHeight}
+  offsetBelow={customPopoverOffsetBelow}
 >
   <div
     class="popover-content"
@@ -215,8 +221,21 @@
               </span>
             {/if}
             <span class="spectrum-Menu-itemLabel">
+              {#if getOptionSubtitle(option, idx)}
+                <span class="subtitle-text"
+                  >{getOptionSubtitle(option, idx)}</span
+                >
+              {/if}
+
               {getOptionLabel(option, idx)}
             </span>
+            {#if option.tag}
+              <span class="option-tag">
+                <Tags>
+                  <Tag icon="LockClosed">{option.tag}</Tag>
+                </Tags>
+              </span>
+            {/if}
             <svg
               class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
               focusable="false"
@@ -242,6 +261,17 @@
     width: 100%;
     box-shadow: none;
   }
+
+  .subtitle-text {
+    font-size: 12px;
+    line-height: 15px;
+    font-weight: 500;
+    top: 10px;
+    color: var(--spectrum-global-color-gray-600);
+    display: block;
+    margin-bottom: var(--spacing-s);
+  }
+
   .spectrum-Picker-label.auto-width {
     margin-right: var(--spacing-xs);
   }
@@ -321,4 +351,12 @@
   .option-extra.icon.field-icon {
     display: flex;
   }
+
+  .option-tag {
+    margin: 0 var(--spacing-m) 0 var(--spacing-m);
+  }
+
+  .option-tag :global(.spectrum-Tags-item > .spectrum-Icon) {
+    margin-top: 2px;
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -21,11 +21,13 @@
   export let sort = false
   export let align
   export let footer = null
+  export let open = false
+  export let tag = null
+  export let customPopoverOffsetBelow
+  export let customPopoverMaxHeight
 
   const dispatch = createEventDispatcher()
 
-  let open = false
-
   $: fieldText = getFieldText(value, options, placeholder)
   $: fieldIcon = getFieldAttribute(getOptionIcon, value, options)
   $: fieldColour = getFieldAttribute(getOptionColour, value, options)
@@ -83,6 +85,9 @@
   {isOptionEnabled}
   {autocomplete}
   {sort}
+  {tag}
+  {customPopoverOffsetBelow}
+  {customPopoverMaxHeight}
   isPlaceholder={value == null || value === ""}
   placeholderOption={placeholder === false ? null : placeholder}
   isOptionSelected={option => option === value}

packages/bbui/src/Form/Select.svelte

@@ -25,7 +25,7 @@
   export let customPopoverHeight
   export let align
   export let footer = null
-
+  export let tag = null
   const dispatch = createEventDispatcher()
   const onChange = e => {
     value = e.detail
@@ -61,6 +61,7 @@
     {isOptionEnabled}
     {autocomplete}
     {customPopoverHeight}
+    {tag}
     on:change={onChange}
     on:click
   />

packages/bbui/src/Modal/Modal.svelte

@@ -9,6 +9,7 @@
   export let fixed = false
   export let inline = false
   export let disableCancel = false
+  export let autoFocus = true
 
   const dispatch = createEventDispatcher()
   let visible = fixed || inline
@@ -53,6 +54,9 @@
   }
 
   async function focusModal(node) {
+    if (!autoFocus) {
+      return
+    }
     await tick()
 
     // Try to focus first input

packages/bbui/src/Popover/Popover.svelte

@@ -19,10 +19,15 @@
   export let useAnchorWidth = false
   export let dismissible = true
   export let offset = 5
+  export let offsetBelow
   export let customHeight
   export let animate = true
   export let customZindex
 
+  export let handlePostionUpdate
+  export let showPopover = true
+  export let clickOutsideOverride = false
+
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
 
   export const show = () => {
@@ -35,7 +40,18 @@
     open = false
   }
 
+  export const toggle = () => {
+    if (!open) {
+      show()
+    } else {
+      hide()
+    }
+  }
+
   const handleOutsideClick = e => {
+    if (clickOutsideOverride) {
+      return
+    }
     if (open) {
       // Stop propagation if the source is the anchor
       let node = e.target
@@ -54,6 +70,9 @@
   }
 
   function handleEscape(e) {
+    if (!clickOutsideOverride) {
+      return
+    }
     if (open && e.key === "Escape") {
       hide()
     }
@@ -71,6 +90,8 @@
         maxWidth,
         useAnchorWidth,
         offset,
+        offsetBelow,
+        customUpdate: handlePostionUpdate,
       }}
       use:clickOutside={{
         callback: dismissible ? handleOutsideClick : () => {},
@@ -79,6 +100,7 @@
       on:keydown={handleEscape}
       class="spectrum-Popover is-open"
       class:customZindex
+      class:hide-popover={open && !showPopover}
       role="presentation"
       style="height: {customHeight}; --customZindex: {customZindex};"
       transition:fly|local={{ y: -20, duration: animate ? 200 : 0 }}
@@ -89,6 +111,10 @@
 {/if}
 
 <style>
+  .hide-popover {
+    display: contents;
+  }
+
   .spectrum-Popover {
     min-width: var(--spectrum-global-dimension-size-2000);
     border-color: var(--spectrum-global-color-gray-300);

packages/bbui/src/Table/Table.svelte

@@ -215,7 +215,7 @@
         const nameA = getDisplayName(a)
         const nameB = getDisplayName(b)
         if (orderA !== orderB) {
-          return orderA < orderB ? orderA : orderB
+          return orderA < orderB ? a : b
         }
         return nameA < nameB ? a : b
       })

packages/bbui/src/Tabs/Tabs.svelte

@@ -57,10 +57,8 @@
   function calculateIndicatorLength() {
     if (!vertical) {
       width = $tab.info?.width + "px"
-      height = $tab.info?.height
     } else {
       height = $tab.info?.height + 4 + "px"
-      width = $tab.info?.width
     }
   }
 

packages/bbui/src/Tooltip/TempTooltip.svelte

@@ -4,7 +4,7 @@
 
   export let text = null
   export let condition = true
-  export let duration = 3000
+  export let duration = 5000
   export let position
   export let type
 

packages/builder/.gitignore

@@ -5,6 +5,4 @@ package-lock.json
 release/
 dist/
 routify
-cypress/videos
-cypress/screenshots
 .routify/

packages/builder/package.json

@@ -43,21 +43,10 @@
       "/node_modules/(?!svelte).+\\.js$",
       ".*string-templates.*"
     ],
-    "modulePathIgnorePatterns": [
-      "<rootDir>/cypress/"
-    ],
     "setupFilesAfterEnv": [
       "@testing-library/jest-dom/extend-expect"
     ]
   },
-  "eslintConfig": {
-    "extends": [
-      "plugin:cypress/recommended"
-    ],
-    "rules": {
-      "cypress/no-unnecessary-waiting": "off"
-    }
-  },
   "dependencies": {
     "@budibase/bbui": "0.0.0",
     "@budibase/frontend-core": "0.0.0",
@@ -104,9 +93,6 @@
     "@testing-library/jest-dom": "5.17.0",
     "@testing-library/svelte": "^3.2.2",
     "babel-jest": "29.6.2",
-    "cypress": "^9.3.1",
-    "cypress-multi-reporters": "^1.6.0",
-    "cypress-terminal-report": "^1.4.1",
     "identity-obj-proxy": "^3.0.0",
     "jest": "29.6.2",
     "jsdom": "^21.1.1",
@@ -133,9 +119,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
+              "@budibase/string-templates"
-              "@budibase/string-templates",
-              "@budibase/types"
             ],
             "target": "build"
           }
@@ -145,9 +129,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
+              "@budibase/string-templates"
-              "@budibase/string-templates",
-              "@budibase/types"
             ],
             "target": "build"
           }
@@ -157,9 +139,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
+              "@budibase/string-templates"
-              "@budibase/string-templates",
-              "@budibase/types"
             ],
             "target": "build"
           }

packages/builder/src/builderStore/dataBinding.js

@@ -6,7 +6,7 @@ import {
   findComponentPath,
   getComponentSettings,
 } from "./componentUtils"
-import { store } from "builderStore"
+import { store, currentAsset } from "builderStore"
 import {
   queries as queriesStores,
   tables as tablesStore,
@@ -22,6 +22,7 @@ import { TableNames } from "../constants"
 import { JSONUtils } from "@budibase/frontend-core"
 import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json"
 import { environment, licensing } from "stores/portal"
+import { convertOldFieldFormat } from "components/design/settings/controls/FieldConfiguration/utils"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
@@ -344,7 +345,7 @@ const getProviderContextBindings = (asset, dataProviders) => {
       if (context.type === "form") {
         // Forms do not need table schemas
         // Their schemas are built from their component field names
-        schema = buildFormSchema(component)
+        schema = buildFormSchema(component, asset)
         readablePrefix = "Fields"
       } else if (context.type === "static") {
         // Static contexts are fully defined by the components
@@ -366,12 +367,19 @@ const getProviderContextBindings = (asset, dataProviders) => {
         schema = info.schema
         table = info.table
 
-        // For JSON arrays, use the array name as the readable prefix.
+        // Determine what to prefix bindings with
-        // Otherwise use the table name
         if (datasource.type === "jsonarray") {
+          // For JSON arrays, use the array name as the readable prefix
           const split = datasource.label.split(".")
           readablePrefix = split[split.length - 1]
+        } else if (datasource.type === "viewV2") {
+          // For views, use the view name
+          const view = Object.values(table?.views || {}).find(
+            view => view.id === datasource.id
+          )
+          readablePrefix = view?.name
         } else {
+          // Otherwise use the table name
           readablePrefix = info.table?.name
         }
       }
@@ -386,6 +394,11 @@ const getProviderContextBindings = (asset, dataProviders) => {
       if (runtimeSuffix) {
         providerId += `-${runtimeSuffix}`
       }
+
+      if (!filterCategoryByContext(component, context)) {
+        return
+      }
+
       const safeComponentId = makePropSafe(providerId)
 
       // Create bindable properties for each schema field
@@ -403,6 +416,12 @@ const getProviderContextBindings = (asset, dataProviders) => {
         }
         readableBinding += `.${fieldSchema.name || key}`
 
+        const bindingCategory = getComponentBindingCategory(
+          component,
+          context,
+          def
+        )
+
         // Create the binding object
         bindings.push({
           type: "context",
@@ -415,8 +434,8 @@ const getProviderContextBindings = (asset, dataProviders) => {
           // Table ID is used by JSON fields to know what table the field is in
           tableId: table?._id,
           component: component._component,
-          category: component._instanceName,
+          category: bindingCategory.category,
-          icon: def.icon,
+          icon: bindingCategory.icon,
           display: {
             name: fieldSchema.name || key,
             type: fieldSchema.type,
@@ -429,12 +448,46 @@ const getProviderContextBindings = (asset, dataProviders) => {
   return bindings
 }
 
+// Exclude a data context based on the component settings
+const filterCategoryByContext = (component, context) => {
+  const { _component } = component
+  if (_component.endsWith("formblock")) {
+    if (
+      (component.actionType == "Create" && context.type === "schema") ||
+      (component.actionType == "View" && context.type === "form")
+    ) {
+      return false
+    }
+  }
+  return true
+}
+
+const getComponentBindingCategory = (component, context, def) => {
+  let icon = def.icon
+  let category = component._instanceName
+
+  if (component._component.endsWith("formblock")) {
+    let contextCategorySuffix = {
+      form: "Fields",
+      schema: "Row",
+    }
+    category = `${component._instanceName} - ${
+      contextCategorySuffix[context.type]
+    }`
+    icon = context.type === "form" ? "Form" : "Data"
+  }
+  return {
+    icon,
+    category,
+  }
+}
+
 /**
  * Gets all bindable properties from the logged in user.
  */
 export const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForTable(TableNames.USERS)
+  const { schema } = getSchemaForDatasourcePlus(TableNames.USERS)
   const keys = Object.keys(schema).sort()
   const safeUser = makePropSafe("user")
 
@@ -467,22 +520,33 @@ const getDeviceBindings = () => {
   let bindings = []
   if (get(store).clientFeatures?.deviceAwareness) {
     const safeDevice = makePropSafe("device")
-    bindings.push({
+
-      type: "context",
+    bindings = [
-      runtimeBinding: `${safeDevice}.${makePropSafe("mobile")}`,
+      {
-      readableBinding: `Device.Mobile`,
+        type: "context",
-      category: "Device",
+        runtimeBinding: `${safeDevice}.${makePropSafe("mobile")}`,
-      icon: "DevicePhone",
+        readableBinding: `Device.Mobile`,
-      display: { type: "boolean", name: "mobile" },
+        category: "Device",
-    })
+        icon: "DevicePhone",
-    bindings.push({
+        display: { type: "boolean", name: "mobile" },
-      type: "context",
+      },
-      runtimeBinding: `${safeDevice}.${makePropSafe("tablet")}`,
+      {
-      readableBinding: `Device.Tablet`,
+        type: "context",
-      category: "Device",
+        runtimeBinding: `${safeDevice}.${makePropSafe("tablet")}`,
-      icon: "DevicePhone",
+        readableBinding: `Device.Tablet`,
-      display: { type: "boolean", name: "tablet" },
+        category: "Device",
-    })
+        icon: "DevicePhone",
+        display: { type: "boolean", name: "tablet" },
+      },
+      {
+        type: "context",
+        runtimeBinding: `${safeDevice}.${makePropSafe("theme")}`,
+        readableBinding: `App.Theme`,
+        category: "Device",
+        icon: "DevicePhone",
+        display: { type: "string", name: "App Theme" },
+      },
+    ]
   }
   return bindings
 }
@@ -523,6 +587,7 @@ const getSelectedRowsBindings = asset => {
         )}.${makePropSafe("selectedRows")}`,
         readableBinding: `${block._instanceName}.Selected rows`,
         category: "Selected rows",
+        icon: "ViewRow",
         display: { name: block._instanceName },
       }))
     )
@@ -598,24 +663,36 @@ const getRoleBindings = () => {
 }
 
 /**
- * Gets all bindable properties exposed in an event action flow up until
+ * Gets all bindable event context properties provided in the component
- * the specified action ID, as well as context provided for the action
+ * setting
- * setting as a whole by the component.
  */
-export const getEventContextBindings = (
+export const getEventContextBindings = ({
-  asset,
-  componentId,
   settingKey,
-  actions,
+  componentInstance,
-  actionId
+  componentId,
-) => {
+  componentDefinition,
+  asset,
+}) => {
   let bindings = []
+
+  const selectedAsset = asset ?? get(currentAsset)
+
   // Check if any context bindings are provided by the component for this
   // setting
-  const component = findComponent(asset.props, componentId)
+  const component =
-  const def = store.actions.components.getDefinition(component?._component)
+    componentInstance ?? findComponent(selectedAsset.props, componentId)
+
+  if (!component) {
+    return bindings
+  }
+
+  const definition =
+    componentDefinition ??
+    store.actions.components.getDefinition(component?._component)
+
   const settings = getComponentSettings(component?._component)
   const eventSetting = settings.find(setting => setting.key === settingKey)
+
   if (eventSetting?.context?.length) {
     eventSetting.context.forEach(contextEntry => {
       bindings.push({
@@ -624,14 +701,23 @@ export const getEventContextBindings = (
           contextEntry.key
         )}`,
         category: component._instanceName,
-        icon: def.icon,
+        icon: definition.icon,
         display: {
           name: contextEntry.label,
         },
       })
     })
   }
+  return bindings
+}
 
+/**
+ * Gets all bindable properties exposed in an event action flow up until
+ * the specified action ID, as well as context provided for the action
+ * setting as a whole by the component.
+ */
+export const getActionBindings = (actions, actionId) => {
+  let bindings = []
   // Get the steps leading up to this value
   const index = actions?.findIndex(action => action.id === actionId)
   if (index == null || index === -1) {
@@ -658,22 +744,29 @@ export const getEventContextBindings = (
       })
     }
   })
-
   return bindings
 }
 
 /**
- * Gets the schema for a certain table ID.
+ * Gets the schema for a certain datasource plus.
  * The options which can be passed in are:
  *   formSchema: whether the schema is for a form
  *   searchableSchema: whether to generate a searchable schema, which may have
  *     fewer fields than a readable schema
- * @param tableId the table ID to get the schema for
+ * @param resourceId the DS+ resource ID
  * @param options options for generating the schema
  * @return {{schema: Object, table: Object}}
  */
-export const getSchemaForTable = (tableId, options) => {
+export const getSchemaForDatasourcePlus = (resourceId, options) => {
-  return getSchemaForDatasource(null, { type: "table", tableId }, options)
+  const isViewV2 = resourceId?.includes("view_")
+  const datasource = isViewV2
+    ? {
+        type: "viewV2",
+        id: resourceId,
+        tableId: resourceId.split("_").slice(1, 3).join("_"),
+      }
+    : { type: "table", tableId: resourceId }
+  return getSchemaForDatasource(null, datasource, options)
 }
 
 /**
@@ -750,9 +843,21 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
     // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
-        // For views, the schema is pulled from the `views` property of the
+        // Old views
-        // table
         schema = cloneDeep(table.views?.[datasource.name]?.schema)
+      } else if (type === "viewV2") {
+        // New views which are DS+
+        const view = Object.values(table.views || {}).find(
+          view => view.id === datasource.id
+        )
+        schema = cloneDeep(view?.schema)
+
+        // Strip hidden fields
+        Object.keys(schema || {}).forEach(field => {
+          if (!schema[field].visible) {
+            delete schema[field]
+          }
+        })
       } else if (
         type === "query" &&
         (options.formSchema || options.searchableSchema)
@@ -798,12 +903,12 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
 
     // Determine if we should add ID and rev to the schema
     const isInternal = table && !table.sql
-    const isTable = ["table", "link"].includes(datasource.type)
+    const isDSPlus = ["table", "link", "viewV2"].includes(datasource.type)
 
     // ID is part of the readable schema for all tables
     // Rev is part of the readable schema for internal tables only
-    let addId = isTable
+    let addId = isDSPlus
-    let addRev = isTable && isInternal
+    let addRev = isDSPlus && isInternal
 
     // Don't add ID or rev for form schemas
     if (options.formSchema) {
@@ -813,7 +918,7 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
 
     // ID is only searchable for internal tables
     else if (options.searchableSchema) {
-      addId = isTable && isInternal
+      addId = isDSPlus && isInternal
     }
 
     // Add schema properties if required
@@ -851,18 +956,38 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
  * Builds a form schema given a form component.
  * A form schema is a schema of all the fields nested anywhere within a form.
  */
-export const buildFormSchema = component => {
+export const buildFormSchema = (component, asset) => {
   let schema = {}
   if (!component) {
     return schema
   }
 
-  // If this is a form block, simply use the fields setting
   if (component._component.endsWith("formblock")) {
     let schema = {}
-    component.fields?.forEach(field => {
+
-      schema[field] = { type: "string" }
+    const datasource = getDatasourceForProvider(asset, component)
-    })
+    const info = getSchemaForDatasource(component, datasource)
+
+    if (!component.fields) {
+      Object.values(info?.schema)
+        .filter(
+          ({ autocolumn, name }) =>
+            !autocolumn && !["_rev", "_id"].includes(name)
+        )
+        .forEach(({ name }) => {
+          schema[name] = { type: info?.schema[name].type }
+        })
+    } else {
+      // Field conversion
+      const patched = convertOldFieldFormat(component.fields || [])
+      patched?.forEach(({ field, active }) => {
+        if (!active) return
+        if (info?.schema[field]) {
+          schema[field] = { type: info?.schema[field].type }
+        }
+      })
+    }
+
     return schema
   }
 
@@ -878,7 +1003,7 @@ export const buildFormSchema = component => {
     }
   }
   component._children?.forEach(child => {
-    const childSchema = buildFormSchema(child)
+    const childSchema = buildFormSchema(child, asset)
     schema = { ...schema, ...childSchema }
   })
   return schema

packages/builder/src/builderStore/index.js

@@ -4,7 +4,7 @@ import { getTemporalStore } from "./store/temporal"
 import { getThemeStore } from "./store/theme"
 import { getUserStore } from "./store/users"
 import { getDeploymentStore } from "./store/deployments"
-import { derived } from "svelte/store"
+import { derived, writable } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
 import { createHistoryStore } from "builderStore/store/history"
@@ -61,6 +61,12 @@ export const selectedLayout = derived(store, $store => {
 export const selectedComponent = derived(
   [store, selectedScreen],
   ([$store, $selectedScreen]) => {
+    if (
+      $selectedScreen &&
+      $store.selectedComponentId?.startsWith(`${$selectedScreen._id}-`)
+    ) {
+      return $selectedScreen?.props
+    }
     if (!$selectedScreen || !$store.selectedComponentId) {
       return null
     }
@@ -141,3 +147,5 @@ export const userSelectedResourceMap = derived(userStore, $userStore => {
 export const isOnlyUser = derived(userStore, $userStore => {
   return $userStore.length < 2
 })
+
+export const screensHeight = writable("210px")

packages/builder/src/builderStore/store/frontend.js

@@ -111,6 +111,7 @@ export const getFrontendStore = () => {
     }
     let clone = cloneDeep(screen)
     const result = patchFn(clone)
+
     if (result === false) {
       return
     }
@@ -225,7 +226,6 @@ export const getFrontendStore = () => {
         // Select new screen
         store.update(state => {
           state.selectedScreenId = screen._id
-          state.selectedComponentId = screen.props?._id
           return state
         })
       },
@@ -627,6 +627,7 @@ export const getFrontendStore = () => {
               component[setting.key] = {
                 label: defaultDS.name,
                 tableId: defaultDS._id,
+                resourceId: defaultDS._id,
                 type: "table",
               }
             } else if (setting.type === "dataProvider") {
@@ -768,9 +769,13 @@ export const getFrontendStore = () => {
         else {
           await store.actions.screens.patch(screen => {
             // Find the selected component
+            let selectedComponentId = state.selectedComponentId
+            if (selectedComponentId.startsWith(`${screen._id}-`)) {
+              selectedComponentId = screen?.props._id
+            }
             const currentComponent = findComponent(
               screen.props,
-              state.selectedComponentId
+              selectedComponentId
             )
             if (!currentComponent) {
               return false
@@ -833,6 +838,7 @@ export const getFrontendStore = () => {
           return
         }
         const patchScreen = screen => {
+          // findComponent looks in the tree not comp.settings[0]
           let component = findComponent(screen.props, componentId)
           if (!component) {
             return false
@@ -993,12 +999,20 @@ export const getFrontendStore = () => {
         const componentId = state.selectedComponentId
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
-
-        // Check we aren't right at the top of the tree
         const index = parent?._children.findIndex(x => x._id === componentId)
-        if (!parent || componentId === screen.props._id) {
+
+        // Check for screen and navigation component edge cases
+        const screenComponentId = `${screen._id}-screen`
+        const navComponentId = `${screen._id}-navigation`
+        if (componentId === screenComponentId) {
           return null
         }
+        if (componentId === navComponentId) {
+          return screenComponentId
+        }
+        if (parent._id === screen.props._id && index === 0) {
+          return navComponentId
+        }
 
         // If we have siblings above us, choose the sibling or a descendant
         if (index > 0) {
@@ -1020,12 +1034,20 @@ export const getFrontendStore = () => {
         return parent._id
       },
       getNext: () => {
+        const state = get(store)
         const component = get(selectedComponent)
         const componentId = component?._id
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
         const index = parent?._children.findIndex(x => x._id === componentId)
 
+        // Check for screen and navigation component edge cases
+        const screenComponentId = `${screen._id}-screen`
+        const navComponentId = `${screen._id}-navigation`
+        if (state.selectedComponentId === screenComponentId) {
+          return navComponentId
+        }
+
         // If we have children, select first child
         if (component._children?.length) {
           return component._children[0]._id
@@ -1206,7 +1228,12 @@ export const getFrontendStore = () => {
         })
       },
       updateSetting: async (name, value) => {
-        await store.actions.components.patch(component => {
+        await store.actions.components.patch(
+          store.actions.components.updateComponentSetting(name, value)
+        )
+      },
+      updateComponentSetting: (name, value) => {
+        return component => {
           if (!name || !component) {
             return false
           }
@@ -1218,6 +1245,13 @@ export const getFrontendStore = () => {
           const settings = getComponentSettings(component._component)
           const updatedSetting = settings.find(setting => setting.key === name)
 
+          const resetFields = settings.filter(
+            setting => name === setting.resetOn
+          )
+          resetFields?.forEach(setting => {
+            component[setting.key] = null
+          })
+
           if (
             updatedSetting?.type === "dataSource" ||
             updatedSetting?.type === "table"
@@ -1234,9 +1268,8 @@ export const getFrontendStore = () => {
               component[key] = columnNames
             })
           }
-
           component[name] = value
-        })
+        }
       },
       requestEjectBlock: componentId => {
         store.actions.preview.sendEvent("eject-block", componentId)

packages/builder/src/builderStore/store/screenTemplates/index.js

@@ -1,10 +1,10 @@
 import rowListScreen from "./rowListScreen"
 import createFromScratchScreen from "./createFromScratchScreen"
 
-const allTemplates = tables => [...rowListScreen(tables)]
+const allTemplates = datasources => [...rowListScreen(datasources)]
 
 // Allows us to apply common behaviour to all create() functions
-const createTemplateOverride = (frontendState, template) => () => {
+const createTemplateOverride = template => () => {
   const screen = template.create()
   screen.name = screen.props._id
   screen.routing.route = screen.routing.route.toLowerCase()
@@ -12,14 +12,13 @@ const createTemplateOverride = (frontendState, template) => () => {
   return screen
 }
 
-export default (frontendState, tables) => {
+export default datasources => {
   const enrichTemplate = template => ({
     ...template,
-    create: createTemplateOverride(frontendState, template),
+    create: createTemplateOverride(template),
   })
-
   const fromScratch = enrichTemplate(createFromScratchScreen)
-  const tableTemplates = allTemplates(tables).map(enrichTemplate)
+  const tableTemplates = allTemplates(datasources).map(enrichTemplate)
   return [
     fromScratch,
     ...tableTemplates.sort((templateA, templateB) => {

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -2,31 +2,29 @@ import sanitizeUrl from "./utils/sanitizeUrl"
 import { Screen } from "./utils/Screen"
 import { Component } from "./utils/Component"
 
-export default function (tables) {
+export default function (datasources) {
-  return tables.map(table => {
+  if (!Array.isArray(datasources)) {
+    return []
+  }
+  return datasources.map(datasource => {
     return {
-      name: `${table.name} - List`,
+      name: `${datasource.label} - List`,
-      create: () => createScreen(table),
+      create: () => createScreen(datasource),
       id: ROW_LIST_TEMPLATE,
-      table: table._id,
+      resourceId: datasource.resourceId,
     }
   })
 }
 
 export const ROW_LIST_TEMPLATE = "ROW_LIST_TEMPLATE"
-export const rowListUrl = table => sanitizeUrl(`/${table.name}`)
+export const rowListUrl = datasource => sanitizeUrl(`/${datasource.label}`)
 
-const generateTableBlock = table => {
+const generateTableBlock = datasource => {
   const tableBlock = new Component("@budibase/standard-components/tableblock")
   tableBlock
     .customProps({
-      title: table.name,
+      title: datasource.label,
-      dataSource: {
+      dataSource: datasource,
-        label: table.name,
-        name: table._id,
-        tableId: table._id,
-        type: "table",
-      },
       sortOrder: "Ascending",
       size: "spectrum--medium",
       paginate: true,
@@ -36,14 +34,14 @@ const generateTableBlock = table => {
       titleButtonText: "Create row",
       titleButtonClickBehaviour: "new",
     })
-    .instanceName(`${table.name} - Table block`)
+    .instanceName(`${datasource.label} - Table block`)
   return tableBlock
 }
 
-const createScreen = table => {
+const createScreen = datasource => {
   return new Screen()
-    .route(rowListUrl(table))
+    .route(rowListUrl(datasource))
-    .instanceName(`${table.name} - List`)
+    .instanceName(`${datasource.label} - List`)
-    .addChild(generateTableBlock(table))
+    .addChild(generateTableBlock(datasource))
     .json()
 }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -73,7 +73,7 @@
     if (!perms["execute"]) {
       role = "BASIC"
     } else {
-      role = perms["execute"]
+      role = perms["execute"].role
     }
   }
 

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -39,7 +39,7 @@
   import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte"
   import { LuceneUtils } from "@budibase/frontend-core"
   import {
-    getSchemaForTable,
+    getSchemaForDatasourcePlus,
     getEnvironmentBindings,
   } from "builderStore/dataBinding"
   import { Utils } from "@budibase/frontend-core"
@@ -67,7 +67,9 @@
   $: table = tableId
     ? $tables.list.find(table => table._id === inputData.tableId)
     : { schema: {} }
-  $: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
+  $: schema = getSchemaForDatasourcePlus(tableId, {
+    searchableSchema: true,
+  }).schema
   $: schemaFields = Object.values(schema || {})
   $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
   $: isTrigger = block?.type === "TRIGGER"
@@ -158,7 +160,7 @@
     // instead fetch the schema in the backend at runtime.
     let schema
     if (e.detail?.tableId) {
-      schema = getSchemaForTable(e.detail.tableId, {
+      schema = getSchemaForDatasourcePlus(e.detail.tableId, {
         searchableSchema: true,
       }).schema
     }

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -26,12 +26,14 @@
   $: id = $tables.selected?._id
   $: isUsersTable = id === TableNames.USERS
   $: isInternal = $tables.selected?.type !== "external"
-
+  $: gridDatasource = {
-  $: datasource = $datasources.list.find(datasource => {
+    type: "table",
+    tableId: id,
+  }
+  $: tableDatasource = $datasources.list.find(datasource => {
     return datasource._id === $tables.selected?.sourceId
   })
-
+  $: relationshipsEnabled = relationshipSupport(tableDatasource)
-  $: relationshipsEnabled = relationshipSupport(datasource)
 
   const relationshipSupport = datasource => {
     const integration = $integrations[datasource?.source]
@@ -54,12 +56,12 @@
 <div class="wrapper">
   <Grid
     {API}
-    tableId={id}
+    datasource={gridDatasource}
-    allowAddRows={!isUsersTable}
+    canAddRows={!isUsersTable}
-    allowDeleteRows={!isUsersTable}
+    canDeleteRows={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
-    on:updatetable={handleGridTableUpdate}
+    on:updatedatasource={handleGridTableUpdate}
   >
     <svelte:fragment slot="filter">
       <GridFilterButton />
@@ -72,9 +74,7 @@
     </svelte:fragment>
 
     <svelte:fragment slot="controls">
-      {#if isInternal}
+      <GridCreateViewButton />
-        <GridCreateViewButton />
-      {/if}
       <GridManageAccessButton />
       {#if relationshipsEnabled}
         <GridRelationshipButton />

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -10,6 +10,7 @@
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
   import { notifications } from "@budibase/bbui"
+  import { ROW_EXPORT_FORMATS } from "constants/backend"
 
   export let view = {}
 
@@ -19,6 +20,14 @@
   let type = "internal"
 
   $: name = view.name
+  $: calculation = view.calculation
+
+  $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {
+    if (calculation && key === ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA) {
+      return false
+    }
+    return true
+  })
 
   // Fetch rows for specified view
   $: fetchViewData(name, view.field, view.groupBy, view.calculation)
@@ -68,5 +77,5 @@
   {/if}
   <ManageAccessButton resourceId={decodeURI(name)} />
   <HideAutocolumnButton bind:hideAutocolumns />
-  <ExportButton view={view.name} />
+  <ExportButton view={view.name} formats={supportedFormats} />
 </Table>

packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte

@@ -0,0 +1,49 @@
+<script>
+  import { viewsV2 } from "stores/backend"
+  import { Grid } from "@budibase/frontend-core"
+  import { API } from "api"
+  import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
+  import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
+  import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
+
+  $: id = $viewsV2.selected?.id
+  $: datasource = {
+    type: "viewV2",
+    id,
+    tableId: $viewsV2.selected?.tableId,
+  }
+
+  const handleGridViewUpdate = async e => {
+    viewsV2.replaceView(id, e.detail)
+  }
+</script>
+
+<div class="wrapper">
+  <Grid
+    {API}
+    {datasource}
+    allowAddRows
+    allowDeleteRows
+    showAvatars={false}
+    on:updatedatasource={handleGridViewUpdate}
+  >
+    <svelte:fragment slot="filter">
+      <GridFilterButton />
+    </svelte:fragment>
+    <svelte:fragment slot="controls">
+      <GridCreateEditRowModal />
+      <GridManageAccessButton />
+    </svelte:fragment>
+  </Grid>
+</div>
+
+<style>
+  .wrapper {
+    flex: 1 1 auto;
+    margin: -28px -40px -40px -40px;
+    display: flex;
+    flex-direction: column;
+    background: var(--background);
+    overflow: hidden;
+  }
+</style>

packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte

@@ -7,6 +7,7 @@
   export let sorting
   export let disabled = false
   export let selectedRows
+  export let formats
 
   let modal
 </script>
@@ -15,5 +16,5 @@
   Export
 </ActionButton>
 <Modal bind:this={modal}>
-  <ExportModal {view} {filters} {sorting} {selectedRows} />
+  <ExportModal {view} {filters} {sorting} {selectedRows} {formats} />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte

@@ -9,19 +9,15 @@
   let modal
   let resourcePermissions
 
-  async function openDropdown() {
+  async function openModal() {
-    resourcePermissions = await permissions.forResource(resourceId)
+    resourcePermissions = await permissions.forResourceDetailed(resourceId)
     modal.show()
   }
 </script>
 
-<ActionButton icon="LockClosed" quiet on:click={openDropdown} {disabled}>
+<ActionButton icon="LockClosed" quiet on:click={openModal} {disabled}>
   Access
 </ActionButton>
 <Modal bind:this={modal}>
-  <ManageAccessModal
+  <ManageAccessModal {resourceId} permissions={resourcePermissions} />
-    {resourceId}
-    levels={$permissions}
-    permissions={resourcePermissions}
-  />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -15,6 +15,7 @@
   $: tempValue = filters || []
   $: schemaFields = Object.values(schema || {})
   $: text = getText(filters)
+  $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
 
   const getText = filters => {
     const count = filters?.filter(filter => filter.field)?.length
@@ -22,13 +23,7 @@
   }
 </script>
 
-<ActionButton
+<ActionButton icon="Filter" quiet {disabled} on:click={modal.show} {selected}>
-  icon="Filter"
-  quiet
-  {disabled}
-  on:click={modal.show}
-  selected={tempValue?.length > 0}
->
   {text}
 </ActionButton>
 <Modal bind:this={modal}>

packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte

@@ -1,18 +1,30 @@
 <script>
   import { getContext } from "svelte"
-  import { Modal, ActionButton } from "@budibase/bbui"
+  import { Modal, ActionButton, TooltipType, TempTooltip } from "@budibase/bbui"
-  import CreateViewModal from "../../modals/CreateViewModal.svelte"
+  import GridCreateViewModal from "../../modals/grid/GridCreateViewModal.svelte"
 
-  const { rows, columns } = getContext("grid")
+  const { rows, columns, filter } = getContext("grid")
 
   let modal
+  let firstFilterUsage = false
 
   $: disabled = !$columns.length || !$rows.length
+  $: {
+    if ($filter?.length && !firstFilterUsage) {
+      firstFilterUsage = true
+    }
+  }
 </script>
 
-<ActionButton {disabled} icon="CollectionAdd" quiet on:click={modal.show}>
+<TempTooltip
-  Add view
+  text="Create a view to save your filters"
-</ActionButton>
+  type={TooltipType.Info}
+  condition={firstFilterUsage}
+>
+  <ActionButton {disabled} icon="CollectionAdd" quiet on:click={modal.show}>
+    Create view
+  </ActionButton>
+</TempTooltip>
 <Modal bind:this={modal}>
-  <CreateViewModal />
+  <GridCreateViewModal />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/grid/GridExportButton.svelte

@@ -2,7 +2,7 @@
   import ExportButton from "../ExportButton.svelte"
   import { getContext } from "svelte"
 
-  const { rows, columns, tableId, sort, selectedRows, filter } =
+  const { rows, columns, datasource, sort, selectedRows, filter } =
     getContext("grid")
 
   $: disabled = !$rows.length || !$columns.length
@@ -12,7 +12,7 @@
 <span data-ignore-click-outside="true">
   <ExportButton
     {disabled}
-    view={$tableId}
+    view={$datasource.tableId}
     filters={$filter}
     sorting={{
       sortColumn: $sort.column,

packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte

@@ -2,22 +2,19 @@
   import TableFilterButton from "../TableFilterButton.svelte"
   import { getContext } from "svelte"
 
-  const { columns, tableId, filter, table } = getContext("grid")
+  const { columns, datasource, filter, definition } = getContext("grid")
-
-  // Wipe filter whenever table ID changes to avoid using stale filters
-  $: $tableId, filter.set([])
 
   const onFilter = e => {
     filter.set(e.detail || [])
   }
 </script>
 
-{#key $tableId}
+{#key $datasource}
   <TableFilterButton
-    schema={$table?.schema}
+    schema={$definition?.schema}
     filters={$filter}
     on:change={onFilter}
     disabled={!$columns.length}
-    tableId={$tableId}
+    tableId={$datasource.tableId}
   />
 {/key}

packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte

@@ -4,12 +4,12 @@
 
   export let disabled = false
 
-  const { rows, tableId, table } = getContext("grid")
+  const { rows, datasource, definition } = getContext("grid")
 </script>
 
 <ImportButton
   {disabled}
-  tableId={$tableId}
+  tableId={$datasource?.tableId}
-  tableType={$table?.type}
+  tableType={$definition?.type}
   on:importrows={rows.actions.refreshData}
 />

packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte

@@ -2,7 +2,16 @@
   import ManageAccessButton from "../ManageAccessButton.svelte"
   import { getContext } from "svelte"
 
-  const { tableId } = getContext("grid")
+  const { datasource } = getContext("grid")
+
+  $: resourceId = getResourceID($datasource)
+
+  const getResourceID = datasource => {
+    if (!datasource) {
+      return null
+    }
+    return datasource.type === "table" ? datasource.tableId : datasource.id
+  }
 </script>
 
-<ManageAccessButton resourceId={$tableId} />
+<ManageAccessButton {resourceId} />

packages/builder/src/components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte

@@ -2,12 +2,12 @@
   import ExistingRelationshipButton from "../ExistingRelationshipButton.svelte"
   import { getContext } from "svelte"
 
-  const { table, rows } = getContext("grid")
+  const { definition, rows } = getContext("grid")
 </script>
 
-{#if $table}
+{#if $definition}
   <ExistingRelationshipButton
-    table={$table}
+    table={$definition}
     on:updatecolumns={() => rows.actions.refreshData()}
   />
 {/if}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -6,13 +6,15 @@
     Select,
     Toggle,
     RadioGroup,
+    Icon,
     DatePicker,
     Modal,
     notifications,
     OptionSelectDnD,
     Layout,
+    AbsTooltip,
   } from "@budibase/bbui"
-  import { createEventDispatcher, getContext } from "svelte"
+  import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@@ -47,12 +49,13 @@
 
   export let field
 
+  let mounted = false
   let fieldDefinitions = cloneDeep(FIELDS)
   let originalName
   let linkEditDisabled
   let primaryDisplay
   let indexes = [...($tables.selected.indexes || [])]
-  let isCreating
+  let isCreating = undefined
 
   let table = $tables.selected
   let confirmDeleteDialog
@@ -72,11 +75,11 @@
   }
 
   const initialiseField = (field, savingColumn) => {
+    isCreating = !field
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
       linkEditDisabled = originalName != null
-      isCreating = originalName == null
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
@@ -413,16 +416,22 @@
     }
     return newError
   }
+
+  onMount(() => {
+    mounted = true
+  })
 </script>
 
 <Layout noPadding gap="S">
-  <Input
+  {#if mounted}
-    bind:value={editableColumn.name}
+    <Input
-    disabled={uneditable ||
+      autofocus
-      (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+      bind:value={editableColumn.name}
-    error={errors?.name}
+      disabled={uneditable ||
-  />
+        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
-
+      error={errors?.name}
+    />
+  {/if}
   <Select
     disabled={!typeEnabled}
     bind:value={editableColumn.type}
@@ -452,12 +461,17 @@
     />
   {:else if editableColumn.type === "longform"}
     <div>
-      <Label
+      <div class="tooltip-alignment">
-        size="M"
+        <Label size="M">Formatting</Label>
-        tooltip="Rich text includes support for images, links, tables, lists and more"
+        <AbsTooltip
-      >
+          position="top"
-        Formatting
+          type="info"
-      </Label>
+          text={"Rich text includes support for images, link"}
+        >
+          <Icon size="XS" name="InfoOutline" />
+        </AbsTooltip>
+      </div>
+
       <Toggle
         bind:value={editableColumn.useRichText}
         text="Enable rich text support (markdown)"
@@ -488,13 +502,18 @@
     </div>
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
-        <Label
+        <div>
-          tooltip={isCreating
+          <Label>Time zones</Label>
-            ? null
+          <AbsTooltip
-            : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+            position="top"
-        >
+            type="info"
-          Time zones
+            text={isCreating
-        </Label>
+              ? null
+              : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+          >
+            <Icon size="XS" name="InfoOutline" />
+          </AbsTooltip>
+        </div>
         <Toggle
           bind:value={editableColumn.ignoreTimezones}
           text="Ignore time zones"
@@ -565,6 +584,7 @@
               { label: "Dynamic", value: "dynamic" },
               { label: "Static", value: "static" },
             ]}
+            disabled={!isCreating}
             getOptionLabel={option => option.label}
             getOptionValue={option => option.value}
             tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
@@ -671,6 +691,12 @@
     align-items: center;
   }
 
+  .tooltip-alignment {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-xs);
+  }
+
   .label-length {
     flex-basis: 40%;
   }

packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte

@@ -1,38 +0,0 @@
-<script>
-  import { Input, notifications, ModalContent } from "@budibase/bbui"
-  import { goto } from "@roxi/routify"
-  import { views as viewsStore } from "stores/backend"
-  import { tables } from "stores/backend"
-
-  let name
-  let field
-
-  $: views = $tables.list.flatMap(table => Object.keys(table.views || {}))
-
-  const saveView = async () => {
-    name = name?.trim()
-    if (views.includes(name)) {
-      notifications.error(`View exists with name ${name}`)
-      return
-    }
-    try {
-      await viewsStore.save({
-        name,
-        tableId: $tables.selected._id,
-        field,
-      })
-      notifications.success(`View ${name} created`)
-      $goto(`../../view/${encodeURIComponent(name)}`)
-    } catch (error) {
-      notifications.error("Error creating view")
-    }
-  }
-</script>
-
-<ModalContent
-  title="Create View"
-  confirmText="Create View"
-  onConfirm={saveView}
->
-  <Input label="View Name" thin bind:value={name} />
-</ModalContent>

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -9,30 +9,43 @@
   import download from "downloadjs"
   import { API } from "api"
   import { Constants, LuceneUtils } from "@budibase/frontend-core"
-
+  import { ROW_EXPORT_FORMATS } from "constants/backend"
-  const FORMATS = [
-    {
-      name: "CSV",
-      key: "csv",
-    },
-    {
-      name: "JSON",
-      key: "json",
-    },
-    {
-      name: "JSON with Schema",
-      key: "jsonWithSchema",
-    },
-  ]
 
   export let view
   export let filters
   export let sorting
   export let selectedRows = []
+  export let formats
 
-  let exportFormat = FORMATS[0].key
+  const FORMATS = [
+    {
+      name: "CSV",
+      key: ROW_EXPORT_FORMATS.CSV,
+    },
+    {
+      name: "JSON",
+      key: ROW_EXPORT_FORMATS.JSON,
+    },
+    {
+      name: "JSON with Schema",
+      key: ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA,
+    },
+  ]
+
+  $: options = FORMATS.filter(format => {
+    if (formats && !formats.includes(format.key)) {
+      return false
+    }
+    return true
+  })
+
+  let exportFormat
   let filterLookup
 
+  $: if (options && !exportFormat) {
+    exportFormat = Array.isArray(options) ? options[0]?.key : []
+  }
+
   $: luceneFilter = LuceneUtils.buildLuceneQuery(filters)
   $: exportOpDisplay = buildExportOpDisplay(sorting, filterDisplay, filters)
 
@@ -190,7 +203,7 @@
   <Select
     label="Format"
     bind:value={exportFormat}
-    options={FORMATS}
+    {options}
     placeholder={null}
     getOptionLabel={x => x.name}
     getOptionValue={x => x.key}

packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { PermissionSource } from "@budibase/types"
   import { roles, permissions as permissionsStore } from "stores/backend"
   import {
     Label,
@@ -7,45 +8,130 @@
     notifications,
     Body,
     ModalContent,
+    Tags,
+    Tag,
+    Icon,
   } from "@budibase/bbui"
   import { capitalise } from "helpers"
+  import { get } from "svelte/store"
 
   export let resourceId
   export let permissions
 
+  const inheritedRoleId = "inherited"
+
   async function changePermission(level, role) {
     try {
-      await permissionsStore.save({
+      if (role === inheritedRoleId) {
-        level,
+        await permissionsStore.remove({
-        role,
+          level,
-        resource: resourceId,
+          role,
-      })
+          resource: resourceId,
+        })
+      } else {
+        await permissionsStore.save({
+          level,
+          role,
+          resource: resourceId,
+        })
+      }
 
       // Show updated permissions in UI: REMOVE
-      permissions = await permissionsStore.forResource(resourceId)
+      permissions = await permissionsStore.forResourceDetailed(resourceId)
       notifications.success("Updated permissions")
     } catch (error) {
       notifications.error("Error updating permissions")
     }
   }
+
+  $: computedPermissions = Object.entries(permissions.permissions).reduce(
+    (p, [level, roleInfo]) => {
+      p[level] = {
+        selectedValue:
+          roleInfo.permissionType === PermissionSource.INHERITED
+            ? inheritedRoleId
+            : roleInfo.role,
+        options: [...get(roles)],
+      }
+
+      if (roleInfo.inheritablePermission) {
+        p[level].inheritOption = roleInfo.inheritablePermission
+        p[level].options.unshift({
+          _id: inheritedRoleId,
+          name: `Inherit (${
+            get(roles).find(x => x._id === roleInfo.inheritablePermission).name
+          })`,
+        })
+      }
+      return p
+    },
+    {}
+  )
+
+  $: requiresPlanToModify = permissions.requiresPlanToModify
+
+  let dependantsInfoMessage
+  async function loadDependantInfo() {
+    const dependantsInfo = await permissionsStore.getDependantsInfo(resourceId)
+
+    const resourceByType = dependantsInfo?.resourceByType
+
+    if (resourceByType) {
+      const total = Object.values(resourceByType).reduce((p, c) => p + c, 0)
+      let resourceDisplay =
+        Object.keys(resourceByType).length === 1 && resourceByType.view
+          ? "view"
+          : "resource"
+
+      if (total === 1) {
+        dependantsInfoMessage = `1 ${resourceDisplay} is inheriting this access.`
+      } else if (total > 1) {
+        dependantsInfoMessage = `${total} ${resourceDisplay}s are inheriting this access.`
+      }
+    }
+  }
+  loadDependantInfo()
 </script>
 
-<ModalContent title="Manage Access" showCancelButton={false} confirmText="Done">
+<ModalContent showCancelButton={false} confirmText="Done">
+  <span slot="header">
+    Manage Access
+    {#if requiresPlanToModify}
+      <span class="lock-tag">
+        <Tags>
+          <Tag icon="LockClosed">{capitalise(requiresPlanToModify)}</Tag>
+        </Tags>
+      </span>
+    {/if}
+  </span>
   <Body size="S">Specify the minimum access level role for this data.</Body>
   <div class="row">
     <Label extraSmall grey>Level</Label>
     <Label extraSmall grey>Role</Label>
-    {#each Object.keys(permissions) as level}
+    {#each Object.keys(computedPermissions) as level}
       <Input value={capitalise(level)} disabled />
       <Select
-        value={permissions[level]}
+        disabled={requiresPlanToModify}
+        placeholder={false}
+        value={computedPermissions[level].selectedValue}
         on:change={e => changePermission(level, e.detail)}
-        options={$roles}
+        options={computedPermissions[level].options}
         getOptionLabel={x => x.name}
         getOptionValue={x => x._id}
       />
     {/each}
   </div>
+
+  {#if dependantsInfoMessage}
+    <div class="inheriting-resources">
+      <Icon name="Alert" />
+      <Body size="S">
+        <i>
+          {dependantsInfoMessage}
+        </i>
+      </Body>
+    </div>
+  {/if}
 </ModalContent>
 
 <style>
@@ -54,4 +140,13 @@
     grid-template-columns: 1fr 1fr;
     grid-gap: var(--spacing-s);
   }
+
+  .lock-tag {
+    padding-left: var(--spacing-s);
+  }
+
+  .inheriting-resources {
+    display: flex;
+    gap: var(--spacing-s);
+  }
 </style>

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte

@@ -2,7 +2,7 @@
   import { getContext } from "svelte"
   import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
 
-  const { rows } = getContext("grid")
+  const { datasource } = getContext("grid")
 </script>
 
-<CreateEditColumn on:updatecolumns={rows.actions.refreshTableDefinition} />
+<CreateEditColumn on:updatecolumns={datasource.actions.refreshDefinition} />

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte

@@ -0,0 +1,60 @@
+<script>
+  import { getContext } from "svelte"
+  import { Input, notifications, ModalContent } from "@budibase/bbui"
+  import { goto } from "@roxi/routify"
+  import { viewsV2 } from "stores/backend"
+
+  const { filter, sort, definition } = getContext("grid")
+
+  let name
+
+  $: views = Object.keys($definition?.views || {}).map(x => x.toLowerCase())
+  $: nameExists = views.includes(name?.trim().toLowerCase())
+
+  const enrichSchema = schema => {
+    // We need to sure that "visible" is set to true for any fields which have
+    // not yet been saved with grid metadata attached
+    const cloned = { ...schema }
+    Object.entries(cloned).forEach(([field, fieldSchema]) => {
+      if (fieldSchema.visible == null) {
+        cloned[field] = { ...cloned[field], visible: true }
+      }
+    })
+    return cloned
+  }
+
+  const saveView = async () => {
+    name = name?.trim()
+    try {
+      const newView = await viewsV2.create({
+        name,
+        tableId: $definition._id,
+        query: $filter,
+        sort: {
+          field: $sort.column,
+          order: $sort.order,
+        },
+        schema: enrichSchema($definition.schema),
+        primaryDisplay: $definition.primaryDisplay,
+      })
+      notifications.success(`View ${name} created`)
+      $goto(`../../view/v2/${newView.id}`)
+    } catch (error) {
+      notifications.error("Error creating view")
+    }
+  }
+</script>
+
+<ModalContent
+  title="Create view"
+  confirmText="Create view"
+  onConfirm={saveView}
+  disabled={nameExists}
+>
+  <Input
+    label="View name"
+    thin
+    bind:value={name}
+    error={nameExists ? "A view already exists with that name" : null}
+  />
+</ModalContent>

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,7 +1,14 @@
 <script>
   import { goto, isActive, params } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
-  import { database, datasources, queries, tables, views } from "stores/backend"
+  import {
+    database,
+    datasources,
+    queries,
+    tables,
+    views,
+    viewsV2,
+  } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
@@ -24,6 +31,7 @@
     $tables,
     $queries,
     $views,
+    $viewsV2,
     openDataSources
   )
   $: openDataSource = enrichedDataSources.find(x => x.open)
@@ -41,6 +49,7 @@
     tables,
     queries,
     views,
+    viewsV2,
     openDataSources
   ) => {
     if (!datasources?.list?.length) {
@@ -57,7 +66,8 @@
         isActive,
         tables,
         queries,
-        views
+        views,
+        viewsV2
       )
       const onlySource = datasources.list.length === 1
       return {
@@ -106,7 +116,8 @@
     isActive,
     tables,
     queries,
-    views
+    views,
+    viewsV2
   ) => {
     // Check for being on a datasource page
     if (params.datasourceId === datasource._id) {
@@ -152,10 +163,16 @@
 
     // Check for a matching view
     const selectedView = views.selected?.name
-    const table = options.find(table => {
+    const viewTable = options.find(table => {
       return table.views?.[selectedView] != null
     })
-    return table != null
+    if (viewTable) {
+      return true
+    }
+
+    // Check for a matching viewV2
+    const viewV2Table = options.find(x => x._id === viewsV2.selected?.tableId)
+    return viewV2Table != null
   }
 </script>