Merge branch 'master' into global-bindings

.dockerignore

@@ -1,9 +1,14 @@
-packages/server/node_modules
-packages/builder
-packages/frontend-core
-packages/backend-core
-packages/worker/node_modules
-packages/cli
-packages/client
-packages/bbui
-packages/string-templates
+*
+!/packages/
+!/scripts/
+/packages/*/node_modules
+packages/server/scripts/
+!packages/server/scripts/integrations/oracle
+!nx.json
+!/hosting/single/
+!/hosting/letsencrypt /
+
+!package.json
+!yarn.lock
+!lerna.json
+!.yarnrc

.github/workflows/budibase_ci.yml

@@ -10,7 +10,6 @@ on:
   push:
     branches:
       - master
-      - develop
   pull_request:
   workflow_dispatch:
 
@@ -20,18 +19,12 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        with:
-          root-reserve-mb: 35000
-          swap-size-mb: 1024
-          remove-android: 'true'
-          remove-dotnet: 'true'
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
@@ -121,7 +114,7 @@ jobs:
           name: codecov-umbrella
           verbose: true
 
-  test-services:
+  test-worker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
@@ -143,12 +136,48 @@ jobs:
           node-version: 18.x
           cache: "yarn"
       - run: yarn --frozen-lockfile
-      - name: Test worker and server
+      - name: Test worker
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
-            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --scope=@budibase/worker --scope=@budibase/server
+            yarn test --scope=@budibase/worker
+          fi
+
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-server:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo and submodules
+        uses: actions/checkout@v3
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
+      - name: Checkout repo only
+        uses: actions/checkout@v3
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+          cache: "yarn"
+      - run: yarn --frozen-lockfile
+      - name: Test server
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/server
           fi
 
       - uses: codecov/codecov-action@v3
@@ -232,20 +261,21 @@ jobs:
           branch="${{ github.base_ref || github.ref_name  }}"
           echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"
 
-          if [[ $branch == "master" ]]; then
-            base_commit=$(git rev-parse origin/master)
+          base_commit=$(git rev-parse origin/master)
+
+          if [[ ! -z $base_commit ]]; then
+            echo "target_branch=$branch"
+            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
+            echo "pro_commit=$pro_commit"
+            echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
+            echo "base_commit=$base_commit"
+            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
           else
-            base_commit=$(git rev-parse origin/develop)
+            echo "Nothing to do - branch to branch merge."
           fi
 
-          echo "target_branch=$branch"
-          echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
-          echo "pro_commit=$pro_commit"
-          echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
-          echo "base_commit=$base_commit"
-          echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
-
-      - name: Check submodule merged to develop
+      - name: Check submodule merged to base branch
+        if: ${{ steps.get_pro_commits.outputs.base_commit != '' }}
         uses: actions/github-script@v4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -254,9 +284,9 @@ jobs:
             const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
 
             if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
+              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.');
               console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
               process.exit(1);
             } else {
               console.log('All good, the submodule had been merged and setup correctly!')
-            }
+            }

.github/workflows/check_unreleased_changes.yml

@@ -1,29 +0,0 @@
-name: check_unreleased_changes
-
-on:
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  check_unreleased:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check for unreleased changes
-        env:
-          REPO: "Budibase/budibase"
-          TOKEN: ${{ secrets.GITHUB_TOKEN }} 
-        run: |
-          RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
-              "https://api.github.com/repos/$REPO/releases/latest" | \
-              jq -r .published_at)
-          COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
-              "https://api.github.com/repos/$REPO/commits/master" | \
-              jq -r .commit.committer.date)
-          RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s")
-          COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s")
-          if (( COMMIT_SECONDS > RELEASE_SECONDS )); then
-              echo "There are unreleased changes. Please release these changes before merging."
-              exit 1
-          fi
-          echo "No unreleased changes detected."

.github/workflows/close-featurebranch.yml

@@ -0,0 +1,27 @@
+name: close-featurebranch
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - master
+  workflow_dispatch:
+    inputs:
+      BRANCH:
+        type: string
+        description: Which featurebranch branch to destroy?
+        required: true
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.BRANCH || github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
+        with:
+          repository: budibase/budibase-deploys
+          event: featurebranch-qa-close
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/deploy-featurebranch.yml

@@ -3,7 +3,7 @@ name: deploy-featurebranch
 on:
   pull_request:
     branches:
-      - develop
+      - master
 
 jobs:
   release:
@@ -12,7 +12,8 @@ jobs:
       - uses: actions/checkout@v3
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
-            BRANCH: ${{ github.head_ref }}
+          PAYLOAD_BRANCH: ${{ github.head_ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

.github/workflows/deploy-preprod.yml

@@ -1,41 +0,0 @@
-name: "deploy-preprod"
-on:
-  workflow_dispatch:
-  workflow_call:
-
-jobs:
-  deploy-to-legacy-preprod-env:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
-        with:
-          repository: budibase/budibase-deploys
-          event: budicloud-preprod-deploy
-          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
-

.github/workflows/release-develop.yml

@@ -1,124 +0,0 @@
-name: Budibase Prerelease
-concurrency:
-  group: release-prerelease
-  cancel-in-progress: false
-
-on:
-  push:
-    tags:
-      - "*-alpha.*"
-  workflow_dispatch:
-
-env:
-  # Posthog token used by ui at build time
-  # disable unless needed for testing
-  # POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
-  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  FEATURE_PREVIEW_URL: https://budirelease.live
-
-jobs:
-  release-images:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Fail if tag is not develop
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/develop; then
-            echo "Tag is not in develop"
-            exit 1
-          fi
-
-      - uses: actions/setup-node@v1
-        with:
-          node-version: 18.x
-
-      - run: yarn install --frozen-lockfile
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - run: yarn build
-      - run: yarn build:sdk
-
-      - name: Publish budibase packages to NPM
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          # setup the username and email.
-          git config --global user.name "Budibase Staging Release Bot"
-          git config --global user.email "<>"
-          git submodule foreach git commit -a -m 'Release process'
-          git commit -a -m 'Release process'
-          echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc 
-          yarn release:develop
-
-      - name: Build/release Docker images
-        run: |
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:develop
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-
-  release-helm-chart:
-    needs: [release-images]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        id: helm-install
-
-      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
-      # we need to create new package in a different dir, merge the index and move the package back
-      - name: Build and release helm chart
-        run: |
-          git config user.name "Budibase Helm Bot"
-          git config user.email "<>"
-          git reset --hard
-          git fetch
-          mkdir sync
-          echo "Packaging chart to sync dir"
-          helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync
-          echo "Packaging successful"
-          git checkout gh-pages
-          echo "Indexing helm repo"         
-          helm repo index --merge docs/index.yaml sync
-          mv -f sync/* docs
-          rm -rf sync
-          echo "Pushing new helm release"
-          git add -A
-          git commit -m "Helm Release: develop"
-          git push
-
-  trigger-deploy-to-qa-env:
-    needs: [release-helm-chart]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the current budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
-        with:
-          repository: budibase/budibase-deploys
-          event: budicloud-qa-deploy
-          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-master.yml

@@ -110,19 +110,13 @@ jobs:
           git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
           git push
 
-  deploy-to-legacy-preprod-env:
-    needs: [release-images]
-    uses: ./.github/workflows/deploy-preprod.yml
-    secrets: inherit
 
-  # Trigger deploy to new EKS preprod environment
-  trigger-deploy-to-preprod-env:
+  trigger-deploy-to-qa-env:
     needs: [release-helm-chart]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get the latest budibase release version
+      - name: Get the current budibase release version
         id: version
         run: |
           release_version=$(cat lerna.json | jq -r '.version')
@@ -133,5 +127,5 @@ jobs:
           PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
         with:
           repository: budibase/budibase-deploys
-          event: budicloud-preprod-deploy
+          event: budicloud-qa-deploy
           github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-singleimage-test.yml

@@ -0,0 +1,69 @@
+name: Test
+
+on:
+  workflow_dispatch:
+
+env:
+  CI: true
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  REGISTRY_URL: registry.hub.docker.com
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+jobs:
+  build:
+    name: "build"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "yarn"
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Run Yarn
+        run: yarn
+      - name: Run Yarn Build
+        run: yarn build --scope @budibase/server --scope @budibase/worker
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_API_KEY }}
+      - name: Get the latest release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo $release_version
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+      - name: Tag and release Budibase service docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          pull: true
+          platforms: linux/amd64,linux/arm64
+          tags: budibase/budibase-test:test
+          file: ./hosting/single/Dockerfile.v2
+          cache-from: type=registry,ref=budibase/budibase-test:test
+          cache-to: type=inline
+      - name: Tag and release Budibase Azure App Service docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64
+          build-args: TARGETBUILD=aas
+          tags: budibase/budibase-test:aas
+          file: ./hosting/single/Dockerfile.v2

.github/workflows/release-singleimage.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Maximize build space
         uses: easimon/maximize-build-space@master
         with:
-          root-reserve-mb: 35000
+          root-reserve-mb: 30000
           swap-size-mb: 1024
           remove-android: 'true'
           remove-dotnet: 'true'
@@ -33,14 +33,6 @@ jobs:
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
@@ -55,10 +47,6 @@ jobs:
         run: yarn
       - name: Update versions
         run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
       - name: Run Yarn Build
         run: yarn build:docker:pre
       - name: Login to Docker Hub

.github/workflows/stale_bot.yml

@@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale
 on:
   workflow_dispatch:
   schedule:
-    - cron: '*/30 * * * *' # Every 30 mins
+    - cron: "*/30 * * * *" # Every 30 mins
 
 jobs:
   stale:
@@ -10,20 +10,37 @@ jobs:
     steps:
       - uses: actions/stale@v8
         with:
-          # stale rules
-          days-before-stale: 60
+          operations-per-run: 1
+          # stale rules for PRs
           days-before-pr-stale: 7
           stale-issue-label: stale
-          stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for 60 days."
-
-          # close rules
-          # days after being marked as stale to close
-          days-before-close: 30
-          close-issue-label: closed-stale
-          close-issue-message: This issue has been automatically closed it has not had any activity in 90 days."
-          days-before-pr-close: 7
-
-          # exemptions
           exempt-pr-labels: pinned,security,roadmap
 
+          days-before-pr-close: 7
 
+      - uses: actions/stale@v8
+        with:
+          operations-per-run: 3
+          # stale rules for high priority bugs
+          days-before-stale: 30
+          only-issue-labels: bug,High priority
+          stale-issue-label: warn
+
+      - uses: actions/stale@v8
+        with:
+          operations-per-run: 3
+          # stale rules for medium priority bugs
+          days-before-stale: 90
+          only-issue-labels: bug,Medium priority
+          stale-issue-label: warn
+
+      - uses: actions/stale@v8
+        with:
+          operations-per-run: 3
+          # stale rules for all bugs
+          days-before-stale: 180
+          stale-issue-label: stale
+          only-issue-labels: bug
+          stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for six months."
+
+          days-before-close: 30

.github/workflows/tag-prerelease.yml

@@ -1,42 +0,0 @@
-name: Tag prerelease
-concurrency:
-  group: tag-prerelease
-  cancel-in-progress: false
-
-on:
-  push:
-    branches:
-      - develop
-    paths:
-      - ".aws/**"
-      - ".github/**"
-      - "charts/**"
-      - "packages/**"
-      - "scripts/**"
-      - "package.json"
-      - "yarn.lock"
-  workflow_dispatch:
-
-jobs:
-  tag-prerelease:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Fail if branch is not develop
-        if: github.ref != 'refs/heads/develop'
-        run: |
-          echo "Ref is not develop, you must run this job from develop."
-          exit 1
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-      - run: cd scripts && yarn
-      - name: Tag prerelease
-        run: |
-          cd scripts
-          # setup the username and email.
-          git config --global user.name "Budibase Staging Release Bot"
-          git config --global user.email "<>"
-          ./versionCommit.sh prerelease

.github/workflows/tag-release.yml

@@ -4,17 +4,6 @@ concurrency:
   cancel-in-progress: false
 
 on:
-  push:
-    branches:
-      - master
-    paths:
-      - ".aws/**"
-      - ".github/**"
-      - "charts/**"
-      - "packages/**"
-      - "scripts/**"
-      - "package.json"
-      - "yarn.lock"
   workflow_dispatch:
     inputs:
       versioning:

.yarnrc

@@ -1 +1 @@
-network-timeout 100000
+network-timeout 1000000

docs/CONTRIBUTING.md

@@ -138,6 +138,8 @@ To develop the Budibase platform you'll need [Docker](https://www.docker.com/) a
 
 `yarn setup` will check that all necessary components are installed and setup the repo for usage.
 
+If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above command.
+
 ##### Manual method
 
 The following commands can be executed to manually get Budibase up and running (assuming Docker/Docker Compose has been installed).
@@ -146,6 +148,8 @@ The following commands can be executed to manually get Budibase up and running (
 
 `yarn build` will build all budibase packages.
 
+If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above commands.
+
 #### 4. Running
 
 To run the budibase server and builder in dev mode (i.e. with live reloading):

hosting/proxy/nginx.prod.conf

@@ -55,7 +55,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
+    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

hosting/single/Dockerfile

@@ -12,14 +12,14 @@ RUN chmod +x /cleanup.sh
 WORKDIR /app
 ADD packages/server .
 COPY yarn.lock .
-RUN yarn install --production=true
+RUN yarn install --production=true --network-timeout 100000
 RUN /cleanup.sh
 
 # build worker
 WORKDIR /worker
 ADD packages/worker .
 COPY yarn.lock .
-RUN yarn install --production=true
+RUN yarn install --production=true --network-timeout 100000
 RUN /cleanup.sh
 
 FROM budibase/couchdb

hosting/single/Dockerfile.v2

@@ -0,0 +1,126 @@
+FROM node:18-slim as build
+
+# install node-gyp dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
+
+
+# copy and install dependencies
+WORKDIR /app
+COPY package.json .
+COPY yarn.lock .
+COPY lerna.json .
+COPY .yarnrc .
+
+COPY packages/server/package.json packages/server/package.json
+COPY packages/worker/package.json packages/worker/package.json
+# string-templates does not get bundled during the esbuild process, so we want to use the local version
+COPY packages/string-templates/package.json packages/string-templates/package.json
+
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh
+
+
+# We will never want to sync pro, but the script is still required
+RUN echo '' > scripts/syncProPackage.js
+RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn  yarn install --production
+
+# copy the actual code
+COPY packages/server/dist packages/server/dist
+COPY packages/server/pm2.config.js packages/server/pm2.config.js
+COPY packages/server/client packages/server/client
+COPY packages/server/builder packages/server/builder
+COPY packages/worker/dist packages/worker/dist
+COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
+COPY packages/string-templates packages/string-templates
+
+
+FROM budibase/couchdb as runner
+ARG TARGETARCH
+ENV TARGETARCH $TARGETARCH
+#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
+# e.g. docker build --build-arg TARGETBUILD=aas ....
+ARG TARGETBUILD=single
+ENV TARGETBUILD $TARGETBUILD
+
+# install base dependencies
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
+
+# Install postgres client for pg_dump utils
+RUN apt install software-properties-common apt-transport-https gpg -y \
+    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
+    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
+    && apt update -y \
+    && apt install postgresql-client-15 -y \
+    && apt remove software-properties-common apt-transport-https gpg -y
+
+# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
+WORKDIR /nodejs
+RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \
+  bash /tmp/nodesource_setup.sh && \
+  apt-get install -y --no-install-recommends libaio1 nodejs && \
+  npm install --global yarn pm2
+
+# setup nginx
+COPY hosting/single/nginx/nginx.conf /etc/nginx
+COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
+RUN mkdir -p /var/log/nginx && \
+  touch /var/log/nginx/error.log && \
+  touch /var/run/nginx.pid && \
+  usermod -a -G tty www-data
+
+WORKDIR /
+RUN mkdir -p scripts/integrations/oracle
+COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle
+RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh
+
+# setup minio
+WORKDIR /minio
+COPY scripts/install-minio.sh ./install.sh
+RUN chmod +x install.sh && ./install.sh
+
+# setup runner file
+WORKDIR /
+COPY hosting/single/runner.sh .
+RUN chmod +x ./runner.sh
+COPY hosting/single/healthcheck.sh .
+RUN chmod +x ./healthcheck.sh
+
+# Script below sets the path for storing data based on $DATA_DIR
+# For Azure App Service install SSH & point data locations to /home
+COPY hosting/single/ssh/sshd_config /etc/
+COPY hosting/single/ssh/ssh_setup.sh /tmp
+RUN /build-target-paths.sh
+
+
+#  setup letsencrypt certificate
+RUN apt-get install -y certbot python3-certbot-nginx
+COPY hosting/letsencrypt /app/letsencrypt
+RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
+
+COPY --from=build /app/node_modules /node_modules
+COPY --from=build /app/package.json /package.json
+COPY --from=build /app/packages/server /app
+COPY --from=build /app/packages/worker /worker
+COPY --from=build /app/packages/string-templates /string-templates
+
+RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates
+
+
+EXPOSE 80
+EXPOSE 443
+# Expose port 2222 for SSH on Azure App Service build
+EXPOSE 2222
+VOLUME /data
+
+
+HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
+
+# must set this just before running
+ENV NODE_ENV=production
+WORKDIR /
+
+CMD ["./runner.sh"]

hosting/single/runner.sh

@@ -7,16 +7,16 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
 [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
-[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
+[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000
 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
 [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
 [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
-[[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
+[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379
 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
 [[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
-[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002
-[[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001
+[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002
+[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001
 [[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app
 #  export CUSTOM_DOMAIN=budi001.custom.com
 
@@ -51,7 +51,7 @@ do
     fi
 done
 if [[ -z "${COUCH_DB_URL}" ]]; then
-    export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984
+    export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984
 fi
 if [ ! -f "${DATA_DIR}/.env" ]; then
     touch ${DATA_DIR}/.env

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.2",
+  "version": "2.11.35",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

nx.json

@@ -8,5 +8,9 @@
       }
     }
   },
-  "targetDefaults": {}
+  "targetDefaults": {
+    "build": {
+      "inputs": ["{workspaceRoot}/scripts/build.js"]
+    }
+  }
 }

package.json

@@ -3,14 +3,11 @@
   "private": true,
   "devDependencies": {
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
-    "@nx/js": "16.4.3",
-    "@rollup/plugin-json": "^4.0.2",
-    "@typescript-eslint/parser": "5.45.0",
+    "@typescript-eslint/parser": "6.7.2",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
     "husky": "^8.0.3",
-    "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
     "lerna": "7.1.1",
     "madge": "^6.0.0",
@@ -19,10 +16,8 @@
     "nx-cloud": "16.0.5",
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
-    "rimraf": "^3.0.2",
-    "rollup-plugin-replace": "^2.2.0",
-    "svelte": "^3.38.2",
-    "typescript": "4.7.3",
+    "svelte": "3.49.0",
+    "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
     "@babel/eslint-parser": "^7.22.5",
     "@babel/preset-env": "^7.22.5",
@@ -51,7 +46,7 @@
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
-    "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
+    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",
     "lint:eslint": "eslint packages qa-core --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@@ -61,7 +56,6 @@
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
     "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:pre": "yarn build && lerna run --stream predocker",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
     "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
@@ -69,12 +63,10 @@
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
     "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",
-    "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
-    "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image",
+    "build:docker:single": "./scripts/build-single-image.sh",
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
-    "build:docs": "lerna run --stream build:docs",
     "release:helm": "node scripts/releaseHelmChart",
     "env:multi:enable": "lerna run --stream env:multi:enable",
     "env:multi:disable": "lerna run --stream env:multi:disable",

packages/backend-core/package.json

@@ -26,24 +26,21 @@
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
     "@techpass/passport-openidconnect": "0.3.2",
-    "aws-cloudfront-sign": "2.2.0",
+    "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
-    "emitter-listener": "1.1.2",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
-    "lodash.isarguments": "3.1.0",
     "node-fetch": "2.6.7",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "passport-oauth2-refresh": "^2.1.0",
     "pino": "8.11.0",
@@ -59,14 +56,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/jest": "29.5.3",
-    "@types/koa": "2.13.4",
+    "@types/cookies": "0.7.8",
+    "@types/jest": "29.5.5",
     "@types/lodash": "4.14.180",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
@@ -80,14 +76,10 @@
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",
-    "koa": "2.13.4",
-    "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3"
+    "typescript": "5.2.2"
   },
   "nx": {
     "targets": {

packages/backend-core/src/cache/user.ts

@@ -120,7 +120,7 @@ export async function getUsers(
 ): Promise<{ users: User[]; notFoundIds?: string[] }> {
   const client = await redis.getUserClient()
   // try cache
-  let usersFromCache = await client.bulkGet(userIds)
+  let usersFromCache = await client.bulkGet<User>(userIds)
   const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
   const users = Object.values(usersFromCache)
   let notFoundIds

packages/backend-core/src/constants/db.ts

@@ -1,5 +1,10 @@
 import { prefixed, DocumentType } from "@budibase/types"
-export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
+export {
+  SEPARATOR,
+  UNICODE_MAX,
+  DocumentType,
+  InternalTable,
+} from "@budibase/types"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -18,7 +23,7 @@ export enum ViewName {
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
   ACCOUNT_BY_EMAIL = "account_by_email",
-  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
+  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase_2",
   USER_BY_GROUP = "user_by_group",
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
@@ -30,10 +35,6 @@ export const DeprecatedViews = {
   ],
 }
 
-export enum InternalTable {
-  USER_METADATA = "ta_users",
-}
-
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",

packages/backend-core/src/db/views.ts

@@ -190,6 +190,10 @@ export const createPlatformUserView = async () => {
     if (doc.tenantId) {
       emit(doc._id.toLowerCase(), doc._id)
     }
+
+    if (doc.ssoId) {
+      emit(doc.ssoId, doc._id)
+    }
   }`
   await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }

packages/backend-core/src/docIds/ids.ts

@@ -45,6 +45,11 @@ export function generateGlobalUserID(id?: any) {
   return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
 }
 
+const isGlobalUserIDRegex = new RegExp(`^${DocumentType.USER}${SEPARATOR}.+`)
+export function isGlobalUserID(id: string) {
+  return isGlobalUserIDRegex.test(id)
+}
+
 /**
  * Generates a new user ID based on the passed in global ID.
  * @param {string} globalId The ID of the global user.

packages/backend-core/src/objectStore/cloudfront.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-const cfsign = require("aws-cloudfront-sign")
+import * as cfsign from "aws-cloudfront-sign"
 
 let PRIVATE_KEY: string | undefined
 
@@ -21,7 +21,7 @@ function getPrivateKey() {
 
 const getCloudfrontSignParams = () => {
   return {
-    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID,
+    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!,
     privateKeyString: getPrivateKey(),
     expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
   }

packages/backend-core/src/platform/users.ts

@@ -5,6 +5,7 @@ import {
   PlatformUser,
   PlatformUserByEmail,
   PlatformUserById,
+  PlatformUserBySsoId,
   User,
 } from "@budibase/types"
 
@@ -45,6 +46,20 @@ function newUserEmailDoc(
   }
 }
 
+function newUserSsoIdDoc(
+  ssoId: string,
+  email: string,
+  userId: string,
+  tenantId: string
+): PlatformUserBySsoId {
+  return {
+    _id: ssoId,
+    userId,
+    email,
+    tenantId,
+  }
+}
+
 /**
  * Add a new user id or email doc if it doesn't exist.
  */
@@ -64,11 +79,24 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
   }
 }
 
-export async function addUser(tenantId: string, userId: string, email: string) {
-  await Promise.all([
+export async function addUser(
+  tenantId: string,
+  userId: string,
+  email: string,
+  ssoId?: string
+) {
+  const promises = [
     addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
     addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
-  ])
+  ]
+
+  if (ssoId) {
+    promises.push(
+      addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId))
+    )
+  }
+
+  await Promise.all(promises)
 }
 
 // DELETE

packages/backend-core/src/plugin/utils.ts

@@ -6,6 +6,7 @@ import {
   AutomationStepIdArray,
   AutomationIOType,
   AutomationCustomIOType,
+  DatasourceFeature,
 } from "@budibase/types"
 import joi from "joi"
 
@@ -67,9 +68,27 @@ function validateDatasource(schema: any) {
     version: joi.string().optional(),
     schema: joi.object({
       docs: joi.string(),
+      plus: joi.boolean().optional(),
+      isSQL: joi.boolean().optional(),
+      auth: joi
+        .object({
+          type: joi.string().required(),
+        })
+        .optional(),
+      features: joi
+        .object(
+          Object.fromEntries(
+            Object.values(DatasourceFeature).map(key => [
+              key,
+              joi.boolean().optional(),
+            ])
+          )
+        )
+        .optional(),
+      relationships: joi.boolean().optional(),
+      description: joi.string().required(),
       friendlyName: joi.string().required(),
       type: joi.string().allow(...DATASOURCE_TYPES),
-      description: joi.string().required(),
       datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
       query: joi
         .object()

packages/backend-core/src/redis/redis.ts

@@ -242,7 +242,7 @@ class RedisWrapper {
     }
   }
 
-  async bulkGet(keys: string[]) {
+  async bulkGet<T>(keys: string[]) {
     const db = this._db
     if (keys.length === 0) {
       return {}
@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: Record<string, any> = {}
+      let final: Record<string, T> = {}
       let count = 0
       for (let result of response) {
         if (result) {

packages/backend-core/src/security/permissions.ts

@@ -1,8 +1,9 @@
-import { PermissionType, PermissionLevel } from "@budibase/types"
-export { PermissionType, PermissionLevel } from "@budibase/types"
+import { PermissionLevel, PermissionType } from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
+export { PermissionType, PermissionLevel } from "@budibase/types"
+
 export type RoleHierarchy = {
   permissionId: string
 }[]
@@ -78,6 +79,7 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -88,6 +90,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   POWER: {
@@ -99,6 +102,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   ADMIN: {
@@ -111,6 +115,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
 }

packages/backend-core/src/security/roles.ts

@@ -215,21 +215,23 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
   return roles
 }
 
+export async function getUserRoleIdHierarchy(
+  userRoleId?: string
+): Promise<string[]> {
+  const roles = await getUserRoleHierarchy(userRoleId)
+  return roles.map(role => role._id!)
+}
+
 /**
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param {string} userRoleId The user's role ID, this can be found in their access token.
- * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
- * @returns {Promise<string[]|object[]>} returns an ordered array of the roles, with the first being their
+ * @returns {Promise<object[]>} returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(
-  userRoleId?: string,
-  opts = { idOnly: true }
-) {
+export async function getUserRoleHierarchy(userRoleId?: string) {
   // special case, if they don't have a role then they are a public user
-  const roles = await getAllUserRoles(userRoleId)
-  return opts.idOnly ? roles.map(role => role._id) : roles
+  return getAllUserRoles(userRoleId)
 }
 
 // this function checks that the provided permissions are in an array format
@@ -249,6 +251,11 @@ export function checkForRoleResourceArray(
   return rolePerms
 }
 
+export async function getAllRoleIds(appId?: string) {
+  const roles = await getAllRoles(appId)
+  return roles.map(role => role._id)
+}
+
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
  * @return {Promise<object[]>} An array of the role objects that were found.
@@ -332,9 +339,7 @@ export class AccessController {
     }
     let roleIds = userRoleId ? this.userHierarchies[userRoleId] : null
     if (!roleIds && userRoleId) {
-      roleIds = (await getUserRoleHierarchy(userRoleId, {
-        idOnly: true,
-      })) as string[]
+      roleIds = await getUserRoleIdHierarchy(userRoleId)
       this.userHierarchies[userRoleId] = roleIds
     }
 

packages/backend-core/src/users/db.ts

@@ -278,7 +278,12 @@ export class UserDB {
         builtUser._rev = response.rev
 
         await eventHelpers.handleSaveEvents(builtUser, dbUser)
-        await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+        await platform.users.addUser(
+          tenantId,
+          builtUser._id!,
+          builtUser.email,
+          builtUser.ssoId
+        )
         await cache.user.invalidateUser(response.id)
 
         await Promise.all(groupPromises)

packages/backend-core/src/users/users.ts

@@ -14,13 +14,14 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
+  ContextUser,
+  SearchQuery,
+  SearchQueryOperators,
   SearchUsersRequest,
   User,
-  ContextUser,
 } from "@budibase/types"
-import { getGlobalDB } from "../context"
 import * as context from "../context"
-import { user as userCache } from "../cache"
+import { getGlobalDB } from "../context"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -39,6 +40,31 @@ function removeUserPassword(users: User | User[]) {
   return users
 }
 
+export const isSupportedUserSearch = (query: SearchQuery) => {
+  const allowed = [
+    { op: SearchQueryOperators.STRING, key: "email" },
+    { op: SearchQueryOperators.EQUAL, key: "_id" },
+  ]
+  for (let [key, operation] of Object.entries(query)) {
+    if (typeof operation !== "object") {
+      return false
+    }
+    const fields = Object.keys(operation || {})
+    // this filter doesn't contain options - ignore
+    if (fields.length === 0) {
+      continue
+    }
+    const allowedOperation = allowed.find(
+      allow =>
+        allow.op === key && fields.length === 1 && fields[0] === allow.key
+    )
+    if (!allowedOperation) {
+      return false
+    }
+  }
+  return true
+}
+
 export const bulkGetGlobalUsersById = async (
   userIds: string[],
   opts?: GetOpts
@@ -211,8 +237,8 @@ export const searchGlobalUsersByEmail = async (
 
 const PAGE_LIMIT = 8
 export const paginatedUsers = async ({
-  page,
-  email,
+  bookmark,
+  query,
   appId,
 }: SearchUsersRequest = {}) => {
   const db = getGlobalDB()
@@ -222,18 +248,20 @@ export const paginatedUsers = async ({
     limit: PAGE_LIMIT + 1,
   }
   // add a startkey if the page was specified (anchor)
-  if (page) {
-    opts.startkey = page
+  if (bookmark) {
+    opts.startkey = bookmark
   }
   // property specifies what to use for the page/anchor
   let userList: User[],
     property = "_id",
     getKey
-  if (appId) {
+  if (query?.equal?._id) {
+    userList = [await getById(query.equal._id)]
+  } else if (appId) {
     userList = await searchGlobalUsersByApp(appId, opts)
     getKey = (doc: any) => getGlobalUserByAppPage(appId, doc)
-  } else if (email) {
-    userList = await searchGlobalUsersByEmail(email, opts)
+  } else if (query?.string?.email) {
+    userList = await searchGlobalUsersByEmail(query?.string?.email, opts)
     property = "email"
   } else {
     // no search, query allDocs

packages/backend-core/src/utils/utils.ts

@@ -10,7 +10,7 @@ import {
   Event,
   TenantResolutionStrategy,
 } from "@budibase/types"
-import { SetOption } from "cookies"
+import type { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -86,8 +86,8 @@ export const useAuditLogs = () => {
   return useFeature(Feature.AUDIT_LOGS)
 }
 
-export const usePublicApiUserRoles = () => {
-  return useFeature(Feature.USER_ROLE_PUBLIC_API)
+export const useExpandedPublicApi = () => {
+  return useFeature(Feature.EXPANDED_PUBLIC_API)
 }
 
 export const useScimIntegration = () => {

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -1,4 +1,4 @@
-import { generator, uuid, quotas } from "."
+import { generator, quotas, uuid } from "."
 import { generateGlobalUserID } from "../../../../src/docIds"
 import {
   Account,
@@ -6,10 +6,11 @@ import {
   AccountSSOProviderType,
   AuthType,
   CloudAccount,
-  Hosting,
-  SSOAccount,
   CreateAccount,
   CreatePassswordAccount,
+  CreateVerifiableSSOAccount,
+  Hosting,
+  SSOAccount,
 } from "@budibase/types"
 import sample from "lodash/sample"
 
@@ -68,6 +69,23 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
   }
 }
 
+export function verifiableSsoAccount(
+  account: Account = cloudAccount()
+): SSOAccount {
+  return {
+    ...account,
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: AccountSSOProvider.MICROSOFT,
+    providerType: AccountSSOProviderType.MICROSOFT,
+    thirdPartyProfile: { id: "abc123" },
+  }
+}
+
 export const cloudCreateAccount: CreatePassswordAccount = {
   email: "cloud@budibase.com",
   tenantId: "cloud",
@@ -91,6 +109,19 @@ export const cloudSSOCreateAccount: CreateAccount = {
   profession: "Software Engineer",
 }
 
+export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
+  email: "cloud-sso@budibase.com",
+  tenantId: "cloud-sso",
+  hosting: Hosting.CLOUD,
+  authType: AuthType.SSO,
+  tenantName: "cloudsso",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+  provider: AccountSSOProvider.MICROSOFT,
+  thirdPartyProfile: { id: "abc123" },
+}
+
 export const selfCreateAccount: CreatePassswordAccount = {
   email: "self@budibase.com",
   tenantId: "self",

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -10,14 +10,13 @@ import { authDetails } from "./sso"
 import { uuid } from "./common"
 import { generator } from "./generator"
 import { tenant } from "."
-import { generateGlobalUserID } from "../../../../src/docIds"
 
 export const newEmail = () => {
   return `${uuid()}@test.com`
 }
 
 export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
-  const userId = userProps?._id || generateGlobalUserID()
+  const userId = userProps?._id
   return {
     _id: userId,
     userId,
@@ -53,7 +52,7 @@ export const adminOnlyUser = (userProps?: any): AdminOnlyUser => {
   }
 }
 
-export const builderUser = (userProps?: any): BuilderUser => {
+export const builderUser = (userProps?: Partial<User>): BuilderUser => {
   return {
     ...user(userProps),
     builder: {

packages/backend-core/tests/extra/DBTestConfiguration.ts

@@ -18,7 +18,7 @@ class DBTestConfiguration {
 
   // TENANCY
 
-  doInTenant(task: any) {
+  doInTenant<T>(task: () => Promise<T>) {
     return context.doInTenant(this.tenantId, () => {
       return task()
     })

packages/backend-core/tests/index.ts

@@ -1 +1,2 @@
 export * from "./core/utilities"
+export * from "./extra"

packages/bbui/package.json

@@ -20,14 +20,12 @@
     "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
-    "cross-env": "^7.0.2",
-    "nollup": "^0.14.1",
     "postcss": "^8.2.9",
     "rollup": "^2.45.2",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "svelte": "^3.38.2"
+    "svelte": "3.49.0"
   },
   "keywords": [
     "svelte"
@@ -84,9 +82,9 @@
     "@spectrum-css/vars": "3.0.1",
     "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
+    "svelte-dnd-action": "^0.9.8",
     "svelte-flatpickr": "3.2.3",
-    "svelte-portal": "^1.0.0",
-    "svelte-dnd-action": "^0.9.8"
+    "svelte-portal": "^1.0.0"
   },
   "resolutions": {
     "loader-utils": "1.4.1"

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,12 +14,12 @@
   export let autocomplete = false
   export let sort = false
   export let autoWidth = false
-  export let fetchTerm = null
-  export let useFetch = false
+  export let searchTerm = null
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let open = false
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -82,6 +82,7 @@
 </script>
 
 <Picker
+  on:loadMore
   {id}
   {error}
   {disabled}
@@ -90,9 +91,8 @@
   {options}
   isPlaceholder={!arrayValue.length}
   {autocomplete}
-  bind:fetchTerm
+  bind:searchTerm
   bind:open
-  {useFetch}
   {isOptionSelected}
   {getOptionLabel}
   {getOptionValue}
@@ -102,4 +102,5 @@
   {customPopoverHeight}
   {customPopoverOffsetBelow}
   {customPopoverMaxHeight}
+  {loading}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -2,7 +2,7 @@
   import "@spectrum-css/picker/dist/index-vars.css"
   import "@spectrum-css/popover/dist/index-vars.css"
   import "@spectrum-css/menu/dist/index-vars.css"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onDestroy } from "svelte"
   import clickOutside from "../../Actions/click_outside"
   import Search from "./Search.svelte"
   import Icon from "../../Icon/Icon.svelte"
@@ -10,6 +10,7 @@
   import Popover from "../../Popover/Popover.svelte"
   import Tags from "../../Tags/Tags.svelte"
   import Tag from "../../Tags/Tag.svelte"
+  import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
 
   export let id = null
   export let disabled = false
@@ -35,19 +36,20 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
-  export let fetchTerm = null
-  export let useFetch = false
+  export let searchTerm = null
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let align = "left"
   export let footer = null
   export let customAnchor = null
+  export let loading
+
   const dispatch = createEventDispatcher()
 
-  let searchTerm = null
   let button
   let popover
+  let component
 
   $: sortedOptions = getSortedOptions(options, getOptionLabel, sort)
   $: filteredOptions = getFilteredOptions(
@@ -82,7 +84,7 @@
   }
 
   const getFilteredOptions = (options, term, getLabel) => {
-    if (autocomplete && term && !fetchTerm) {
+    if (autocomplete && term) {
       const lowerCaseTerm = term.toLowerCase()
       return options.filter(option => {
         return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm)
@@ -90,6 +92,20 @@
     }
     return options
   }
+
+  const onScroll = e => {
+    const scrollPxThreshold = 100
+    const scrollPositionFromBottom =
+      e.target.scrollHeight - e.target.clientHeight - e.target.scrollTop
+    if (scrollPositionFromBottom < scrollPxThreshold) {
+      dispatch("loadMore")
+    }
+  }
+
+  $: component?.addEventListener("scroll", onScroll)
+  onDestroy(() => {
+    component?.removeEventListener("scroll", null)
+  })
 </script>
 
 <button
@@ -163,14 +179,13 @@
   >
     {#if autocomplete}
       <Search
-        value={useFetch ? fetchTerm : searchTerm}
-        on:change={event =>
-          useFetch ? (fetchTerm = event.detail) : (searchTerm = event.detail)}
+        value={searchTerm}
+        on:change={event => (searchTerm = event.detail)}
         {disabled}
         placeholder="Search"
       />
     {/if}
-    <ul class="spectrum-Menu" role="listbox">
+    <ul class="spectrum-Menu" role="listbox" bind:this={component}>
       {#if placeholderOption}
         <li
           class="spectrum-Menu-item placeholder"
@@ -248,6 +263,12 @@
       {/if}
     </ul>
 
+    {#if loading}
+      <div class="loading" class:loading--withAutocomplete={autocomplete}>
+        <ProgressCircle size="S" />
+      </div>
+    {/if}
+
     {#if footer}
       <div class="footer">
         {footer}
@@ -323,18 +344,19 @@
   /* Search styles inside popover */
   .popover-content :global(.spectrum-Search) {
     margin-top: -1px;
-    margin-left: -1px;
-    width: calc(100% + 2px);
+    width: 100%;
   }
   .popover-content :global(.spectrum-Search input) {
     height: auto;
     border-bottom-left-radius: 0;
     border-bottom-right-radius: 0;
+    border-left: 0;
+    border-right: 0;
     padding-top: var(--spectrum-global-dimension-size-100);
     padding-bottom: var(--spectrum-global-dimension-size-100);
   }
   .popover-content :global(.spectrum-Search .spectrum-ClearButton) {
-    right: 1px;
+    right: 2px;
     top: 2px;
   }
   .popover-content :global(.spectrum-Search .spectrum-Textfield-icon) {
@@ -359,4 +381,14 @@
   .option-tag :global(.spectrum-Tags-item > .spectrum-Icon) {
     margin-top: 2px;
   }
+
+  .loading {
+    position: fixed;
+    justify-content: center;
+    right: var(--spacing-s);
+    top: var(--spacing-s);
+  }
+  .loading--withAutocomplete {
+    top: calc(34px + var(--spacing-m));
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -25,6 +25,8 @@
   export let tag = null
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
+  export let searchTerm = null
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -65,6 +67,8 @@
 <Picker
   on:click
   bind:open
+  bind:searchTerm
+  on:loadMore
   {quiet}
   {id}
   {error}
@@ -92,4 +96,5 @@
   placeholderOption={placeholder === false ? null : placeholder}
   isOptionSelected={option => option === value}
   onSelectOption={selectOption}
+  {loading}
 />

packages/bbui/src/Form/Core/TextField.svelte

@@ -96,8 +96,8 @@
     {disabled}
     {readonly}
     {id}
-    value={value || ""}
-    placeholder={placeholder || ""}
+    value={value ?? ""}
+    placeholder={placeholder ?? ""}
     on:click
     on:blur
     on:focus

packages/bbui/src/Form/Multiselect.svelte

@@ -16,8 +16,7 @@
   export let sort = false
   export let autoWidth = false
   export let autocomplete = false
-  export let fetchTerm = null
-  export let useFetch = false
+  export let searchTerm = null
   export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
@@ -41,8 +40,7 @@
     {autoWidth}
     {autocomplete}
     {customPopoverHeight}
-    bind:fetchTerm
-    {useFetch}
+    bind:searchTerm
     on:change={onChange}
     on:click
   />

packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte

@@ -21,14 +21,6 @@
     "hsla(240, 90%, 75%, 0.3)",
     "hsla(320, 90%, 75%, 0.3)",
   ]
-  $: {
-    if (constraints.inclusion.length) {
-      options = constraints.inclusion.map(value => ({
-        name: value,
-        id: Math.random(),
-      }))
-    }
-  }
   const removeInput = idx => {
     delete optionColors[options[idx].name]
     constraints.inclusion = constraints.inclusion.filter((e, i) => i !== idx)
@@ -80,6 +72,11 @@
     // Initialize anchor arrays on mount, assuming 'options' is already populated
     colorPopovers = constraints.inclusion.map(() => undefined)
     anchors = constraints.inclusion.map(() => undefined)
+
+    options = constraints.inclusion.map(value => ({
+      name: value,
+      id: Math.random(),
+    }))
   })
 </script>
 

packages/bbui/src/Table/CellRenderer.svelte

@@ -25,6 +25,7 @@
     longform: StringRenderer,
     array: ArrayRenderer,
     internal: InternalRenderer,
+    bb_reference: RelationshipRenderer,
   }
   $: type = getType(schema)
   $: customRenderer = customRenderers?.find(x => x.column === schema?.name)

packages/builder/build/copy.js

@@ -1,8 +0,0 @@
-const ncp = require("ncp").ncp
-
-ncp("./dist", "../server/builder", function (err) {
-  if (err) {
-    return console.error(err)
-  }
-  console.log("Copied dist folder to ../server/builder")
-})

packages/builder/package.json

@@ -65,7 +65,6 @@
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
     "@sentry/browser": "5.19.1",
-    "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
@@ -75,20 +74,19 @@
     "lodash": "4.17.21",
     "posthog-js": "^1.36.0",
     "remixicon": "2.5.0",
+    "sanitize-html": "^2.7.0",
     "shortid": "2.2.15",
     "svelte-dnd-action": "^0.9.8",
     "svelte-loading-spinners": "^0.1.1",
     "svelte-portal": "1.0.0",
-    "uuid": "8.3.1",
     "yup": "0.29.2"
   },
   "devDependencies": {
     "@babel/core": "^7.12.14",
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-env": "^7.13.12",
-    "@babel/runtime": "^7.13.10",
-    "@rollup/plugin-replace": "^2.4.2",
-    "@roxi/routify": "2.18.5",
+    "@rollup/plugin-replace": "^5.0.3",
+    "@roxi/routify": "2.18.12",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
     "@testing-library/jest-dom": "5.17.0",
     "@testing-library/svelte": "^3.2.2",
@@ -96,26 +94,19 @@
     "identity-obj-proxy": "^3.0.0",
     "jest": "29.6.2",
     "jsdom": "^21.1.1",
-    "mochawesome": "^7.1.3",
-    "mochawesome-merge": "^4.2.1",
-    "mochawesome-report-generator": "^6.2.0",
     "ncp": "^2.0.0",
-    "rimraf": "^3.0.2",
-    "rollup": "^2.44.0",
-    "rollup-plugin-copy": "^3.4.0",
-    "start-server-and-test": "^1.12.1",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3",
-    "vite": "^3.0.8",
-    "vite-plugin-static-copy": "^0.16.0",
+    "vite": "^4.4.11",
+    "vite-plugin-static-copy": "^0.17.0",
     "vitest": "^0.29.2"
   },
   "nx": {
     "targets": {
       "build": {
+        "outputs": [
+          "{workspaceRoot}/packages/server/builder"
+        ],
         "dependsOn": [
           {
             "projects": [

packages/builder/src/analytics/constants.js

@@ -2,6 +2,7 @@ export const Events = {
   COMPONENT_CREATED: "component:created",
   COMPONENT_UPDATED: "component:updated",
   APP_VIEW_PUBLISHED: "app:view_published",
+  BLOCK_EJECTED: "block:ejected",
 }
 
 export const EventSource = {

packages/builder/src/builderStore/dataBinding.js

@@ -996,12 +996,15 @@ export const buildFormSchema = (component, asset) => {
 
   if (component._component.endsWith("formblock")) {
     let schema = {}
-
     const datasource = getDatasourceForProvider(asset, component)
     const info = getSchemaForDatasource(component, datasource)
 
+    if (!info?.schema) {
+      return schema
+    }
+
     if (!component.fields) {
-      Object.values(info?.schema)
+      Object.values(info.schema)
         .filter(
           ({ autocolumn, name }) =>
             !autocolumn && !["_rev", "_id"].includes(name)

packages/builder/src/builderStore/store/automation/index.js

@@ -221,18 +221,6 @@ const automationActions = store => ({
     newAutomation.definition.steps.splice(blockIdx, 0, block)
     await store.actions.save(newAutomation)
   },
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  toggleRowControl: async (block, rowControl) => {
-    const newBlock = { ...block, rowControl }
-    const newAutomation = store.actions.getUpdatedDefinition(
-      get(selectedAutomation),
-      newBlock
-    )
-    await store.actions.save(newAutomation)
-  },
   deleteAutomationBlock: async block => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)

packages/builder/src/builderStore/store/frontend.js

@@ -64,6 +64,7 @@ const INITIAL_FRONTEND_STATE = {
   },
   features: {
     componentValidation: false,
+    disableUserMetadata: false,
   },
   errors: [],
   hasAppPackage: false,
@@ -1286,6 +1287,11 @@ export const getFrontendStore = () => {
             return false
           }
 
+          // Log event
+          analytics.captureEvent(Events.BLOCK_EJECTED, {
+            block: block._component,
+          })
+
           // Attach block children back into ejected definition, using the
           // _containsSlot flag to know where to insert them
           const slotContainer = findAllMatchingComponents(

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -33,6 +33,8 @@ const generateTableBlock = datasource => {
       showTitleButton: true,
       titleButtonText: "Create row",
       titleButtonClickBehaviour: "new",
+      sidePanelSaveLabel: "Save",
+      sidePanelDeleteLabel: "Delete",
     })
     .instanceName(`${datasource.label} - Table block`)
   return tableBlock

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -7,7 +7,6 @@
     Detail,
     Modal,
     Button,
-    Select,
     ActionButton,
     notifications,
     Label,
@@ -39,9 +38,6 @@
     step => step.stepId === ActionStepID.COLLECT
   )
   $: automationId = $selectedAutomation?._id
-  $: showBindingPicker =
-    block.stepId === ActionStepID.CREATE_ROW ||
-    block.stepId === ActionStepID.UPDATE_ROW
   $: isTrigger = block.type === "TRIGGER"
   $: steps = $selectedAutomation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
@@ -96,15 +92,6 @@
     }
   }
 
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  function toggleRowControl(evt) {
-    const rowControl = evt.detail !== "Use values"
-    automationStore.actions.toggleRowControl(block, rowControl)
-  }
-
   async function addLooping() {
     const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
     const loopBlock = automationStore.actions.constructBlock(
@@ -189,16 +176,6 @@
                   Add Looping
                 </ActionButton>
               {/if}
-              {#if showBindingPicker}
-                <Select
-                  on:change={toggleRowControl}
-                  defaultValue="Use values"
-                  autoWidth
-                  value={block.rowControl ? "Use bindings" : "Use values"}
-                  options={["Use values", "Use bindings"]}
-                  placeholder={null}
-                />
-              {/if}
               <ActionButton
                 on:click={() => deleteStep()}
                 icon="DeleteOutline"

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -23,6 +23,7 @@
   import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import CodeEditorModal from "./CodeEditorModal.svelte"
   import QuerySelector from "./QuerySelector.svelte"
@@ -82,33 +83,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  /**
-   * TODO - Remove after November 2023
-   * *******************************
-   * Code added to provide backwards compatibility between Values 1,2,3,4,5
-   * and the new JSON body.
-   */
-  let deprecatedSchemaProperties
-  $: {
-    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
-      deprecatedSchemaProperties = schemaProperties.filter(
-        prop => !prop[0].startsWith("value")
-      )
-      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
-        deprecatedSchemaProperties.push([
-          "body",
-          {
-            title: "Payload",
-            type: "json",
-          },
-        ])
-      }
-    } else {
-      deprecatedSchemaProperties = schemaProperties
-    }
-  }
-  /****************************************************/
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -118,30 +92,6 @@
       newInputData = cloneDeep(blockInputs)
     }
 
-    /**
-     * TODO - Remove after November 2023
-     * *******************************
-     * Code added to provide backwards compatibility between Values 1,2,3,4,5
-     * and the new JSON body.
-     */
-    if (
-      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
-      !newInputData?.body?.value
-    ) {
-      let deprecatedValues = {
-        ...newInputData,
-      }
-      delete deprecatedValues.url
-      delete deprecatedValues.body
-      newInputData = {
-        url: newInputData.url,
-        body: {
-          value: JSON.stringify(deprecatedValues),
-        },
-      }
-    }
-    /**********************************/
-
     inputData = newInputData
     setDefaultEnumValues()
   }
@@ -337,7 +287,7 @@
 </script>
 
 <div class="fields">
-  {#each deprecatedSchemaProperties as [key, value]}
+  {#each schemaProperties as [key, value]}
     {#if canShowField(key, value)}
       <div class="block-field">
         {#if key !== "fields" && value.type !== "boolean"}
@@ -362,18 +312,6 @@
             mode="json"
             value={inputData[key]?.value}
             on:change={e => {
-              /**
-               * TODO - Remove after November 2023
-               * *******************************
-               * Code added to provide backwards compatibility between Values 1,2,3,4,5
-               * and the new JSON body.
-               */
-              delete inputData.value1
-              delete inputData.value2
-              delete inputData.value3
-              delete inputData.value4
-              delete inputData.value5
-              /***********************/
               onChange(e, key)
             }}
           />
@@ -386,10 +324,23 @@
             />
           </div>
         {:else if value.type === "date"}
-          <DatePicker
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            panel={AutomationBindingPanel}
+            type={"date"}
             value={inputData[key]}
             on:change={e => onChange(e, key)}
-          />
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
+            <DatePicker
+              value={inputData[key]}
+              on:change={e => onChange(e, key)}
+            />
+          </DrawerBindableSlot>
         {:else if value.customType === "column"}
           <Select
             on:change={e => onChange(e, key)}
@@ -469,7 +420,6 @@
           />
         {:else if value.customType === "row"}
           <RowSelector
-            {block}
             value={inputData[key]}
             meta={inputData["meta"] || {}}
             on:change={e => {

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,18 +1,16 @@
 <script>
   import { tables } from "stores/backend"
   import { Select, Checkbox } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
-  import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let value
   export let meta
   export let bindings
-  export let block
   export let isTestModal
   export let isUpdateRow
 
@@ -25,16 +23,6 @@
   let table
   let schemaFields
 
-  let placeholders = {
-    number: 10,
-    boolean: "true",
-    datetime: "2022-02-16T12:00:00.000Z ",
-    options: "1",
-    array: "1 2 3 4",
-    link: "ro_ta_123_456",
-    longform: "long form text",
-  }
-  $: rowControl = block.rowControl
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
     schemaFields = Object.entries(table?.schema ?? {})
@@ -57,19 +45,13 @@
       return value
     }
 
-    if (type === "boolean") {
-      if (typeof value === "boolean") {
-        return value
-      }
-      return value === "true"
-    }
     if (type === "number") {
       if (typeof value === "number") {
         return value
       }
       return Number(value)
     }
-    if (type === "options") {
+    if (type === "options" || type === "boolean") {
       return value
     }
     if (type === "array") {
@@ -83,8 +65,7 @@
       if (Array.isArray(value)) {
         return value
       }
-
-      return [value]
+      return value.split(",").map(x => x.trim())
     }
 
     if (type === "json") {
@@ -128,9 +109,31 @@
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
-      {#if !schema.autocolumn}
-        {#if schema.type !== "attachment"}
-          {#if !rowControl}
+      {#if !schema.autocolumn && schema.type !== "attachment"}
+        {#if isTestModal}
+          <RowSelectorTypes
+            {isTestModal}
+            {field}
+            {schema}
+            bindings={parsedBindings}
+            {value}
+            {onChange}
+          />
+        {:else}
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            label={field}
+            panel={AutomationBindingPanel}
+            type={schema.type}
+            {schema}
+            value={value[field]}
+            on:change={e => onChange(e, field)}
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
             <RowSelectorTypes
               {isTestModal}
               {field}
@@ -139,37 +142,19 @@
               {value}
               {onChange}
             />
-          {:else}
-            <div>
-              <svelte:component
-                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
-                placeholder={placeholders[schema.type]}
-                panel={AutomationBindingPanel}
-                value={Array.isArray(value[field])
-                  ? value[field].join(" ")
-                  : value[field]}
-                on:change={e => onChange(e, field, schema.type)}
-                label={field}
-                type="string"
-                bindings={parsedBindings}
-                fillWidth={true}
-                allowJS={true}
-                updateOnChange={false}
-              />
-              {#if isUpdateRow && schema.type === "link"}
-                <div class="checkbox-field">
-                  <Checkbox
-                    value={meta.fields?.[field]?.clearRelationships}
-                    text={"Clear relationships if empty?"}
-                    size={"S"}
-                    on:change={e => onChangeSetting(e, field)}
-                  />
-                </div>
-              {/if}
-            </div>
-          {/if}
+          </DrawerBindableSlot>
         {/if}
       {/if}
+      {#if isUpdateRow && schema.type === "link"}
+        <div class="checkbox-field">
+          <Checkbox
+            value={meta.fields?.[field]?.clearRelationships}
+            text={"Clear relationships if empty?"}
+            size={"S"}
+            on:change={e => onChangeSetting(e, field)}
+          />
+        </div>
+      {/if}
     {/each}
   </div>
 {/if}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,7 +1,6 @@
 <script>
   import {
     Select,
-    Toggle,
     DatePicker,
     Multiselect,
     TextArea,
@@ -45,19 +44,28 @@
     on:change={e => onChange(e, field)}
   />
 {:else if schema.type === "boolean"}
-  <Toggle
-    text={field}
-    value={value[field]}
+  <Select
     on:change={e => onChange(e, field)}
+    label={field}
+    value={value[field]}
+    options={[
+      { label: "True", value: "true" },
+      { label: "False", value: "false" },
+    ]}
   />
 {:else if schema.type === "array"}
   <Multiselect
     bind:value={value[field]}
     label={field}
     options={schema.constraints.inclusion}
+    on:change={e => onChange(e, field)}
   />
 {:else if schema.type === "longform"}
-  <TextArea label={field} bind:value={value[field]} />
+  <TextArea
+    label={field}
+    bind:value={value[field]}
+    on:change={e => onChange(e, field)}
+  />
 {:else if schema.type === "json"}
   <span>
     <Label>{field}</Label>
@@ -73,7 +81,11 @@
     />
   </span>
 {:else if schema.type === "link"}
-  <LinkedRowSelector bind:linkedRows={value[field]} {schema} />
+  <LinkedRowSelector
+    bind:linkedRows={value[field]}
+    {schema}
+    on:change={e => onChange(e, field)}
+  />
 {:else if schema.type === "string" || schema.type === "number"}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}

packages/builder/src/components/backend/DataTable/ExternalDataSourceTable.svelte

@@ -1,33 +0,0 @@
-<script>
-  import Table from "./Table.svelte"
-
-  export let query = {}
-  export let data = []
-  export let editRows = false
-
-  let loading = false
-  let error = false
-  let type = "external"
-</script>
-
-{#if error}
-  <div class="errors">{error}</div>
-{/if}
-<Table
-  schema={query.schema}
-  {data}
-  {loading}
-  {type}
-  rowCount={5}
-  allowEditing={editRows}
-/>
-
-<style>
-  .errors {
-    color: var(--red);
-    background: var(--red-light);
-    padding: var(--spacing-m);
-    border-radius: var(--border-radius-m);
-    margin-bottom: var(--spacing-m);
-  }
-</style>

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -4,6 +4,7 @@
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
+  import { store } from "builderStore"
   import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
@@ -14,6 +15,7 @@
   import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
   import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
+  import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
 
   const userSchemaOverrides = {
     firstName: { displayName: "First name", disabled: true },
@@ -59,22 +61,22 @@
     datasource={gridDatasource}
     canAddRows={!isUsersTable}
     canDeleteRows={!isUsersTable}
+    canEditRows={!isUsersTable || !$store.features.disableUserMetadata}
+    canEditColumns={!isUsersTable || !$store.features.disableUserMetadata}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     on:updatedatasource={handleGridTableUpdate}
   >
     <svelte:fragment slot="filter">
+      {#if isUsersTable && $store.features.disableUserMetadata}
+        <GridUsersTableButton />
+      {/if}
       <GridFilterButton />
     </svelte:fragment>
-    <svelte:fragment slot="edit-column">
-      <GridEditColumnModal />
-    </svelte:fragment>
-    <svelte:fragment slot="add-column">
-      <GridAddColumnModal />
-    </svelte:fragment>
-
     <svelte:fragment slot="controls">
-      <GridCreateViewButton />
+      {#if !isUsersTable}
+        <GridCreateViewButton />
+      {/if}
       <GridManageAccessButton />
       {#if relationshipsEnabled}
         <GridRelationshipButton />
@@ -84,7 +86,6 @@
       {:else}
         <GridImportButton />
       {/if}
-
       <GridExportButton />
       {#if isUsersTable}
         <GridEditUserModal />
@@ -92,6 +93,12 @@
         <GridCreateEditRowModal />
       {/if}
     </svelte:fragment>
+    <svelte:fragment slot="edit-column">
+      <GridEditColumnModal />
+    </svelte:fragment>
+    <svelte:fragment slot="add-column">
+      <GridAddColumnModal />
+    </svelte:fragment>
   </Grid>
 </div>
 

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -13,7 +13,13 @@
   let modal
 
   $: tempValue = filters || []
-  $: schemaFields = Object.values(schema || {})
+  $: schemaFields = Object.entries(schema || {}).map(
+    ([fieldName, fieldSchema]) => ({
+      name: fieldName, // Using the key as name if not defined in the schema, for example in some autogenerated columns
+      ...fieldSchema,
+    })
+  )
+
   $: text = getText(filters)
   $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -5,7 +5,6 @@
     Label,
     Select,
     Toggle,
-    RadioGroup,
     Icon,
     DatePicker,
     Modal,
@@ -26,16 +25,18 @@
     ALLOWABLE_STRING_TYPES,
     ALLOWABLE_NUMBER_TYPES,
     SWITCHABLE_TYPES,
+    PrettyRelationshipDefinitions,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { truncate } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
+  import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
+  import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
-  const AUTO_TYPE = "auto"
+  const AUTO_TYPE = FIELDS.AUTO.type
   const FORMULA_TYPE = FIELDS.FORMULA.type
   const LINK_TYPE = FIELDS.LINK.type
   const STRING_TYPE = FIELDS.STRING.type
@@ -50,13 +51,36 @@
   export let field
 
   let mounted = false
-  let fieldDefinitions = cloneDeep(FIELDS)
+  const fieldDefinitions = Object.values(FIELDS).reduce(
+    // Storing the fields by complex field id
+    (acc, field) => ({
+      ...acc,
+      [makeFieldId(field.type, field.subtype)]: field,
+    }),
+    {}
+  )
+
+  function makeFieldId(type, subtype, autocolumn) {
+    // don't make field IDs for auto types
+    if (type === AUTO_TYPE || autocolumn) {
+      return type.toUpperCase()
+    } else {
+      return `${type}${subtype || ""}`.toUpperCase()
+    }
+  }
+
   let originalName
   let linkEditDisabled
   let primaryDisplay
   let indexes = [...($tables.selected.indexes || [])]
   let isCreating = undefined
 
+  let relationshipPart1 = PrettyRelationshipDefinitions.Many
+  let relationshipPart2 = PrettyRelationshipDefinitions.One
+
+  let relationshipTableIdPrimary = null
+  let relationshipTableIdSecondary = null
+
   let table = $tables.selected
   let confirmDeleteDialog
   let savingColumn
@@ -64,18 +88,65 @@
   let jsonSchemaModal
   let allowedTypes = []
   let editableColumn = {
-    type: "string",
-    constraints: fieldDefinitions.STRING.constraints,
+    type: FIELDS.STRING.type,
+    constraints: FIELDS.STRING.constraints,
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
+  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
 
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
 
+  let relationshipMap = {
+    [RelationshipType.ONE_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.ONE,
+    },
+    [RelationshipType.MANY_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
+    [RelationshipType.MANY_TO_ONE]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
+  }
+
+  $: {
+    // this parses any changes the user has made when creating a new internal relationship
+    // into what we expect the schema to look like
+    if (editableColumn.type === LINK_TYPE) {
+      relationshipTableIdPrimary = table._id
+      if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts2 = relationshipOpts2.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+      }
+
+      if (relationshipPart2 === PrettyRelationshipDefinitions.ONE) {
+        relationshipOpts1 = relationshipOpts1.filter(
+          opt => opt !== PrettyRelationshipDefinitions.ONE
+        )
+      } else {
+        relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+      }
+      // Determine the relationship type based on the selected values of both parts
+      editableColumn.relationshipType = Object.entries(relationshipMap).find(
+        ([_, parts]) =>
+          parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2
+      )?.[0]
+      // Set the tableId based on the selected table
+      editableColumn.tableId = relationshipTableIdSecondary
+    }
+  }
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
+
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
@@ -83,6 +154,20 @@
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
+
+      // Here we are setting the relationship values based on the editableColumn
+      // This part of the code is used when viewing an existing field hence the check
+      // for the tableId
+      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+        relationshipTableIdPrimary = table._id
+        relationshipTableIdSecondary = editableColumn.tableId
+        if (editableColumn.relationshipType in relationshipMap) {
+          const { part1, part2 } =
+            relationshipMap[editableColumn.relationshipType]
+          relationshipPart1 = part1
+          relationshipPart2 = part2
+        }
+      }
     } else if (!savingColumn) {
       let highestNumber = 0
       Object.keys(table.schema).forEach(columnName => {
@@ -99,7 +184,19 @@
         editableColumn.name = "Column 01"
       }
     }
-    allowedTypes = getAllowedTypes()
+
+    if (!savingColumn) {
+      editableColumn.fieldId = makeFieldId(
+        editableColumn.type,
+        editableColumn.subtype,
+        editableColumn.autocolumn
+      )
+
+      allowedTypes = getAllowedTypes().map(t => ({
+        fieldId: makeFieldId(t.type, t.subtype),
+        ...t,
+      }))
+    }
   }
 
   $: initialiseField(field, savingColumn)
@@ -157,14 +254,10 @@
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
-  $: relationshipOptions = getRelationshipOptions(editableColumn)
   $: external = table.type === "external"
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
-    opt =>
-      opt._id !== $tables.selected._id &&
-      opt.type === table.type &&
-      table.sourceId === opt.sourceId
+    opt => opt.type === table.type && table.sourceId === opt.sourceId
   )
   $: typeEnabled =
     !originalName ||
@@ -180,6 +273,8 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
+    delete saveColumn.fieldId
+
     if (saveColumn.type === AUTO_TYPE) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
@@ -200,10 +295,7 @@
       dispatch("updatecolumns")
       gridDispatch("close-edit-column")
 
-      if (
-        saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
-      ) {
+      if (saveColumn.type === LINK_TYPE) {
         // Fetching the new tables
         tables.fetch()
         // Fetching the new relationships
@@ -235,31 +327,43 @@
         confirmDeleteDialog.hide()
         dispatch("updatecolumns")
         gridDispatch("close-edit-column")
+
+        if (editableColumn.type === LINK_TYPE) {
+          // Updating the relationships
+          datasources.fetch()
+        }
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
     }
   }
 
-  function handleTypeChange(event) {
+  function onHandleTypeChange(event) {
+    handleTypeChange(event.detail)
+  }
+
+  function handleTypeChange(type) {
     // remove any extra fields that may not be related to this type
     delete editableColumn.autocolumn
     delete editableColumn.subtype
     delete editableColumn.tableId
     delete editableColumn.relationshipType
     delete editableColumn.formulaType
+    delete editableColumn.constraints
 
     // Add in defaults and initial definition
-    const definition = fieldDefinitions[event.detail?.toUpperCase()]
+    const definition = fieldDefinitions[type?.toUpperCase()]
     if (definition?.constraints) {
       editableColumn.constraints = definition.constraints
     }
 
+    editableColumn.type = definition.type
+    editableColumn.subtype = definition.subtype
+
     // Default relationships many to many
     if (editableColumn.type === LINK_TYPE) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
-    }
-    if (editableColumn.type === FORMULA_TYPE) {
+    } else if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
     }
   }
@@ -288,35 +392,6 @@
     return match ? parseInt(match[1]) : 0
   }
 
-  function getRelationshipOptions(field) {
-    if (!field || !field.tableId) {
-      return null
-    }
-    const linkTable = tableOptions?.find(table => table._id === field.tableId)
-    if (!linkTable) {
-      return null
-    }
-    const thisName = truncate(table.name, { length: 14 }),
-      linkName = truncate(linkTable.name, { length: 14 })
-    return [
-      {
-        name: `Many ${thisName} rows → many ${linkName} rows`,
-        alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipType.MANY_TO_MANY,
-      },
-      {
-        name: `One ${linkName} row → many ${thisName} rows`,
-        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipType.ONE_TO_MANY,
-      },
-      {
-        name: `One ${thisName} row → many ${linkName} rows`,
-        alt: `One ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipType.MANY_TO_ONE,
-      },
-    ]
-  }
-
   function getAllowedTypes() {
     if (
       originalName &&
@@ -328,10 +403,29 @@
       ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
     ) {
       return ALLOWABLE_NUMBER_OPTIONS
-    } else if (!external) {
+    }
+
+    const isUsers =
+      editableColumn.type === FieldType.BB_REFERENCE &&
+      editableColumn.subtype === FieldSubtype.USERS
+
+    if (!external) {
       return [
-        ...Object.values(fieldDefinitions),
-        { name: "Auto Column", type: AUTO_TYPE },
+        FIELDS.STRING,
+        FIELDS.BARCODEQR,
+        FIELDS.LONGFORM,
+        FIELDS.OPTIONS,
+        FIELDS.ARRAY,
+        FIELDS.NUMBER,
+        FIELDS.BIGINT,
+        FIELDS.BOOLEAN,
+        FIELDS.DATETIME,
+        FIELDS.ATTACHMENT,
+        FIELDS.LINK,
+        FIELDS.FORMULA,
+        FIELDS.JSON,
+        isUsers ? FIELDS.USERS : FIELDS.USER,
+        FIELDS.AUTO,
       ]
     } else {
       let fields = [
@@ -344,6 +438,7 @@
         FIELDS.BOOLEAN,
         FIELDS.FORMULA,
         FIELDS.BIGINT,
+        isUsers ? FIELDS.USERS : FIELDS.USER,
       ]
       // no-sql or a spreadsheet
       if (!external || table.sql) {
@@ -357,8 +452,9 @@
     if (!fieldToCheck) {
       return
     }
+
     // most types need this, just make sure its always present
-    if (fieldToCheck && !fieldToCheck.constraints) {
+    if (!fieldToCheck.constraints) {
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
@@ -417,6 +513,13 @@
     return newError
   }
 
+  function isUsersColumn(column) {
+    return (
+      column.type === FieldType.BB_REFERENCE &&
+      [FieldSubtype.USER, FieldSubtype.USERS].includes(column.subtype)
+    )
+  }
+
   onMount(() => {
     mounted = true
   })
@@ -434,14 +537,14 @@
   {/if}
   <Select
     disabled={!typeEnabled}
-    bind:value={editableColumn.type}
-    on:change={handleTypeChange}
+    bind:value={editableColumn.fieldId}
+    on:change={onHandleTypeChange}
     options={allowedTypes}
     getOptionLabel={field => field.name}
-    getOptionValue={field => field.type}
+    getOptionValue={field => field.fieldId}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
-      if (option.type == AUTO_TYPE) {
+      if (option.type === AUTO_TYPE) {
         return availableAutoColumnKeys?.length > 0
       }
       return true
@@ -500,7 +603,7 @@
         <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
       </div>
     </div>
-    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
+    {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
       <div>
         <div class="row">
           <Label>Time zones</Label>
@@ -520,6 +623,7 @@
         />
       </div>
     {/if}
+    <Toggle bind:value={editableColumn.dateOnly} text="Date only" />
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
@@ -546,30 +650,17 @@
       </div>
     </div>
   {:else if editableColumn.type === "link"}
-    <Select
-      label="Table"
-      disabled={linkEditDisabled}
-      bind:value={editableColumn.tableId}
-      options={tableOptions}
-      getOptionLabel={table => table.name}
-      getOptionValue={table => table._id}
-    />
-    {#if relationshipOptions && relationshipOptions.length > 0}
-      <RadioGroup
-        disabled={linkEditDisabled}
-        label="Define the relationship"
-        bind:value={editableColumn.relationshipType}
-        options={relationshipOptions}
-        getOptionLabel={option => option.name}
-        getOptionValue={option => option.value}
-        getOptionTitle={option => option.alt}
-      />
-    {/if}
-    <Input
-      disabled={linkEditDisabled}
-      label={`Column name in other table`}
-      bind:value={editableColumn.fieldName}
-      error={errors.relatedName}
+    <RelationshipSelector
+      bind:relationshipPart1
+      bind:relationshipPart2
+      bind:relationshipTableIdPrimary
+      bind:relationshipTableIdSecondary
+      bind:editableColumn
+      {relationshipOpts1}
+      {relationshipOpts2}
+      {linkEditDisabled}
+      {tableOptions}
+      {errors}
     />
   {:else if editableColumn.type === FORMULA_TYPE}
     {#if !table.sql}
@@ -616,6 +707,20 @@
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
+  {:else if isUsersColumn(editableColumn) && datasource?.source !== SourceName.GOOGLE_SHEETS}
+    <Toggle
+      value={editableColumn.subtype === FieldSubtype.USERS}
+      on:change={e =>
+        handleTypeChange(
+          makeFieldId(
+            FieldType.BB_REFERENCE,
+            e.detail ? FieldSubtype.USERS : FieldSubtype.USER
+          )
+        )}
+      disabled={!isCreating}
+      thin
+      text="Allow multiple users"
+    />
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select

packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte

@@ -0,0 +1,54 @@
+<script>
+  import { ActionButton, Popover, Heading, Body, Button } from "@budibase/bbui"
+  import { store } from "builderStore"
+
+  let anchor
+  let open = false
+
+  const openSidePanel = () => {
+    store.update(state => ({
+      ...state,
+      builderSidePanel: true,
+    }))
+    open = false
+  }
+</script>
+
+<div bind:this={anchor}>
+  <ActionButton on:click={() => (open = true)} icon="Help" quiet>
+    Why can't I edit this table?
+  </ActionButton>
+</div>
+
+<Popover bind:open {anchor} align="left">
+  <div class="content">
+    <Heading size="XS">The app users table is read only</Heading>
+    <Body size="S">
+      You can continue to view the users that have access to your application.
+    </Body>
+    <Body size="S">
+      Manage and invite more application users using the user side panel in the
+      top right of your screen.
+    </Body>
+    <div class="button">
+      <Button cta on:click={openSidePanel}>Open users panel</Button>
+    </div>
+  </div>
+</Popover>
+
+<style>
+  .content {
+    width: 300px;
+    padding: var(--spacing-l);
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-l);
+  }
+  .content :global(.spectrum-Heading) {
+    font-weight: 400;
+  }
+  .button {
+    display: flex;
+    justify-content: flex-end;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -180,7 +180,7 @@
   <div class="hierarchy-items-container">
     <NavItem
       icon="UserGroup"
-      text="Users"
+      text="App users"
       selected={$isActive("./table/:tableId") &&
         $tables.selected?._id === TableNames.USERS}
       on:click={() => selectTable(TableNames.USERS)}

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -13,6 +13,8 @@
   import { Helpers } from "@budibase/bbui"
   import { RelationshipErrorChecker } from "./relationshipErrors"
   import { onMount } from "svelte"
+  import RelationshipSelector from "components/common/RelationshipSelector.svelte"
+  import { PrettyRelationshipDefinitions } from "constants/backend"
 
   export let save
   export let datasource
@@ -22,16 +24,21 @@
   export let selectedFromTable
   export let close
 
-  const relationshipTypes = [
-    {
-      label: "One to Many",
-      value: RelationshipType.MANY_TO_ONE,
+  let relationshipMap = {
+    [RelationshipType.MANY_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.MANY,
     },
-    {
-      label: "Many to Many",
-      value: RelationshipType.MANY_TO_MANY,
+    [RelationshipType.MANY_TO_ONE]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
     },
-  ]
+  }
+  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+
+  let relationshipPart1 = PrettyRelationshipDefinitions.MANY
+  let relationshipPart2 = PrettyRelationshipDefinitions.ONE
 
   let originalFromColumnName = toRelationship.name,
     originalToColumnName = fromRelationship.name
@@ -49,15 +56,34 @@
   )
   let errors = {}
   let fromPrimary, fromForeign, fromColumn, toColumn
-  let fromId, toId, throughId, throughToKey, throughFromKey
+
+  let throughId, throughToKey, throughFromKey
   let isManyToMany, isManyToOne, relationshipType
   let hasValidated = false
 
+  $: fromId = null
+  $: toId = null
+
   $: tableOptions = plusTables.map(table => ({
     label: table.name,
     value: table._id,
+    name: table.name,
+    _id: table._id,
   }))
-  $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
+
+  $: {
+    // Determine the relationship type based on the selected values of both parts
+    relationshipType = Object.entries(relationshipMap).find(
+      ([_, parts]) =>
+        parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2
+    )?.[0]
+
+    changed(() => {
+      hasValidated = false
+    })
+  }
+  $: valid =
+    getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
   $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
 
@@ -114,7 +140,7 @@
     return Object.entries(errors).filter(entry => !!entry[1]).length
   }
 
-  function allRequiredAttributesSet() {
+  function allRequiredAttributesSet(relationshipType) {
     const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
     if (relationshipType === RelationshipType.MANY_TO_ONE) {
       return base && fromPrimary && fromForeign
@@ -124,9 +150,10 @@
   }
 
   function validate() {
-    if (!allRequiredAttributesSet() && !hasValidated) {
+    if (!allRequiredAttributesSet(relationshipType) && !hasValidated) {
       return
     }
+
     hasValidated = true
     errorChecker.setType(relationshipType)
     const fromTable = getTable(fromId),
@@ -336,33 +363,34 @@
   onConfirm={saveRelationship}
   disabled={!valid}
 >
-  <Select
-    label="Relationship type"
-    options={relationshipTypes}
-    bind:value={relationshipType}
-    bind:error={errors.relationshipType}
-    on:change={() =>
-      changed(() => {
-        hasValidated = false
-      })}
-  />
   <div class="headings">
     <Detail>Tables</Detail>
   </div>
-  {#if !selectedFromTable}
-    <Select
-      label="Select from table"
-      options={tableOptions}
-      bind:value={fromId}
-      bind:error={errors.fromTable}
-      on:change={e =>
-        changed(() => {
-          const table = plusTables.find(tbl => tbl._id === e.detail)
-          fromColumn = table?.name || ""
-          fromPrimary = table?.primary?.[0]
-        })}
-    />
-  {/if}
+
+  <RelationshipSelector
+    bind:relationshipPart1
+    bind:relationshipPart2
+    bind:relationshipTableIdPrimary={fromId}
+    bind:relationshipTableIdSecondary={toId}
+    {relationshipOpts1}
+    {relationshipOpts2}
+    {tableOptions}
+    {errors}
+    primaryDisabled={selectedFromTable}
+    primaryTableChanged={e =>
+      changed(() => {
+        const table = plusTables.find(tbl => tbl._id === e.detail)
+        fromColumn = table?.name || ""
+        fromPrimary = table?.primary?.[0]
+      })}
+    secondaryTableChanged={e =>
+      changed(() => {
+        const table = plusTables.find(tbl => tbl._id === e.detail)
+        toColumn = table.name || ""
+        fromForeign = null
+      })}
+  />
+
   {#if isManyToOne && fromId}
     <Select
       label={`Primary Key (${getTable(fromId).name})`}
@@ -372,18 +400,6 @@
       on:change={changed}
     />
   {/if}
-  <Select
-    label={"Select to table"}
-    options={tableOptions}
-    bind:value={toId}
-    bind:error={errors.toTable}
-    on:change={e =>
-      changed(() => {
-        const table = plusTables.find(tbl => tbl._id === e.detail)
-        toColumn = table.name || ""
-        fromForeign = null
-      })}
-  />
   {#if isManyToMany}
     <Select
       label={"Through"}

packages/builder/src/components/backend/Datasources/TableImportSelection/index.svelte

@@ -57,7 +57,7 @@
       {#if $store.error}
         <InlineAlert
           type="error"
-          header={$store.error.title}
+          header="Error fetching {tableType}"
           message={$store.error.description}
         />
       {/if}

packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js

@@ -1,6 +1,6 @@
 import { derived, writable, get } from "svelte/store"
 import { keepOpen, notifications } from "@budibase/bbui"
-import { datasources, ImportTableError, tables } from "stores/backend"
+import { datasources, tables } from "stores/backend"
 
 export const createTableSelectionStore = (integration, datasource) => {
   const tableNamesStore = writable([])
@@ -30,12 +30,7 @@ export const createTableSelectionStore = (integration, datasource) => {
       notifications.success(`Tables fetched successfully.`)
       await onComplete()
     } catch (err) {
-      if (err instanceof ImportTableError) {
-        errorStore.set(err)
-      } else {
-        notifications.error("Error fetching tables.")
-      }
-
+      errorStore.set(err)
       return keepOpen
     }
   }

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -49,6 +49,15 @@
       label: "Long Form Text",
       value: FIELDS.LONGFORM.type,
     },
+
+    {
+      label: "User",
+      value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`,
+    },
+    {
+      label: "Users",
+      value: `${FIELDS.USERS.type}${FIELDS.USERS.subtype}`,
+    },
   ]
 
   $: {
@@ -143,7 +152,7 @@
       <div class="field">
         <span>{name}</span>
         <Select
-          value={schema[name]?.type}
+          value={`${schema[name]?.type}${schema[name]?.subtype || ""}`}
           options={typeOptions}
           placeholder={null}
           getOptionLabel={option => option.label}

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -3,6 +3,7 @@
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
+  import { canBeDisplayColumn } from "@budibase/shared-core"
 
   export let rows = []
   export let schema = {}
@@ -10,36 +11,82 @@
   export let displayColumn = null
   export let promptUpload = false
 
-  const typeOptions = [
-    {
+  const typeOptions = {
+    [FIELDS.STRING.type]: {
       label: "Text",
       value: FIELDS.STRING.type,
+      config: {
+        type: FIELDS.STRING.type,
+        constraints: FIELDS.STRING.constraints,
+      },
     },
-    {
+    [FIELDS.NUMBER.type]: {
       label: "Number",
       value: FIELDS.NUMBER.type,
+      config: {
+        type: FIELDS.NUMBER.type,
+        constraints: FIELDS.NUMBER.constraints,
+      },
     },
-    {
+    [FIELDS.DATETIME.type]: {
       label: "Date",
       value: FIELDS.DATETIME.type,
+      config: {
+        type: FIELDS.DATETIME.type,
+        constraints: FIELDS.DATETIME.constraints,
+      },
     },
-    {
+    [FIELDS.OPTIONS.type]: {
       label: "Options",
       value: FIELDS.OPTIONS.type,
+      config: {
+        type: FIELDS.OPTIONS.type,
+        constraints: FIELDS.OPTIONS.constraints,
+      },
     },
-    {
+    [FIELDS.ARRAY.type]: {
       label: "Multi-select",
       value: FIELDS.ARRAY.type,
+      config: {
+        type: FIELDS.ARRAY.type,
+        constraints: FIELDS.ARRAY.constraints,
+      },
     },
-    {
+    [FIELDS.BARCODEQR.type]: {
       label: "Barcode/QR",
       value: FIELDS.BARCODEQR.type,
+      config: {
+        type: FIELDS.BARCODEQR.type,
+        constraints: FIELDS.BARCODEQR.constraints,
+      },
     },
-    {
+    [FIELDS.LONGFORM.type]: {
       label: "Long Form Text",
       value: FIELDS.LONGFORM.type,
+      config: {
+        type: FIELDS.LONGFORM.type,
+        constraints: FIELDS.LONGFORM.constraints,
+      },
     },
-  ]
+    user: {
+      label: "User",
+      value: "user",
+      config: {
+        type: FIELDS.USER.type,
+        subtype: FIELDS.USER.subtype,
+        constraints: FIELDS.USER.constraints,
+      },
+    },
+    users: {
+      label: "Users",
+      value: "users",
+      config: {
+        type: FIELDS.USERS.type,
+        subtype: FIELDS.USERS.subtype,
+        constraints: FIELDS.USERS.constraints,
+      },
+    },
+  }
 
   let fileInput
   let error = null
@@ -48,10 +95,16 @@
   let validation = {}
   let validateHash = ""
   let errors = {}
+  let selectedColumnTypes = {}
 
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
-    return validation[column]
+    return validation[column] && canBeDisplayColumn(schema[column].type)
   })
+
+  $: if (displayColumn && !canBeDisplayColumn(schema[displayColumn].type)) {
+    displayColumn = null
+  }
+
   $: {
     // binding in consumer is causing double renders here
     const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
@@ -72,6 +125,13 @@
       rows = response.rows
       schema = response.schema
       fileName = response.fileName
+      selectedColumnTypes = Object.entries(response.schema).reduce(
+        (acc, [colName, fieldConfig]) => ({
+          ...acc,
+          [colName]: fieldConfig.type,
+        }),
+        {}
+      )
     } catch (e) {
       loading = false
       error = e
@@ -98,8 +158,10 @@
   }
 
   const handleChange = (name, e) => {
-    schema[name].type = e.detail
-    schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
+    const { config } = typeOptions[e.detail]
+    schema[name].type = config.type
+    schema[name].subtype = config.subtype
+    schema[name].constraints = config.constraints
   }
 
   const openFileUpload = (promptUpload, fileInput) => {
@@ -142,9 +204,9 @@
       <div class="field">
         <span>{column.name}</span>
         <Select
-          bind:value={column.type}
+          bind:value={selectedColumnTypes[column.name]}
           on:change={e => handleChange(name, e)}
-          options={typeOptions}
+          options={Object.values(typeOptions)}
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -3,16 +3,19 @@
   import { API } from "api"
   import { Select, Label, Multiselect } from "@budibase/bbui"
   import { capitalise } from "../../helpers"
+  import { createEventDispatcher } from "svelte"
 
   export let schema
   export let linkedRows = []
 
+  const dispatch = createEventDispatcher()
+
   let rows = []
-  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+  let linkedIds = []
+
+  $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
   )
-
-  $: linkedRows = linkedIds
   $: label = capitalise(schema.name)
   $: linkedTableId = schema.tableId
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
@@ -44,7 +47,10 @@
     options={rows}
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
-    on:change={e => (linkedIds = e.detail ? [e.detail] : [])}
+    on:change={e => {
+      linkedIds = e.detail ? [e.detail] : []
+      dispatch("change", linkedIds)
+    }}
     {label}
     sort
   />
@@ -56,5 +62,6 @@
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
     sort
+    on:change={() => dispatch("change", linkedIds)}
   />
 {/if}

packages/builder/src/components/common/NavItem.svelte

@@ -102,7 +102,7 @@
       </div>
     {/if}
     <div class="text" title={showTooltip ? text : null}>
-      {text}
+      <span title={text}>{text}</span>
       {#if selectedBy}
         <UserAvatars size="XS" users={selectedBy} />
       {/if}
@@ -227,9 +227,6 @@
   .text {
     font-weight: 600;
     font-size: 12px;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
     flex: 1 1 auto;
     color: var(--spectrum-global-color-gray-900);
     order: 2;
@@ -238,6 +235,11 @@
     align-items: center;
     gap: 8px;
   }
+  .text span {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
   .scrollable .text {
     flex: 0 0 auto;
     max-width: 160px;

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -0,0 +1,84 @@
+<script>
+  import { Select, Input } from "@budibase/bbui"
+
+  export let relationshipPart1
+  export let relationshipPart2
+  export let relationshipTableIdPrimary
+  export let relationshipTableIdSecondary
+  export let editableColumn
+  export let linkEditDisabled = false
+  export let tableOptions
+  export let errors
+  export let relationshipOpts1
+  export let relationshipOpts2
+  export let primaryTableChanged
+  export let secondaryTableChanged
+  export let primaryDisabled = true
+</script>
+
+<div class="relationship-container">
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipPart1}
+      options={relationshipOpts1}
+      bind:error={errors.relationshipType}
+    />
+  </div>
+  <div class="relationship-label">in</div>
+  <div class="relationship-part">
+    <Select
+      disabled={primaryDisabled}
+      options={tableOptions}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+      bind:value={relationshipTableIdPrimary}
+      on:change={primaryTableChanged}
+      bind:error={errors.fromTable}
+    />
+  </div>
+</div>
+<div class="relationship-container">
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipPart2}
+      options={relationshipOpts2}
+      getOptionLabel={option => "To " + option.toLowerCase()}
+    />
+  </div>
+  <div class="relationship-label">in</div>
+  <div class="relationship-part">
+    <Select
+      disabled={linkEditDisabled}
+      bind:value={relationshipTableIdSecondary}
+      bind:error={errors.toTable}
+      options={tableOptions.filter(
+        table => table._id !== relationshipTableIdPrimary
+      )}
+      getOptionLabel={table => table.name}
+      getOptionValue={table => table._id}
+      on:change={secondaryTableChanged}
+    />
+  </div>
+</div>
+{#if editableColumn}
+  <Input
+    disabled={linkEditDisabled}
+    label={`Column name in other table`}
+    bind:value={editableColumn.fieldName}
+    error={errors.relatedName}
+  />
+{/if}
+
+<style>
+  .relationship-container {
+    display: flex;
+    align-items: center;
+    gap: 20px;
+  }
+
+  .relationship-part {
+    flex-basis: 60%;
+  }
+</style>

packages/builder/src/components/common/UndoRedoControl.svelte

@@ -38,14 +38,12 @@
     hoverable
     on:click={store.undo}
     disabled={!$store.canUndo}
-    tooltip="Undo latest change"
   />
   <Icon
     name="Redo"
     hoverable
     on:click={store.redo}
     disabled={!$store.canRedo}
-    tooltip="Redo latest undo"
   />
 </div>
 

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -0,0 +1,250 @@
+<script>
+  import { Icon, Input, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { createEventDispatcher, setContext } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let panel = ClientBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let fillWidth
+  export let allowJS = true
+  export let allowHelpers = true
+  export let updateOnChange = true
+  export let drawerLeft
+  export let type
+  export let schema
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  let valid = true
+  let currentVal = value
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    onBlur()
+    bindingDrawer.hide()
+  }
+
+  setContext("binding-drawer-actions", {
+    save: saveBinding,
+  })
+
+  const onChange = value => {
+    if (type === "link" && value && hasValidLinks(value)) {
+      currentVal = value.split(",")
+    } else if (type === "array" && value && hasValidOptions(value)) {
+      currentVal = value.split(",")
+    } else {
+      currentVal = readableToRuntimeBinding(bindings, value)
+    }
+    dispatch("change", currentVal)
+  }
+
+  const onBlur = () => {
+    dispatch("blur", currentVal)
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link => schema?.constraints?.inclusion?.includes(link))
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    link: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    json: value => !isJSBinding(value),
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[type]
+    return validate ? validate(value) : true
+  }
+
+  const getIconClass = (value, type) => {
+    if (type === "longform" && !isJSBinding(value)) {
+      return "text-area-slot-icon"
+    }
+    if (type === "json" && !isJSBinding(value)) {
+      return "json-slot-icon"
+    }
+    if (type !== "string" && type !== "number") {
+      return "slot-icon"
+    }
+    return ""
+  }
+</script>
+
+<div class="control" class:disabled>
+  {#if !isValid(value)}
+    <Input
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      on:change={event => onChange(event.detail)}
+      on:blur={onBlur}
+      {placeholder}
+      {updateOnChange}
+    />
+    <div
+      class="icon"
+      on:click={() => {
+        if (!isJS) {
+          dispatch("change", "")
+        }
+      }}
+    >
+      <Icon disabled={isJS} size="S" name="Close" />
+    </div>
+  {:else}
+    <slot
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      {placeholder}
+      {updateOnChange}
+    />
+  {/if}
+  {#if !disabled && type !== "formula"}
+    <div
+      class={`icon ${getIconClass(value, type)}`}
+      on:click={() => {
+        bindingDrawer.show()
+      }}
+    >
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer
+  on:drawerHide
+  on:drawerShow
+  {fillWidth}
+  bind:this={bindingDrawer}
+  {title}
+  left={drawerLeft}
+  headless
+>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+    Save
+  </Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    bind:valid
+    value={readableValue}
+    on:change={event => (tempValue = event.detail)}
+    {bindings}
+    {allowJS}
+    {allowHelpers}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .slot-icon {
+    right: 31px !important;
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: 0px !important;
+    border-bottom-right-radius: 0px !important;
+  }
+
+  .text-area-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 26px !important;
+  }
+  .json-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 23px !important;
+    right: 0px !important;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
+  .control:not(.disabled) :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
+</style>

packages/builder/src/components/deploy/AppActions.svelte

@@ -224,10 +224,10 @@
               <span
                 class="app-link"
                 on:click={() => {
+                  appActionPopover.hide()
                   if (isPublished) {
                     viewApp()
                   } else {
-                    appActionPopover.hide()
                     updateAppModal.show()
                   }
                 }}

packages/builder/src/components/design/Panel.svelte

@@ -12,11 +12,19 @@
   export let borderLeft = false
   export let borderRight = false
   export let wide = false
+  export let extraWide = false
+  export let closeButtonIcon = "Close"
 
   $: customHeaderContent = $$slots["panel-header-content"]
 </script>
 
-<div class="panel" class:wide class:borderLeft class:borderRight>
+<div
+  class="panel"
+  class:wide
+  class:extraWide
+  class:borderLeft
+  class:borderRight
+>
   <div class="header" class:custom={customHeaderContent}>
     {#if showBackButton}
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
@@ -33,7 +41,7 @@
       </div>
     {/if}
     {#if showCloseButton}
-      <Icon name="Close" hoverable on:click={onClickCloseButton} />
+      <Icon name={closeButtonIcon} hoverable on:click={onClickCloseButton} />
     {/if}
   </div>
 
@@ -70,6 +78,10 @@
     width: 310px;
     flex: 0 0 310px;
   }
+  .panel.extraWide {
+    width: 450px;
+    flex: 0 0 450px;
+  }
   .header {
     flex: 0 0 48px;
     display: flex;

packages/builder/src/components/design/settings/componentSettings.js

@@ -65,6 +65,7 @@ const componentMap = {
   "field/array": FormFieldSelect,
   "field/json": FormFieldSelect,
   "field/barcodeqr": FormFieldSelect,
+  "field/bb_reference": FormFieldSelect,
   // Some validation types are the same as others, so not all types are
   // explicitly listed here. e.g. options uses string validation
   "validation/string": ValidationEditor,
@@ -74,6 +75,7 @@ const componentMap = {
   "validation/datetime": ValidationEditor,
   "validation/attachment": ValidationEditor,
   "validation/link": ValidationEditor,
+  "validation/bb_reference": ValidationEditor,
 }
 
 export const getComponentForSetting = setting => {

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte

@@ -3,7 +3,6 @@
   import { createEventDispatcher } from "svelte"
   import { notifications } from "@budibase/bbui"
   import ButtonActionDrawer from "./ButtonActionDrawer.svelte"
-  import { automationStore } from "builderStore"
   import { cloneDeep } from "lodash/fp"
 
   const dispatch = createEventDispatcher()
@@ -24,47 +23,11 @@
   }
 
   const saveEventData = async () => {
-    // any automations that need created from event triggers
-    const automationsToCreate = tmpValue.filter(
-      action => action["##eventHandlerType"] === "Trigger Automation"
-    )
-    for (let action of automationsToCreate) {
-      await createAutomation(action.parameters)
-    }
-
     dispatch("change", tmpValue)
     notifications.success("Component actions saved.")
     drawer.hide()
   }
 
-  // called by the parent modal when actions are saved
-  const createAutomation = async parameters => {
-    if (parameters.automationId || !parameters.newAutomationName) {
-      return
-    }
-    try {
-      let trigger = automationStore.actions.constructBlock(
-        "TRIGGER",
-        "APP",
-        $automationStore.blockDefinitions.TRIGGER.APP
-      )
-      trigger.inputs = {
-        fields: Object.keys(parameters.fields ?? {}).reduce((fields, key) => {
-          fields[key] = "string"
-          return fields
-        }, {}),
-      }
-      const automation = await automationStore.actions.create(
-        parameters.newAutomationName,
-        trigger
-      )
-      parameters.automationId = automation._id
-      delete parameters.newAutomationName
-    } catch (error) {
-      notifications.error("Error creating automation")
-    }
-  }
-
   $: actionCount = value?.length
   $: actionText = `${actionCount || "No"} action${
     actionCount !== 1 ? "s" : ""

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ScrollTo.svelte

@@ -1,6 +1,5 @@
 <script>
   import { currentAsset, store } from "builderStore"
-  import { onMount } from "svelte"
   import { Label, Combobox, Select } from "@budibase/bbui"
   import {
     getActionProviderComponents,
@@ -10,12 +9,6 @@
 
   export let parameters
 
-  onMount(() => {
-    if (!parameters.type) {
-      parameters.type = "top"
-    }
-  })
-
   $: formComponent = findComponent($currentAsset.props, parameters.componentId)
   $: formSchema = buildFormSchema(formComponent)
   $: fieldOptions = Object.keys(formSchema || {})

packages/builder/src/components/design/settings/controls/FieldConfiguration.svelte

@@ -1,91 +0,0 @@
-<script>
-  import { Button, ActionButton, Drawer } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
-  import ColumnDrawer from "./ColumnEditor/ColumnDrawer.svelte"
-  import { cloneDeep } from "lodash/fp"
-  import {
-    getDatasourceForProvider,
-    getSchemaForDatasource,
-  } from "builderStore/dataBinding"
-  import { currentAsset } from "builderStore"
-  import { getFields } from "helpers/searchFields"
-
-  export let componentInstance
-  export let value = []
-  export let allowCellEditing = true
-  export let subject = "Table"
-
-  const dispatch = createEventDispatcher()
-
-  let drawer
-  let boundValue
-
-  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchema($currentAsset, datasource)
-  $: options = allowCellEditing
-    ? Object.keys(schema || {})
-    : enrichedSchemaFields?.map(field => field.name)
-  $: sanitisedValue = getValidColumns(value, options)
-  $: updateBoundValue(sanitisedValue)
-  $: enrichedSchemaFields = getFields(Object.values(schema || {}), {
-    allowLinks: true,
-  })
-
-  const getSchema = (asset, datasource) => {
-    const schema = getSchemaForDatasource(asset, datasource).schema
-
-    // Don't show ID and rev in tables
-    if (schema) {
-      delete schema._id
-      delete schema._rev
-    }
-
-    return schema
-  }
-
-  const updateBoundValue = value => {
-    boundValue = cloneDeep(value)
-  }
-
-  const getValidColumns = (columns, options) => {
-    if (!Array.isArray(columns) || !columns.length) {
-      return []
-    }
-    // We need to account for legacy configs which would just be an array
-    // of strings
-    if (typeof columns[0] === "string") {
-      columns = columns.map(col => ({
-        name: col,
-        displayName: col,
-      }))
-    }
-    return columns.filter(column => {
-      return options.includes(column.name)
-    })
-  }
-
-  const open = () => {
-    updateBoundValue(sanitisedValue)
-    drawer.show()
-  }
-
-  const save = () => {
-    dispatch("change", getValidColumns(boundValue, options))
-    drawer.hide()
-  }
-</script>
-
-<ActionButton on:click={open}>Configure columns</ActionButton>
-<Drawer bind:this={drawer} title="{subject} Columns">
-  <svelte:fragment slot="description">
-    Configure the columns in your {subject.toLowerCase()}.
-  </svelte:fragment>
-  <Button cta slot="buttons" on:click={save}>Save</Button>
-  <ColumnDrawer
-    slot="body"
-    bind:columns={boundValue}
-    {options}
-    {schema}
-    {allowCellEditing}
-  />
-</Drawer>

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte

@@ -37,7 +37,7 @@
   }
 
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: resourceId = datasource.resourceId || datasource.tableId
+  $: resourceId = datasource?.resourceId || datasource?.tableId
 
   $: if (!isEqual(value, cachedValue)) {
     cachedValue = cloneDeep(value)

packages/builder/src/components/design/settings/controls/FieldConfiguration/utils.js

@@ -43,4 +43,5 @@ export const FieldTypeToComponentMap = {
   link: "relationshipfield",
   json: "jsonfield",
   barcodeqr: "codescanner",
+  bb_reference: "bbreferencefield",
 }

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -3,21 +3,23 @@
     Body,
     Button,
     Combobox,
-    Multiselect,
     DatePicker,
     DrawerContent,
     Icon,
     Input,
-    Layout,
-    Select,
     Label,
+    Layout,
+    Multiselect,
+    Select,
   } from "@budibase/bbui"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { generate } from "shortid"
-  import { LuceneUtils, Constants } from "@budibase/frontend-core"
+  import { Constants, LuceneUtils } from "@budibase/frontend-core"
   import { getFields } from "helpers/searchFields"
+  import { FieldType } from "@budibase/types"
   import { createEventDispatcher, onMount } from "svelte"
+  import FilterUsers from "./FilterUsers.svelte"
 
   export let schemaFields
   export let filters = []
@@ -29,7 +31,6 @@
 
   const dispatch = createEventDispatcher()
   const { OperatorOptions } = Constants
-  const { getValidOperatorsForType } = LuceneUtils
   const KeyedFieldRegex = /\d[0-9]*:/g
   const behaviourOptions = [
     { value: "and", label: "Match all filters" },
@@ -120,22 +121,19 @@
     return enrichedSchemaFields.find(field => field.name === filter.field)
   }
 
-  const santizeTypes = filter => {
+  const sanitizeTypes = filter => {
     // Update type based on field
     const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
     filter.type = fieldSchema?.type
+    filter.subtype = fieldSchema?.subtype
 
     // Update external type based on field
     filter.externalType = getSchema(filter)?.externalType
   }
 
-  const santizeOperator = filter => {
+  const sanitizeOperator = filter => {
     // Ensure a valid operator is selected
-    const operators = getValidOperatorsForType(
-      filter.type,
-      filter.field,
-      datasource
-    ).map(x => x.value)
+    const operators = getValidOperatorsForType(filter).map(x => x.value)
     if (!operators.includes(filter.operator)) {
       filter.operator = operators[0] ?? OperatorOptions.Equals.value
     }
@@ -148,7 +146,7 @@
     filter.noValue = noValueOptions.includes(filter.operator)
   }
 
-  const santizeValue = filter => {
+  const sanitizeValue = (filter, previousType) => {
     // Check if the operator allows a value at all
     if (filter.noValue) {
       filter.value = null
@@ -162,28 +160,47 @@
       }
     } else if (filter.type === "array" && filter.valueType === "Value") {
       filter.value = []
+    } else if (
+      previousType !== filter.type &&
+      (previousType === FieldType.BB_REFERENCE ||
+        filter.type === FieldType.BB_REFERENCE)
+    ) {
+      filter.value = filter.type === "array" ? [] : null
     }
   }
 
   const onFieldChange = filter => {
-    santizeTypes(filter)
-    santizeOperator(filter)
-    santizeValue(filter)
+    const previousType = filter.type
+    sanitizeTypes(filter)
+    sanitizeOperator(filter)
+    sanitizeValue(filter, previousType)
   }
 
   const onOperatorChange = filter => {
-    santizeOperator(filter)
-    santizeValue(filter)
+    sanitizeOperator(filter)
+    sanitizeValue(filter, filter.type)
   }
 
   const onValueTypeChange = filter => {
-    santizeValue(filter)
+    sanitizeValue(filter)
   }
 
   const getFieldOptions = field => {
     const schema = enrichedSchemaFields.find(x => x.name === field)
     return schema?.constraints?.inclusion || []
   }
+
+  const getValidOperatorsForType = filter => {
+    if (!filter?.field) {
+      return []
+    }
+
+    return LuceneUtils.getValidOperatorsForType(
+      { type: filter.type, subtype: filter.subtype },
+      filter.field,
+      datasource
+    )
+  }
 </script>
 
 <DrawerContent>
@@ -228,11 +245,7 @@
               />
               <Select
                 disabled={!filter.field}
-                options={getValidOperatorsForType(
-                  filter.type,
-                  filter.field,
-                  datasource
-                )}
+                options={getValidOperatorsForType(filter)}
                 bind:value={filter.operator}
                 on:change={() => onOperatorChange(filter)}
                 placeholder={null}
@@ -285,6 +298,15 @@
                   timeOnly={getSchema(filter)?.timeOnly}
                   bind:value={filter.value}
                 />
+              {:else if filter.type === FieldType.BB_REFERENCE}
+                <FilterUsers
+                  bind:value={filter.value}
+                  multiselect={[
+                    OperatorOptions.In.value,
+                    OperatorOptions.ContainsAny.value,
+                  ].includes(filter.operator)}
+                  disabled={filter.noValue}
+                />
               {:else}
                 <DrawerBindableInput disabled />
               {/if}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte

@@ -0,0 +1,34 @@
+<script>
+  import { Select, Multiselect } from "@budibase/bbui"
+  import { fetchData } from "@budibase/frontend-core"
+
+  import { API } from "api"
+
+  export let value = null
+  export let disabled
+  export let multiselect = false
+
+  $: fetch = fetchData({
+    API,
+    datasource: {
+      type: "user",
+    },
+    options: {
+      limit: 100,
+    },
+  })
+
+  $: options = $fetch.rows
+
+  $: component = multiselect ? Multiselect : Select
+</script>
+
+<svelte:component
+  this={component}
+  bind:value
+  autocomplete
+  {options}
+  getOptionLabel={option => option.email}
+  getOptionValue={option => option._id}
+  {disabled}
+/>

packages/builder/src/components/integration/ExtraQueryConfig.svelte

@@ -18,31 +18,20 @@
 </script>
 
 {#each extraFields as { key, displayName, type }}
-  <div class="config-field">
-    <Label>{displayName}</Label>
-    {#if type === "string"}
-      <Input
-        on:change={() => populateExtraQuery(extraQueryFields)}
-        bind:value={extraQueryFields[key]}
-      />
-    {/if}
+  <Label>{displayName}</Label>
+  {#if type === "string"}
+    <Input
+      on:change={() => populateExtraQuery(extraQueryFields)}
+      bind:value={extraQueryFields[key]}
+    />
+  {/if}
 
-    {#if type === "list"}
-      <Select
-        on:change={() => populateExtraQuery(extraQueryFields)}
-        bind:value={extraQueryFields[key]}
-        options={config[key].data[query.queryVerb]}
-        getOptionLabel={current => current}
-      />
-    {/if}
-  </div>
+  {#if type === "list"}
+    <Select
+      on:change={() => populateExtraQuery(extraQueryFields)}
+      bind:value={extraQueryFields[key]}
+      options={config[key].data[query.queryVerb]}
+      getOptionLabel={current => current}
+    />
+  {/if}
 {/each}
-
-<style>
-  .config-field {
-    display: grid;
-    grid-template-columns: 20% 1fr;
-    grid-gap: var(--spacing-l);
-    align-items: center;
-  }
-</style>

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -34,6 +34,7 @@
   export let bindings = []
   export let bindingDrawerLeft
   export let allowHelpers = true
+  export let customButtonText = null
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
     name,
@@ -158,9 +159,13 @@
 {/if}
 {#if !readOnly && !noAddButton}
   <div>
-    <ActionButton icon="Add" secondary thin outline on:click={addEntry}
-      >Add{name ? ` ${lowercase(name)}` : ""}</ActionButton
-    >
+    <ActionButton icon="Add" secondary thin outline on:click={addEntry}>
+      {#if customButtonText}
+        {customButtonText}
+      {:else}
+        {`Add${name ? ` ${lowercase(name)}` : ""}`}
+      {/if}
+    </ActionButton>
   </div>
 {/if}
 

packages/builder/src/components/integration/QueryViewer.svelte

@@ -1,364 +1,444 @@
 <script>
-  import { goto, beforeUrlChange } from "@roxi/routify"
+  import { goto } from "@roxi/routify"
+  import { datasources, integrations, queries } from "stores/backend"
   import {
     Icon,
     Select,
-    Button,
-    ButtonGroup,
-    Body,
-    Label,
-    Layout,
     Input,
-    Heading,
-    Tabs,
-    Tab,
-    Modal,
-    ModalContent,
+    Label,
     notifications,
+    Heading,
+    Body,
     Divider,
+    Button,
   } from "@budibase/bbui"
-  import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
-  import IntegrationQueryEditor from "components/integration/index.svelte"
-  import ExternalDataSourceTable from "components/backend/DataTable/ExternalDataSourceTable.svelte"
-  import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
-  import { datasources, integrations, queries } from "stores/backend"
-  import { capitalise } from "../../helpers"
-  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
-  import JSONPreview from "./JSONPreview.svelte"
-  import { SchemaTypeOptions } from "constants/backend"
-  import KeyValueBuilder from "./KeyValueBuilder.svelte"
-  import { fieldsToSchema, schemaToFields } from "helpers/data/utils"
+  import { capitalise } from "helpers"
   import AccessLevelSelect from "./AccessLevelSelect.svelte"
+  import IntegrationQueryEditor from "components/integration/index.svelte"
+  import QueryViewerSidePanel from "./QueryViewerSidePanel/index.svelte"
+  import { cloneDeep } from "lodash/fp"
+  import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
   import { ValidQueryNameRegex } from "@budibase/shared-core"
+  import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
+  import QueryViewerSavePromptModal from "./QueryViewerSavePromptModal.svelte"
+  import { Utils } from "@budibase/frontend-core"
 
   export let query
+  let queryHash
 
-  const resumeNavigation = () => {
-    if (typeof navigateTo == "string") {
-      $goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
-    }
+  let loading = false
+  let modified = false
+  let scrolling = false
+  let showSidePanel = false
+  let nameError
+
+  let newQuery
+
+  let datasource
+  let integration
+  let schemaType
+
+  let autoSchema = {}
+  let rows = []
+
+  const parseQuery = query => {
+    modified = false
+
+    datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
+    integration = $integrations[datasource.source]
+    schemaType = integration.query[query.queryVerb].type
+
+    newQuery = cloneDeep(query)
+    // Set the location where the query code will be written to an empty string so that it doesn't
+    // get changed from undefined -> "" by the input, breaking our unsaved changes checks
+    newQuery.fields[schemaType] ??= ""
+
+    queryHash = JSON.stringify(newQuery)
   }
 
-  const transformerDocs = "https://docs.budibase.com/docs/transformers"
+  $: parseQuery(query)
 
-  let fields = query?.schema ? schemaToFields(query.schema) : []
-  let parameters
-  let data = []
-  let saveId
-  let currentTab = "JSON"
-  let saveModal
-  let override = false
-  let navigateTo = null
-  let nameError = null
+  const checkIsModified = newQuery => {
+    const newQueryHash = JSON.stringify(newQuery)
+    modified = newQueryHash !== queryHash
 
-  // seed the transformer
-  if (query && !query.transformer) {
-    query.transformer = "return data"
+    return modified
   }
 
-  // initialise a new empty schema
-  if (query && !query.schema) {
-    query.schema = {}
-  }
+  const debouncedCheckIsModified = Utils.debounce(checkIsModified, 1000)
 
-  let queryStr = JSON.stringify(query)
+  $: debouncedCheckIsModified(newQuery)
 
-  $beforeUrlChange(event => {
-    const updated = JSON.stringify(query)
-
-    if (updated !== queryStr && !override) {
-      navigateTo = event.type == "pushstate" ? event.url : null
-      saveModal.show()
-      return false
-    } else return true
-  })
-
-  $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
-  $: query.schema = fieldsToSchema(fields)
-  $: datasourceType = datasource?.source
-  $: integrationInfo = datasourceType ? $integrations[datasourceType] : null
-  $: queryConfig = integrationInfo?.query
-  $: shouldShowQueryConfig = queryConfig && query.queryVerb
-  $: readQuery = query.queryVerb === "read" || query.readable
-  $: queryInvalid = !query.name || nameError || (readQuery && data.length === 0)
-
-  //Cast field in query preview response to number if specified by schema
-  $: {
-    for (let i = 0; i < data.length; i++) {
-      let row = data[i]
-      for (let fieldName of Object.keys(fields)) {
-        if (fields[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
-          row[fieldName] = Number(row[fieldName])
-        } else {
-          row[fieldName] = row[fieldName]?.toString()
-        }
-      }
-    }
-  }
-
-  function resetDependentFields() {
-    if (query.fields.extra) {
-      query.fields.extra = {}
-    }
-  }
-
-  function populateExtraQuery(extraQueryFields) {
-    query.fields.extra = extraQueryFields
-  }
-
-  async function previewQuery() {
+  async function runQuery({ suppressErrors = true }) {
     try {
-      const response = await queries.preview(query)
+      showSidePanel = true
+      loading = true
+      const response = await queries.preview(newQuery)
       if (response.rows.length === 0) {
         notifications.info(
           "Query results empty. Please execute a query with results to create your schema."
         )
         return
       }
-      data = response.rows
-      // need to merge fields that already exist/might have changed
-      if (fields) {
-        for (let key of Object.keys(response.schema)) {
-          if (fields[key]) {
-            response.schema[key] = fields[key]
-          }
-        }
+
+      if (Object.keys(newQuery.schema).length === 0) {
+        // Assign this to a variable instead of directly to the newQuery.schema so that a user
+        // can change the table they're querying and have the schema update until they first
+        // edit it
+        autoSchema = response.schema
       }
-      fields = response.schema
-      currentTab = "JSON"
+
+      rows = response.rows
+
       notifications.success("Query executed successfully")
     } catch (error) {
       notifications.error(`Query Error: ${error.message}`)
+
+      if (!suppressErrors) {
+        throw error
+      }
+    } finally {
+      loading = false
     }
   }
 
-  // return the query.
   async function saveQuery() {
     try {
-      const response = await queries.save(query.datasourceId, query)
-      saveId = response._id
-
-      if (response?._rev) {
-        queryStr = JSON.stringify(query)
-      }
+      showSidePanel = true
+      loading = true
+      const response = await queries.save(newQuery.datasourceId, {
+        ...newQuery,
+        schema:
+          Object.keys(newQuery.schema).length === 0
+            ? autoSchema
+            : newQuery.schema,
+      })
 
       notifications.success("Query saved successfully")
       return response
     } catch (error) {
       notifications.error(error.message || "Error saving query")
+    } finally {
+      loading = false
     }
   }
+
+  function resetDependentFields() {
+    if (newQuery.fields.extra) {
+      newQuery.fields.extra = {}
+    }
+  }
+
+  function populateExtraQuery(extraQueryFields) {
+    newQuery.fields.extra = extraQueryFields
+  }
+
+  const handleScroll = e => {
+    scrolling = e.target.scrollTop !== 0
+  }
 </script>
 
-<Modal
-  bind:this={saveModal}
-  on:hide={() => {
-    navigateTo = null
-  }}
->
-  <ModalContent
-    title="You have unsaved changes"
-    confirmText="Save and Continue"
-    cancelText="Discard Changes"
-    size="L"
-    onConfirm={async () => {
-      await saveQuery()
-      override = true
-      resumeNavigation()
-    }}
-    onCancel={async () => {
-      override = true
-      resumeNavigation()
-    }}
-  >
-    <Body>Leaving this section will mean losing and changes to your query</Body>
-  </ModalContent>
-</Modal>
-
-<div class="wrapper">
-  <Layout gap="S" noPadding>
-    <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
-    <Divider />
-    <Heading size="S">Config</Heading>
-    <div class="config">
-      <div class="config-field">
-        <Label>Query Name</Label>
-        <Input
-          value={query.name}
-          on:input={e => {
-            let newValue = e.target.value || ""
-            if (newValue.match(ValidQueryNameRegex)) {
-              query.name = newValue.trim()
-              nameError = null
-            } else {
-              nameError = "Invalid query name"
-            }
-          }}
-          error={nameError}
-        />
-      </div>
-      {#if queryConfig}
-        <div class="config-field">
-          <Label>Function</Label>
-          <Select
-            bind:value={query.queryVerb}
-            on:change={resetDependentFields}
-            options={Object.keys(queryConfig)}
-            getOptionLabel={verb =>
-              queryConfig[verb]?.displayName || capitalise(verb)}
-          />
-        </div>
-        <div class="config-field">
-          <AccessLevelSelect {saveId} {query} label="Access Level" />
-        </div>
-        {#if integrationInfo?.extra && query.queryVerb}
-          <ExtraQueryConfig
-            {query}
-            {populateExtraQuery}
-            config={integrationInfo.extra}
-          />
-        {/if}
-        {#key query.parameters}
-          <div class="binding-wrap">
-            <BindingBuilder
-              queryBindings={query.parameters}
-              bindable={false}
-              on:change={e => {
-                query.parameters = e.detail.map(binding => {
-                  return {
-                    name: binding.name,
-                    default: binding.value,
-                  }
-                })
-              }}
-            />
-          </div>
-        {/key}
-      {/if}
-    </div>
-    {#if shouldShowQueryConfig}
-      <Divider />
-      <div class="config">
-        <Heading size="S">Fields</Heading>
-        <Body size="S">Fill in the fields specific to this query.</Body>
-        <IntegrationQueryEditor
-          {datasource}
-          {query}
-          height={200}
-          schema={queryConfig[query.queryVerb]}
-          bind:parameters
-        />
-        <Divider />
-      </div>
-      <div class="config">
-        <div class="help-heading">
-          <Heading size="S">Transformer</Heading>
-          <Icon
-            on:click={() => window.open(transformerDocs)}
-            hoverable
-            name="Help"
-            size="L"
-          />
-        </div>
+<QueryViewerSavePromptModal
+  checkIsModified={() => checkIsModified(newQuery)}
+  attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}
+/>
+<div class="queryViewer">
+  <div class="main">
+    <div class="header" class:scrolling>
+      <div class="title">
         <Body size="S">
-          Add a JavaScript function to transform the query result.
+          {newQuery.name || "Untitled query"}<span class="unsaved"
+            >{modified ? "*" : ""}</span
+          >
         </Body>
-        <CodeMirrorEditor
-          height={200}
-          label="Transformer"
-          value={query.transformer}
-          resize="vertical"
-          on:change={e => (query.transformer = e.detail)}
-        />
-        <Divider />
       </div>
-      <div class="viewer-controls">
-        <Heading size="S">Results</Heading>
-        <ButtonGroup gap="S">
+      <div class="controls">
+        <Button disabled={loading} on:click={runQuery} overBackground>
+          <Icon size="S" name="Play" />
+          Run query</Button
+        >
+        <div class="tooltip" title="Run your query to enable saving">
           <Button
-            cta
-            disabled={queryInvalid}
             on:click={async () => {
-              await saveQuery()
-              // Go to the correct URL if we just created a new query
-              if (!query._rev) {
-                $goto(`../../${query._id}`)
+              const response = await saveQuery()
+
+              // When creating a new query the initally passed in query object will have no id.
+              if (response._id && !newQuery._id) {
+                // Set the comparison query hash to match the new query so that the user doesn't
+                // get nagged when navigating to the edit view
+                queryHash = JSON.stringify(newQuery)
+                $goto(`../../${response._id}`)
               }
             }}
+            disabled={loading ||
+              !newQuery.name ||
+              nameError ||
+              rows.length === 0}
+            overBackground
           >
-            Save Query
+            <Icon size="S" name="SaveFloppy" />
+            Save
           </Button>
-          <Button secondary on:click={previewQuery}>Run Query</Button>
-        </ButtonGroup>
+        </div>
       </div>
-      <Body size="S">
-        Below, you can preview the results from your query and change the
-        schema.
-      </Body>
-      <section class="viewer">
-        {#if data}
-          <Tabs bind:selected={currentTab}>
-            <Tab title="JSON">
-              <JSONPreview data={data[0]} minHeight="120" />
-            </Tab>
-            <Tab title="Schema">
-              <KeyValueBuilder
-                bind:object={fields}
-                name="field"
-                headings
-                options={SchemaTypeOptions}
+    </div>
+
+    <div class="body" on:scroll={handleScroll}>
+      <div class="bodyInner">
+        <div class="configField">
+          <Label>Name</Label>
+          <Input
+            value={newQuery.name}
+            on:input={e => {
+              let newValue = e.target.value || ""
+              if (newValue.match(ValidQueryNameRegex)) {
+                newQuery.name = newValue.trim()
+                nameError = null
+              } else {
+                nameError = "Invalid query name"
+              }
+            }}
+            error={nameError}
+          />
+          {#if integration.query}
+            <Label>Function</Label>
+            <Select
+              bind:value={newQuery.queryVerb}
+              on:change={resetDependentFields}
+              options={Object.keys(integration.query)}
+              getOptionLabel={verb =>
+                integration.query[verb]?.displayName || capitalise(verb)}
+            />
+            <Label>Access</Label>
+            <AccessLevelSelect query={newQuery} />
+            {#if integration?.extra && newQuery.queryVerb}
+              <ExtraQueryConfig
+                query={newQuery}
+                {populateExtraQuery}
+                config={integration.extra}
               />
-            </Tab>
-            <Tab title="Preview">
-              <ExternalDataSourceTable {query} {data} />
-            </Tab>
-          </Tabs>
-        {/if}
-      </section>
-    {/if}
-  </Layout>
+            {/if}
+          {/if}
+        </div>
+
+        <Divider />
+
+        <div class="heading">
+          <Heading weight="L" size="XS">Query</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            {#if schemaType === "sql"}
+              Add some SQL to query your data
+            {:else if schemaType === "json"}
+              Add some JSON to query your data
+            {:else if schemaType === "fields"}
+              Add some fields to query your data
+            {:else}
+              Enter your query below
+            {/if}
+          </Body>
+        </div>
+        <IntegrationQueryEditor
+          noLabel
+          {datasource}
+          bind:query={newQuery}
+          height={200}
+          schema={integration.query[newQuery.queryVerb]}
+        />
+
+        <Divider />
+
+        <div class="heading">
+          <Heading weight="L" size="XS">Bindings</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            Bindings come in two parts: the binding name, and a default/fallback
+            value. These bindings can be used as Handlebars expressions
+            throughout the query.
+          </Body>
+        </div>
+        {#key newQuery.parameters}
+          <BindingBuilder
+            hideHeading
+            queryBindings={newQuery.parameters}
+            on:change={e => {
+              newQuery.parameters = e.detail.map(binding => {
+                return {
+                  name: binding.name,
+                  default: binding.value,
+                }
+              })
+            }}
+          />
+        {/key}
+
+        <Divider />
+        <div class="heading">
+          <Heading weight="L" size="XS">Transformer</Heading>
+        </div>
+        <div class="copy">
+          <Body size="S">
+            Add a JavaScript function to transform the query result.
+          </Body>
+        </div>
+        <CodeMirrorEditor
+          height={200}
+          value={newQuery.transformer}
+          resize="vertical"
+          on:change={e => (newQuery.transformer = e.detail)}
+        />
+      </div>
+    </div>
+  </div>
+
+  <div class:showSidePanel class="sidePanel">
+    <QueryViewerSidePanel
+      onClose={() => (showSidePanel = false)}
+      onSchemaChange={newSchema => {
+        newQuery.schema = newSchema
+      }}
+      {rows}
+      schema={Object.keys(newQuery.schema).length === 0
+        ? autoSchema
+        : newQuery.schema}
+    />
+  </div>
 </div>
 
 <style>
-  .wrapper {
-    width: 640px;
+  .unsaved {
+    color: var(--grey-5);
+    font-style: italic;
+  }
+
+  .queryViewer {
+    height: 100%;
+    margin: -28px -40px -40px -40px;
+    display: flex;
+    flex: 1;
+  }
+
+  .queryViewer :global(.spectrum-Divider) {
+    margin: 35px 0;
+  }
+
+  .main {
+    flex-grow: 1;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+  }
+
+  .header {
+    align-items: center;
+    padding: 8px 10px 8px 16px;
+    display: flex;
+    border-bottom: 2px solid transparent;
+    transition: border-bottom 130ms ease-out, background 130ms ease-out;
+  }
+
+  .header.scrolling {
+    border-bottom: var(--border-light);
+    background: var(--background);
+  }
+
+  .body {
+    flex-grow: 1;
+    overflow-y: scroll;
+    padding: 23px 23px 80px;
+    box-sizing: border-box;
+  }
+
+  .bodyInner {
+    max-width: 520px;
     margin: auto;
   }
 
-  .config {
-    display: grid;
-    grid-gap: var(--spacing-s);
-    z-index: 1;
+  .title {
+    /* width 0 paired with flex-grow necessary here for the truncation to work properly*/
+    width: 0;
+    flex-grow: 1;
   }
 
-  .config-field {
+  .title :global(p) {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .controls {
+    flex-shrink: 0;
+  }
+
+  .tooltip {
+    display: inline-block;
+  }
+
+  .controls :global(button) {
+    border: none;
+    color: var(--grey-7);
+    font-weight: 300;
+  }
+
+  .controls :global(button):hover {
+    background-color: transparent;
+    color: var(--ink);
+  }
+
+  .controls :global(.is-disabled) {
+    pointer-events: none;
+    background-color: transparent;
+    color: var(--grey-3);
+  }
+
+  .controls :global(span) {
+    display: flex;
+    align-items: center;
+  }
+
+  .controls :global(.icon) {
+    margin-right: 8px;
+  }
+
+  .configField {
     display: grid;
     grid-template-columns: 20% 1fr;
     grid-gap: var(--spacing-l);
     align-items: center;
   }
 
-  .help-heading {
-    display: flex;
-    justify-content: space-between;
+  .configField :global(label) {
+    color: var(--grey-6);
   }
 
-  .viewer {
-    min-height: 200px;
-    width: 640px;
+  .heading {
+    margin-bottom: 8px;
   }
 
-  .viewer-controls {
-    display: flex;
-    flex-direction: row;
-    justify-content: space-between;
-    gap: var(--spacing-m);
-    min-width: 150px;
-    align-items: center;
+  .copy {
+    margin-bottom: 14px;
   }
 
-  .binding-wrap :global(div.container) {
-    padding-left: 0px;
-    padding-right: 0px;
+  .copy :global(p) {
+    color: var(--grey-7);
+  }
+
+  .sidePanel {
+    flex-shrink: 0;
+    height: 100%;
+    width: 0;
+    overflow: hidden;
+    transition: width 150ms;
+  }
+
+  .sidePanel :global(.panel) {
+    height: 100%;
+  }
+
+  .showSidePanel {
+    width: 450px;
   }
 </style>

packages/builder/src/components/integration/QueryViewerBindingBuilder.svelte

@@ -1,9 +1,9 @@
 <script>
-  import { Body, Heading, Layout } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { getUserBindings } from "builderStore/dataBinding"
   export let bindable = true
   export let queryBindings = []
+  export let hideHeading = false
 
   const userBindings = getUserBindings()
 
@@ -13,44 +13,16 @@
   }, {})
 </script>
 
-<Layout noPadding={bindable} gap="S">
-  <div class="controls" class:height={!bindable}>
-    <Heading size="XS">Bindings</Heading>
-  </div>
-  <Body size="S">
-    {#if !bindable}
-      Bindings come in two parts: the binding name, and a default/fallback
-      value. These bindings can be used as Handlebars expressions throughout the
-      query.
-    {:else}
-      Enter a value for each binding. The default values will be used for any
-      values left blank.
-    {/if}
-  </Body>
-  <div class="bindings" class:bindable>
-    <KeyValueBuilder
-      bind:object={internalBindings}
-      tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
-      name="binding"
-      headings
-      keyPlaceholder="Binding name"
-      valuePlaceholder="Default"
-      bindings={[...userBindings]}
-      bindingDrawerLeft="260px"
-      allowHelpers={false}
-      on:change
-    />
-  </div>
-</Layout>
-
-<style>
-  .controls {
-    display: flex;
-    align-items: center;
-    justify-content: space-between;
-  }
-
-  .height {
-    height: 40px;
-  }
-</style>
+<KeyValueBuilder
+  bind:object={internalBindings}
+  tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
+  name="binding"
+  customButtonText="Bindings"
+  headings
+  keyPlaceholder="Binding name"
+  valuePlaceholder="Default"
+  bindings={[...userBindings]}
+  bindingDrawerLeft="260px"
+  allowHelpers={false}
+  on:change
+/>

packages/builder/src/components/integration/QueryViewerSavePromptModal.svelte

@@ -0,0 +1,53 @@
+<script>
+  import { goto, beforeUrlChange } from "@roxi/routify"
+  import { Body, Modal, ModalContent } from "@budibase/bbui"
+
+  export let checkIsModified = () => {}
+  export let attemptSave = () => {}
+  let modal
+  let navigateTo
+  let override = false
+
+  $beforeUrlChange(event => {
+    if (checkIsModified() && !override) {
+      navigateTo = event.type == "pushstate" ? event.url : null
+      modal.show()
+      return false
+    } else return true
+  })
+
+  const resumeNavigation = () => {
+    if (typeof navigateTo == "string") {
+      $goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
+    }
+  }
+</script>
+
+<Modal
+  bind:this={modal}
+  on:hide={() => {
+    navigateTo = null
+  }}
+>
+  <ModalContent
+    title="You have unsaved changes"
+    confirmText="Save and Continue"
+    cancelText="Discard Changes"
+    size="L"
+    onConfirm={async () => {
+      try {
+        await attemptSave()
+        override = true
+        resumeNavigation()
+      } catch (e) {
+        navigateTo = false
+      }
+    }}
+    onCancel={async () => {
+      override = true
+      resumeNavigation()
+    }}
+  >
+    <Body>Leaving this section will mean losing any changes to your query</Body>
+  </ModalContent>
+</Modal>

packages/builder/src/components/integration/QueryViewerSidePanel/JSONPanel.svelte

@@ -0,0 +1,22 @@
+<script>
+  export let data
+
+  $: string = JSON.stringify(data || {}, null, 2)
+</script>
+
+<textarea class="json" disabled value={string} />
+
+<style>
+  .json {
+    width: 100%;
+    height: 100%;
+    color: var(--ink);
+    padding: 12px;
+    box-sizing: border-box;
+    resize: none;
+    background-color: var(
+      --spectrum-textfield-m-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+  }
+</style>

packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte

@@ -0,0 +1,33 @@
+<script>
+  import Table from "components/backend/DataTable/Table.svelte"
+  import { cloneDeep } from "lodash/fp"
+
+  export let schema = {}
+  export let rows = []
+
+  $: rowsCopy = cloneDeep(rows)
+
+  // Cast field in query preview response to number if specified by schema
+  $: {
+    for (let i = 0; i < rowsCopy.length; i++) {
+      let row = rowsCopy[i]
+      for (let fieldName of Object.keys(schema)) {
+        if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
+          row[fieldName] = Number(row[fieldName])
+        } else {
+          row[fieldName] = row[fieldName]?.toString()
+        }
+      }
+    }
+  }
+</script>
+
+<div class="table">
+  <Table {schema} data={rowsCopy} type="external" allowEditing={false} />
+</div>
+
+<style>
+  .table :global(.spectrum-Table-cell) {
+    min-width: 100px;
+  }
+</style>

packages/builder/src/components/integration/QueryViewerSidePanel/SchemaPanel.svelte

@@ -0,0 +1,29 @@
+<script>
+  import KeyValueBuilder from "../KeyValueBuilder.svelte"
+  import { SchemaTypeOptions } from "constants/backend"
+
+  export let schema
+  export let onSchemaChange = () => {}
+
+  const handleChange = e => {
+    let newSchema = {}
+
+    // KeyValueBuilder on change event returns an array of objects with each object
+    // containing a field
+    e.detail.forEach(({ name, value }) => {
+      newSchema[name] = value
+    })
+
+    onSchemaChange(newSchema)
+  }
+</script>
+
+{#key schema}
+  <KeyValueBuilder
+    on:change={handleChange}
+    object={schema}
+    name="field"
+    headings
+    options={SchemaTypeOptions}
+  />
+{/key}

packages/builder/src/components/integration/QueryViewerSidePanel/index.svelte

@@ -0,0 +1,66 @@
+<script>
+  import Panel from "components/design/Panel.svelte"
+  import { ActionButton } from "@budibase/bbui"
+  import JSONPanel from "./JSONPanel.svelte"
+  import SchemaPanel from "./SchemaPanel.svelte"
+  import PreviewPanel from "./PreviewPanel.svelte"
+
+  export let rows
+  export let schema
+  export let onSchemaChange = () => {}
+  export let onClose = () => {}
+
+  const tabs = ["JSON", "Schema", "Preview"]
+  let activeTab = "JSON"
+</script>
+
+<Panel
+  showCloseButton
+  closeButtonIcon="RailRightClose"
+  onClickCloseButton={onClose}
+  title="Query results"
+  icon={"SQLQuery"}
+  borderLeft
+  extraWide
+>
+  <div slot="panel-header-content">
+    <div class="settings-tabs">
+      {#each tabs as tab}
+        <ActionButton
+          size="M"
+          quiet
+          selected={activeTab === tab}
+          on:click={() => {
+            activeTab = tab
+          }}
+        >
+          {tab}
+        </ActionButton>
+      {/each}
+    </div>
+  </div>
+  <div class="content">
+    {#if activeTab === "JSON"}
+      <JSONPanel data={rows[0] || {}} />
+    {:else if activeTab === "Schema"}
+      <SchemaPanel {onSchemaChange} {schema} />
+    {:else}
+      <PreviewPanel {schema} {rows} />
+    {/if}
+  </div>
+</Panel>
+
+<style>
+  .settings-tabs {
+    display: flex;
+    gap: var(--spacing-s);
+    padding: 0 var(--spacing-l);
+    padding-bottom: var(--spacing-l);
+  }
+
+  .content {
+    padding: 14px;
+    height: 100%;
+    overflow: scroll;
+  }
+</style>

packages/builder/src/components/integration/index.svelte

@@ -23,6 +23,7 @@
   export let schema
   export let editable = true
   export let height = 500
+  export let noLabel = false
 
   let stepEditors = []
 
@@ -75,7 +76,7 @@
     {#if schema.type === QueryTypes.SQL}
       <Editor
         editorHeight={height}
-        label="Query"
+        label={noLabel ? null : "Query"}
         mode="sql"
         on:change={updateQuery}
         readOnly={!editable}
@@ -85,7 +86,7 @@
     {:else if shouldDisplayJsonBox}
       <Editor
         editorHeight={height}
-        label="Query"
+        label={noLabel ? null : "Query"}
         mode="json"
         on:change={updateQuery}
         readOnly={!editable}

packages/builder/src/components/start/ImportAppModal.svelte

@@ -0,0 +1,70 @@
+<script>
+  import {
+    ModalContent,
+    Toggle,
+    Input,
+    Layout,
+    Dropzone,
+    notifications,
+    Body,
+  } from "@budibase/bbui"
+  import { API } from "api"
+  import { automationStore, store } from "../../builderStore"
+
+  export let app
+
+  $: disabled = (encrypted && !password) || !file
+  let encrypted = false,
+    password
+  let file
+
+  async function updateApp() {
+    try {
+      let data = new FormData()
+      data.append("appExport", file)
+      if (encrypted) {
+        data.append("encryptionPassword", password.trim())
+      }
+      const appId = app.devId
+      await API.updateAppFromExport(appId, data)
+      const pkg = await API.fetchAppPackage(appId)
+      await store.actions.initialise(pkg)
+      await automationStore.actions.fetch()
+      notifications.success("App updated successfully")
+    } catch (err) {
+      notifications.error(`Failed to update app - ${err.message || err}`)
+    }
+  }
+</script>
+
+<ModalContent
+  title={`Update ${app.name}`}
+  confirmText="Update"
+  onConfirm={updateApp}
+  bind:disabled
+>
+  <Body size="S"
+    >Updating an app using an app export will replace all tables, datasources,
+    queries, screens and automations. It is recommended to perform a backup
+    before running this operation.</Body
+  >
+  <Layout noPadding gap="XS">
+    <Dropzone
+      gallery={false}
+      label="App export"
+      on:change={e => {
+        file = e.detail?.[0]
+        encrypted = file?.name?.endsWith(".enc.tar.gz")
+      }}
+    />
+    <Toggle text="Encrypted" bind:value={encrypted} />
+    {#if encrypted}
+      <Input
+        type="password"
+        label="Password"
+        placeholder="Type here..."
+        bind:value={password}
+      />
+    {/if}
+  </Layout>
+</ModalContent>

packages/builder/src/constants/backend/index.js

@@ -1,126 +1,4 @@
-export const FIELDS = {
-  STRING: {
-    name: "Text",
-    type: "string",
-    icon: "Text",
-    constraints: {
-      type: "string",
-      length: {},
-      presence: false,
-    },
-  },
-  BARCODEQR: {
-    name: "Barcode/QR",
-    type: "barcodeqr",
-    icon: "Camera",
-    constraints: {
-      type: "string",
-      length: {},
-      presence: false,
-    },
-  },
-  LONGFORM: {
-    name: "Long Form Text",
-    type: "longform",
-    icon: "TextAlignLeft",
-    constraints: {
-      type: "string",
-      length: {},
-      presence: false,
-    },
-  },
-  OPTIONS: {
-    name: "Options",
-    type: "options",
-    icon: "Dropdown",
-    constraints: {
-      type: "string",
-      presence: false,
-      inclusion: [],
-    },
-  },
-  ARRAY: {
-    name: "Multi-select",
-    type: "array",
-    icon: "Duplicate",
-    constraints: {
-      type: "array",
-      presence: false,
-      inclusion: [],
-    },
-  },
-  NUMBER: {
-    name: "Number",
-    type: "number",
-    icon: "123",
-    constraints: {
-      type: "number",
-      presence: false,
-      numericality: { greaterThanOrEqualTo: "", lessThanOrEqualTo: "" },
-    },
-  },
-  BIGINT: {
-    name: "BigInt",
-    type: "bigint",
-    icon: "TagBold",
-  },
-  BOOLEAN: {
-    name: "Boolean",
-    type: "boolean",
-    icon: "Boolean",
-    constraints: {
-      type: "boolean",
-      presence: false,
-    },
-  },
-  DATETIME: {
-    name: "Date/Time",
-    type: "datetime",
-    icon: "Calendar",
-    constraints: {
-      type: "string",
-      length: {},
-      presence: false,
-      datetime: {
-        latest: "",
-        earliest: "",
-      },
-    },
-  },
-  ATTACHMENT: {
-    name: "Attachment",
-    type: "attachment",
-    icon: "Folder",
-    constraints: {
-      type: "array",
-      presence: false,
-    },
-  },
-  LINK: {
-    name: "Relationship",
-    type: "link",
-    icon: "Link",
-    constraints: {
-      type: "array",
-      presence: false,
-    },
-  },
-  FORMULA: {
-    name: "Formula",
-    type: "formula",
-    icon: "Calculator",
-    constraints: {},
-  },
-  JSON: {
-    name: "JSON",
-    type: "json",
-    icon: "Brackets",
-    constraints: {
-      type: "object",
-      presence: false,
-    },
-  },
-}
+import { FieldType, FieldSubtype } from "@budibase/types"
 
 export const AUTO_COLUMN_SUB_TYPES = {
   AUTO_ID: "autoID",
@@ -138,6 +16,151 @@ export const AUTO_COLUMN_DISPLAY_NAMES = {
   UPDATED_AT: "Updated At",
 }
 
+export const FIELDS = {
+  STRING: {
+    name: "Text",
+    type: FieldType.STRING,
+    icon: "Text",
+    constraints: {
+      type: "string",
+      length: {},
+      presence: false,
+    },
+  },
+  BARCODEQR: {
+    name: "Barcode/QR",
+    type: FieldType.BARCODEQR,
+    icon: "Camera",
+    constraints: {
+      type: "string",
+      length: {},
+      presence: false,
+    },
+  },
+  LONGFORM: {
+    name: "Long Form Text",
+    type: FieldType.LONGFORM,
+    icon: "TextAlignLeft",
+    constraints: {
+      type: "string",
+      length: {},
+      presence: false,
+    },
+  },
+  OPTIONS: {
+    name: "Options",
+    type: FieldType.OPTIONS,
+    icon: "Dropdown",
+    constraints: {
+      type: "string",
+      presence: false,
+      inclusion: [],
+    },
+  },
+  ARRAY: {
+    name: "Multi-select",
+    type: FieldType.ARRAY,
+    icon: "Duplicate",
+    constraints: {
+      type: "array",
+      presence: false,
+      inclusion: [],
+    },
+  },
+  NUMBER: {
+    name: "Number",
+    type: FieldType.NUMBER,
+    icon: "123",
+    constraints: {
+      type: "number",
+      presence: false,
+      numericality: { greaterThanOrEqualTo: "", lessThanOrEqualTo: "" },
+    },
+  },
+  BIGINT: {
+    name: "BigInt",
+    type: FieldType.BIGINT,
+    icon: "TagBold",
+  },
+  BOOLEAN: {
+    name: "Boolean",
+    type: FieldType.BOOLEAN,
+    icon: "Boolean",
+    constraints: {
+      type: "boolean",
+      presence: false,
+    },
+  },
+  DATETIME: {
+    name: "Date/Time",
+    type: FieldType.DATETIME,
+    icon: "Calendar",
+    constraints: {
+      type: "string",
+      length: {},
+      presence: false,
+      datetime: {
+        latest: "",
+        earliest: "",
+      },
+    },
+  },
+  ATTACHMENT: {
+    name: "Attachment",
+    type: FieldType.ATTACHMENT,
+    icon: "Folder",
+    constraints: {
+      type: "array",
+      presence: false,
+    },
+  },
+  LINK: {
+    name: "Relationship",
+    type: FieldType.LINK,
+    icon: "Link",
+    constraints: {
+      type: "array",
+      presence: false,
+    },
+  },
+  AUTO: {
+    name: "Auto Column",
+    type: FieldType.AUTO,
+    icon: "MagicWand",
+    constraints: {},
+  },
+  FORMULA: {
+    name: "Formula",
+    type: FieldType.FORMULA,
+    icon: "Calculator",
+    constraints: {},
+  },
+  JSON: {
+    name: "JSON",
+    type: FieldType.JSON,
+    icon: "Brackets",
+    constraints: {
+      type: "object",
+      presence: false,
+    },
+  },
+  USER: {
+    name: "User",
+    type: FieldType.BB_REFERENCE,
+    subtype: FieldSubtype.USER,
+    icon: "User",
+  },
+  USERS: {
+    name: "Users",
+    type: FieldType.BB_REFERENCE,
+    subtype: FieldSubtype.USERS,
+    icon: "User",
+    constraints: {
+      type: "array",
+    },
+  },
+}
+
 export const FILE_TYPES = {
   IMAGE: ["png", "tiff", "gif", "raw", "jpg", "jpeg"],
   CODE: ["js", "rs", "py", "java", "rb", "hs", "yml"],
@@ -170,6 +193,11 @@ export const RelationshipType = {
   MANY_TO_ONE: "many-to-one",
 }
 
+export const PrettyRelationshipDefinitions = {
+  MANY: "Many rows",
+  ONE: "One row",
+}
+
 export const ALLOWABLE_STRING_OPTIONS = [
   FIELDS.STRING,
   FIELDS.OPTIONS,

packages/builder/src/helpers/duplicate.js

@@ -3,16 +3,17 @@
  * e.g.
  *    name           all names          result
  *   ------        -----------         --------
- * ("foo")       ["foo"]              "foo (1)"
- * ("foo")       ["foo", "foo (1)"]   "foo (2)"
- * ("foo (1)")   ["foo", "foo (1)"]   "foo (2)"
- * ("foo")       ["foo", "foo (2)"]   "foo (1)"
+ * ("foo")       ["foo"]              "foo 1"
+ * ("foo")       ["foo", "foo 1"]     "foo 2"
+ * ("foo 1")     ["foo", "foo 1"]     "foo 2"
+ * ("foo")       ["foo", "foo 2"]     "foo 1"
  *
  * Repl
  */
 export const duplicateName = (name, allNames) => {
-  const baseName = name.split(" (")[0]
-  const isDuplicate = new RegExp(`${baseName}\\s\\((\\d+)\\)$`)
+  const duplicatePattern = new RegExp(`\\s(\\d+)$`)
+  const baseName = name.split(duplicatePattern)[0]
+  const isDuplicate = new RegExp(`${baseName}\\s(\\d+)$`)
 
   // get the sequence from matched names
   const sequence = []
@@ -28,7 +29,6 @@ export const duplicateName = (name, allNames) => {
     return false
   })
   sequence.sort((a, b) => a - b)
-
   // get the next number in the sequence
   let number
   if (sequence.length === 0) {
@@ -46,5 +46,5 @@ export const duplicateName = (name, allNames) => {
     }
   }
 
-  return `${baseName} (${number})`
+  return `${baseName} ${number}`
 }

packages/builder/src/helpers/tests/duplicate.test.js

@@ -9,34 +9,34 @@ describe("duplicate", () => {
 
       const duplicate = duplicateName(name, names)
 
-      expect(duplicate).toBe("foo (1)")
+      expect(duplicate).toBe("foo 1")
     })
 
     it("with multiple existing", async () => {
-      const names = ["foo", "foo (1)", "foo (2)"]
+      const names = ["foo", "foo 1", "foo 2"]
       const name = "foo"
 
       const duplicate = duplicateName(name, names)
 
-      expect(duplicate).toBe("foo (3)")
+      expect(duplicate).toBe("foo 3")
     })
 
     it("with mixed multiple existing", async () => {
-      const names = ["foo", "foo (1)", "foo (2)", "bar", "bar (1)", "bar (2)"]
+      const names = ["foo", "foo 1", "foo 2", "bar", "bar 1", "bar 2"]
       const name = "foo"
 
       const duplicate = duplicateName(name, names)
 
-      expect(duplicate).toBe("foo (3)")
+      expect(duplicate).toBe("foo 3")
     })
 
     it("with incomplete sequence", async () => {
-      const names = ["foo", "foo (2)", "foo (3)"]
+      const names = ["foo", "foo 2", "foo 3"]
       const name = "foo"
 
       const duplicate = duplicateName(name, names)
 
-      expect(duplicate).toBe("foo (1)")
+      expect(duplicate).toBe("foo 1")
     })
   })
 })