Merge branch 'design-section-feature-branch' of github.com:Budibase/budibase into screen-theme-rightpanel

.github/workflows/budibase_ci.yml

@@ -18,6 +18,8 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  NX_BASE_BRANCH: origin/${{ github.base_ref }}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
 
 jobs:
   lint:
@@ -25,20 +27,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn lint
 
   build:
@@ -46,45 +48,66 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
+
       # Run build all the projects
-      - run: yarn build
+      - name: Build
+        run: |
+          yarn build
       # Check the types of the projects built via esbuild
-      - run: yarn check:types
+      - name: Check types
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn check:types
+          fi
 
   test-libraries:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
-      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+      - run: yarn --frozen-lockfile
+      - name: Test
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+          fi
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -96,21 +119,31 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
-      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
+      - run: yarn --frozen-lockfile
+      - name: Test worker and server
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/worker --scope=@budibase/server
+          fi
+
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -119,60 +152,67 @@ jobs:
 
   test-pro:
     runs-on: ubuntu-latest
-    if: github.repository == github.event.pull_request.head.repo.full_name
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
-      - run: yarn test --scope=@budibase/pro
+      - run: yarn --frozen-lockfile
+      - name: Test
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/pro
+          fi
 
   integration-test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
-      - run: yarn build
+      - run: yarn --frozen-lockfile
+      - name: Build packages
+        run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core
       - name: Run tests
         run: |
           cd qa-core
           yarn setup
-          yarn test:ci
+          yarn serve:test:self:ci
         env:
           BB_ADMIN_USER_EMAIL: admin
           BB_ADMIN_USER_PASSWORD: admin
 
   check-pro-submodule:
     runs-on: ubuntu-latest
-    if: github.repository == github.event.pull_request.head.repo.full_name
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 0
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Check pro commit
@@ -190,6 +230,8 @@ jobs:
             base_commit=$(git rev-parse origin/develop)
           fi
 
+          echo "target_branch=$branch"
+          echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
           echo "pro_commit=$pro_commit"
           echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
           echo "base_commit=$base_commit"
@@ -204,7 +246,7 @@ jobs:
             const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
 
             if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the develop branch.');
+              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
               console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
               process.exit(1);
             } else {

.github/workflows/check_unreleased_changes.yml

@@ -0,0 +1,29 @@
+name: check_unreleased_changes
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check_unreleased:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for unreleased changes
+        env:
+          REPO: "Budibase/budibase"
+          TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+        run: |
+          RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
+              "https://api.github.com/repos/$REPO/releases/latest" | \
+              jq -r .published_at)
+          COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
+              "https://api.github.com/repos/$REPO/commits/master" | \
+              jq -r .commit.committer.date)
+          RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s")
+          COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s")
+          if (( COMMIT_SECONDS > RELEASE_SECONDS )); then
+              echo "There are unreleased changes. Please release these changes before merging."
+              exit 1
+          fi
+          echo "No unreleased changes detected."

.github/workflows/release-develop.yml

@@ -6,7 +6,7 @@ concurrency:
 on:
   push:
     tags:
-      - v*-alpha.*
+      - "*-alpha.*"
   workflow_dispatch:
 
 env:
@@ -44,7 +44,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - run: yarn install --frozen-lockfile
       - name: Update versions

.github/workflows/release-master.yml

@@ -6,9 +6,9 @@ concurrency:
 on:
   push:
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+"
       # Exclude all pre-releases
-      - "!v*[0-9]+.[0-9]+.[0-9]+-*"
+      - "!*[0-9]+.[0-9]+.[0-9]+-*"
 
 env:
   # Posthog token used by ui at build time
@@ -60,9 +60,9 @@ jobs:
       - name: "Get Current tag"
         id: currenttag
         run: |
-          version=v$(./scripts/getCurrentVersion.sh)
-          echo 'Using tag $version'
-          echo "::set-output name=tag::$resversionult"
+          version=$(./scripts/getCurrentVersion.sh)
+          echo "Using tag $version"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
 
       - name: Build/release Docker images
         run: |
@@ -71,7 +71,7 @@ jobs:
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.tag }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

.github/workflows/release-singleimage.yml

@@ -15,6 +15,13 @@ jobs:
       matrix:
         node-version: [14.x]
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 

.gitignore

@@ -101,8 +101,6 @@ packages/builder/cypress.env.json
 packages/builder/cypress/reports
 stats.html
 
-# TypeScript cache
-*.tsbuildinfo
 
 # plugins
 budibase-component

.nvmrc

@@ -1 +1 @@
-v14.20.1
+v18.17.0

.tool-versions

@@ -1,2 +1,3 @@
-nodejs 14.21.3
+nodejs 18.17.0
 python 3.10.0
+yarn 1.22.19

.vscode/launch.json

@@ -1,3 +1,4 @@
+
 {
   // Use IntelliSense to learn about possible attributes.
   // Hover to view descriptions of existing attributes.
@@ -8,30 +9,18 @@
       "name": "Budibase Server",
       "type": "node",
       "request": "launch",
-            "runtimeArgs": [
-                "--nolazy",
-                "-r",
-                "ts-node/register/transpile-only"
-            ],
-            "args": [
-                "${workspaceFolder}/packages/server/src/index.ts"
-            ],
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+      "args": ["${workspaceFolder}/packages/server/src/index.ts"],
       "cwd": "${workspaceFolder}/packages/server"
     },
     {
       "name": "Budibase Worker",
       "type": "node",
       "request": "launch",
-            "runtimeArgs": [
-                "--nolazy",
-                "-r",
-                "ts-node/register/transpile-only"
-            ],
-            "args": [
-                "${workspaceFolder}/packages/worker/src/index.ts"
-            ],
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+      "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
       "cwd": "${workspaceFolder}/packages/worker"
-        },
+    }
   ],
   "compounds": [
     {

charts/budibase/templates/app-service-deployment.yaml

@@ -120,6 +120,8 @@ spec:
         {{ end }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.apps.logLevel | quote }}
         - name: REDIS_PASSWORD
@@ -201,25 +203,24 @@ spec:
 
         image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
+        {{- if .Values.services.apps.startupProbe }}
+        {{- with .Values.services.apps.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.apps.livenessProbe }}
+        {{- with .Values.services.apps.livenessProbe }}
         livenessProbe:
-          httpGet:
-            path: /health
-            port: {{ .Values.services.apps.port }}
-          initialDelaySeconds: 10
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.apps.readinessProbe }}
+        {{- with .Values.services.apps.readinessProbe }}
         readinessProbe:
-          httpGet:
-            path: /health
-            port: {{ .Values.services.apps.port }}
-          initialDelaySeconds: 5
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
-
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
         name: bbapps
         ports:
         - containerPort: {{ .Values.services.apps.port }}

charts/budibase/templates/proxy-service-deployment.yaml

@@ -40,6 +40,24 @@ spec:
       - image: budibase/proxy:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         name: proxy-service
+        {{- if .Values.services.proxy.startupProbe }}
+        {{- with .Values.services.proxy.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.proxy.livenessProbe }}
+        {{- with .Values.services.proxy.livenessProbe }}
+        livenessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.proxy.readinessProbe }}
+        {{- with .Values.services.proxy.readinessProbe }}
+        readinessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
         ports:
         - containerPort: {{ .Values.services.proxy.port }}
         env:

charts/budibase/templates/worker-service-deployment.yaml

@@ -116,6 +116,8 @@ spec:
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.worker.logLevel | quote }}
         - name: REDIS_PASSWORD
@@ -190,24 +192,24 @@ spec:
         {{ end }}
         image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
+        {{- if .Values.services.worker.startupProbe }}
+        {{- with .Values.services.worker.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.worker.livenessProbe }}
+        {{- with .Values.services.worker.livenessProbe }}
         livenessProbe:
-          httpGet:
-            path: /health
-            port: {{ .Values.services.worker.port }}
-          initialDelaySeconds: 10
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.worker.readinessProbe }}
+        {{- with .Values.services.worker.readinessProbe }}
         readinessProbe:
-          httpGet:
-            path: /health
-            port: {{ .Values.services.worker.port }}
-          initialDelaySeconds: 5
-          periodSeconds: 5
-          successThreshold: 1
-          failureThreshold: 3
-          timeoutSeconds: 3
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
         name: bbworker
         ports:
         - containerPort: {{ .Values.services.worker.port }}

charts/budibase/values.yaml

@@ -82,6 +82,7 @@ globals:
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  offlineMode: "0" # set to 1 to enable offline mode
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""
@@ -119,15 +120,36 @@ services:
     port: 10000
     replicaCount: 1
     upstreams:
-      apps: 'http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}'
-      worker: 'http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}'
-      minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}'
-      couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}'
+      apps: "http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}"
+      worker: "http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}"
+      minio: "http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}"
+      couchdb: "http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}"
     resources: {}
-#    annotations:
-#      co.elastic.logs/module: nginx
-#      co.elastic.logs/fileset.stdout: access
-#      co.elastic.logs/fileset.stderr: error
+    startupProbe:
+      httpGet:
+        path: /health
+        port: 10000
+        scheme: HTTP
+      failureThreshold: 30
+      periodSeconds: 3
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 10000
+        scheme: HTTP
+      periodSeconds: 3
+      failureThreshold: 1
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 10000
+        scheme: HTTP
+      failureThreshold: 3
+      periodSeconds: 5
+  #    annotations:
+  #      co.elastic.logs/module: nginx
+  #      co.elastic.logs/fileset.stdout: access
+  #      co.elastic.logs/fileset.stderr: error
 
   apps:
     port: 4002
@@ -135,23 +157,65 @@ services:
     logLevel: info
     httpLogging: 1
     resources: {}
-#    nodeDebug: "" # set the value of NODE_DEBUG
-#    annotations:
-#      co.elastic.logs/multiline.type: pattern
-#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
-#      co.elastic.logs/multiline.negate: false
-#      co.elastic.logs/multiline.match: after
+    startupProbe:
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      failureThreshold: 30
+      periodSeconds: 3
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      periodSeconds: 3
+      failureThreshold: 1
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      failureThreshold: 3
+      periodSeconds: 5
+  #    nodeDebug: "" # set the value of NODE_DEBUG
+  #    annotations:
+  #      co.elastic.logs/multiline.type: pattern
+  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+  #      co.elastic.logs/multiline.negate: false
+  #      co.elastic.logs/multiline.match: after
   worker:
     port: 4003
     replicaCount: 1
     logLevel: info
     httpLogging: 1
     resources: {}
-#    annotations:
-#      co.elastic.logs/multiline.type: pattern
-#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
-#      co.elastic.logs/multiline.negate: false
-#      co.elastic.logs/multiline.match: after
+    startupProbe:
+      httpGet:
+        path: /health
+        port: 4003
+        scheme: HTTP
+      failureThreshold: 30
+      periodSeconds: 3
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: 4003
+        scheme: HTTP
+      periodSeconds: 3
+      failureThreshold: 1
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 4003
+        scheme: HTTP
+      failureThreshold: 3
+      periodSeconds: 5
+  #    annotations:
+  #      co.elastic.logs/multiline.type: pattern
+  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+  #      co.elastic.logs/multiline.negate: false
+  #      co.elastic.logs/multiline.match: after
 
   couchdb:
     enabled: true
@@ -344,14 +408,12 @@ couchdb:
   ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
   # FOR COUCHDB
   livenessProbe:
-    enabled: true
     failureThreshold: 3
     initialDelaySeconds: 0
     periodSeconds: 10
     successThreshold: 1
     timeoutSeconds: 1
   readinessProbe:
-    enabled: true
     failureThreshold: 3
     initialDelaySeconds: 0
     periodSeconds: 10

docs/CONTRIBUTING.md

@@ -90,7 +90,7 @@ Component libraries are collections of components as well as the definition of t
 
 #### 1. Prerequisites
 
-- NodeJS version `14.x.x`
+- NodeJS version `18.x.x`
 - Python version `3.x`
 
 ### Using asdf (recommended)
@@ -231,18 +231,33 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README.
 
 ### Pro
 
-@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g.
+@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you need to make an update to pro and have access to the repo, then you can update your submodule within the mono-repo by running `git submodule update --init` - from here you can use normal submodule flow to develop a change within pro.
+
+Once you have updated to use the pro submodule, it will be linked into all of your local dependencies by NX as with all other monorepo packages. If you have been using the NPM version of `@budibase/pro` then you may need to run a `git reset --hard` to fix all of the pro versions back to `0.0.0` to be monorepo aware.
+
+From here - to develop a change in pro, you can follow the below flow:
 
 ```
-.
-|_ budibase
-|_ budibase-pro
+# enter the pro submodule
+cd packages/pro
+# get the base branch you are working from (same as monorepo)
+git fetch
+git checkout <develop | master>
+# create a branch, named the same as the branch in your monorepo
+git checkout -b <some branch>
+... make changes
+# commit the changes you've made, with a message for pro
+git commit <something>
+# within the monorepo, add the pro reference to your branch, commit it with a message like "Update pro ref"
+cd ../..
+git add packages/pro
+git commit <add the new reference to main repo>
 ```
 
+From here, you will have created a branch in the pro repository and commited the reference to your branch on the monorepo. When you eventually PR this work back into the mainline branch, you will need to first merge your pro PR to the pro mainline, then go into your PR in the monorepo and update the reference again to the new mainline.
+
 Note that only budibase maintainers will be able to access the pro repo.
 
-By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
-
 ### Troubleshooting
 
 Sometimes, things go wrong. This can be due to incompatible updates on the budibase platform. To clear down your development environment and start again follow **Step 6. Cleanup**, then proceed from **Step 3. Install and Build** in the setup guide above to create a fresh Budibase installation.

hosting/couchdb/Dockerfile

@@ -5,11 +5,11 @@ ENV COUCHDB_PASSWORD admin
 EXPOSE 5984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
-    wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \
+    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo apt-key add - && \
     apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
     apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
-    apt-add-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \
-    apt-get update && apt-get install -y --no-install-recommends adoptopenjdk-8-hotspot && \
+    apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \
+    apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
     rm -rf /var/lib/apt/lists/
 
 # setup clouseau

hosting/docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
+      OFFLINE_MODE: ${OFFLINE_MODE}
     depends_on:
       - worker-service
       - redis-service
@@ -54,6 +55,7 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
+      OFFLINE_MODE: ${OFFLINE_MODE}
     depends_on:
       - redis-service
       - minio-service

hosting/single/Dockerfile

@@ -1,7 +1,7 @@
-FROM node:14-slim as build
+FROM node:18-slim as build
 
 # install node-gyp dependencies
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python3
 
 # add pin script
 WORKDIR /

jestTestcontainersConfigGenerator.js

@@ -1,9 +1,26 @@
 module.exports = () => {
   return {
-    dockerCompose: {
-      composeFilePath: "../../hosting",
-      composeFile: "docker-compose.test.yaml",
-      startupTimeout: 10000,
+    couchdb: {
+      image: "budibase/couchdb",
+      ports: [5984],
+      env: {
+        COUCHDB_PASSWORD: "budibase",
+        COUCHDB_USER: "budibase",
       },
+      wait: {
+        type: "ports",
+        timeout: 10000,
+      }
+    }
   }
 }
+
+// module.exports = () => {
+//   return {
+//     dockerCompose: {
+//       composeFilePath: "../../hosting",
+//       composeFile: "docker-compose.test.yaml",
+//       startupTimeout: 10000,
+//     },
+//   }
+// }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.16-alpha.0",
+  "version": "2.9.30-alpha.9",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

nx.json

@@ -3,24 +3,10 @@
     "default": {
       "runner": "nx-cloud",
       "options": {
-        "cacheableOperations": [
-          "build",
-          "test"
-        ],
-        "accessToken": "YWNiYzc5NTEtMzMzZC00NDhjLTgyNjktZTllMjI1MzM4OGQxfHJlYWQtd3JpdGU="
+        "cacheableOperations": ["build", "test", "check:types"],
+        "accessToken": "MmM4OGYxNzItMDBlYy00ZmE3LTk4MTYtNmJhYWMyZjBjZTUyfHJlYWQ="
       }
     }
   },
-  "targetDefaults": {
-    "dev:builder": {
-      "dependsOn": [
-        {
-          "projects": [
-            "@budibase/string-templates"
-          ],
-          "target": "build"
-        }
-      ]
-    }
-  }
+  "targetDefaults": {}
 }

package.json

@@ -6,8 +6,8 @@
     "@nx/js": "16.4.3",
     "@rollup/plugin-json": "^4.0.2",
     "@typescript-eslint/parser": "5.45.0",
-    "esbuild": "^0.17.18",
-    "esbuild-node-externals": "^1.7.0",
+    "esbuild": "^0.18.17",
+    "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
     "eslint-plugin-cypress": "^2.11.3",
     "husky": "^8.0.3",
@@ -34,9 +34,9 @@
     "preinstall": "node scripts/syncProPackage.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
-    "build": "yarn nx run-many -t=build",
+    "build": "lerna run build --stream",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run check:types --skip-nx-cache",
+    "check:types": "lerna run check:types",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
     "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
     "build:sdk": "lerna run --stream build:sdk",
@@ -51,7 +51,7 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna run --stream dev:builder --stream",
+    "dev": "yarn run kill-all && lerna run --stream dev:builder",
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
@@ -108,5 +108,8 @@
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0"
   },
+  "engines": {
+    "node": ">=18.0.0 <19.0.0"
+  },
   "dependencies": {}
 }

packages/backend-core/.npmignore

@@ -0,0 +1,4 @@
+*
+!dist/**/*
+dist/tsconfig.build.tsbuildinfo
+!package.json

packages/backend-core/jest.config.ts

@@ -1,8 +1,6 @@
 import { Config } from "@jest/types"
-const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
-  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./tests/jestSetup.ts"],
@@ -11,6 +9,7 @@ const baseConfig: Config.InitialProjectOptions = {
   },
   moduleNameMapper: {
     "@budibase/types": "<rootDir>/../types/src",
+    "@budibase/shared-core": ["<rootDir>/../shared-core/src"],
   },
 }
 

packages/backend-core/package.json

@@ -2,11 +2,11 @@
   "name": "@budibase/backend-core",
   "version": "0.0.0",
   "description": "Budibase backend core libraries used in server and worker",
-  "main": "dist/src/index.js",
+  "main": "dist/index.js",
   "types": "dist/src/index.d.ts",
   "exports": {
-    ".": "./dist/src/index.js",
-    "./tests": "./dist/tests/index.js",
+    ".": "./dist/index.js",
+    "./tests": "./dist/tests.js",
     "./*": "./dist/*.js"
   },
   "author": "Budibase",
@@ -14,16 +14,17 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
+    "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
@@ -58,12 +59,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.5.0",
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
+    "@jest/test-sequencer": "29.6.2",
+    "@shopify/jest-koa-mocks": "5.1.1",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/jest": "29.5.0",
+    "@types/jest": "29.5.3",
     "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
@@ -75,15 +77,14 @@
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
     "ioredis-mock": "8.7.0",
-    "jest": "29.5.0",
-    "jest-environment-node": "29.5.0",
-    "jest-serial-runner": "^1.2.1",
+    "jest": "29.6.2",
+    "jest-environment-node": "29.6.2",
+    "jest-serial-runner": "1.2.1",
     "koa": "2.13.4",
     "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
@@ -94,6 +95,7 @@
         "dependsOn": [
           {
             "projects": [
+              "@budibase/shared-core",
               "@budibase/types"
             ],
             "target": "build"
@@ -101,6 +103,5 @@
         ]
       }
     }
-  },
-  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
+  }
 }

packages/backend-core/plugins.ts

@@ -1 +0,0 @@
-export * from "./src/plugin"

packages/backend-core/scripts/build.js

@@ -0,0 +1,6 @@
+#!/usr/bin/node
+const coreBuild = require("../../../scripts/build")
+
+coreBuild("./src/plugin/index.ts", "./dist/plugins.js")
+coreBuild("./src/index.ts", "./dist/index.js")
+coreBuild("./tests/index.ts", "./dist/tests.js")

packages/backend-core/scripts/test.sh

@@ -8,6 +8,6 @@ then
   jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --forceExit"
-  jest --coverage --forceExit
+  echo "jest --coverage --detectOpenHandles"
+  jest --coverage --detectOpenHandles
 fi

packages/backend-core/src/cache/appMetadata.ts

@@ -2,9 +2,14 @@ import { getAppClient } from "../redis/init"
 import { doWithDB, DocumentType } from "../db"
 import { Database, App } from "@budibase/types"
 
-const AppState = {
-  INVALID: "invalid",
+export enum AppState {
+  INVALID = "invalid",
 }
+
+export interface DeletedApp {
+  state: AppState
+}
+
 const EXPIRY_SECONDS = 3600
 
 /**
@@ -31,7 +36,7 @@ function isInvalid(metadata?: { state: string }) {
  * @param {string} appId the id of the app to get metadata from.
  * @returns {object} the app metadata.
  */
-export async function getAppMetadata(appId: string) {
+export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
   const client = await getAppClient()
   // try cache
   let metadata = await client.get(appId)
@@ -61,11 +66,8 @@ export async function getAppMetadata(appId: string) {
     }
     await client.store(appId, metadata, expiry)
   }
-  // we've stored in the cache an object to tell us that it is currently invalid
-  if (isInvalid(metadata)) {
-    throw { status: 404, message: "No app metadata found" }
-  }
-  return metadata as App
+
+  return metadata
 }
 
 /**

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -36,7 +36,7 @@ describe("writethrough", () => {
           _id: docId,
           value: 1,
         })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         current = output
         expect(output.value).toBe(1)
       })
@@ -45,7 +45,7 @@ describe("writethrough", () => {
     it("second put shouldn't update DB", async () => {
       await config.doInTenant(async () => {
         const response = await writethrough.put({ ...current, value: 2 })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         expect(current._rev).toBe(output._rev)
         expect(output.value).toBe(1)
       })
@@ -55,7 +55,7 @@ describe("writethrough", () => {
       await config.doInTenant(async () => {
         tk.freeze(Date.now() + DELAY + 1)
         const response = await writethrough.put({ ...current, value: 3 })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         expect(response.rev).not.toBe(current._rev)
         expect(output.value).toBe(3)
 
@@ -79,7 +79,7 @@ describe("writethrough", () => {
           expect.arrayContaining([current._rev, current._rev, newRev])
         )
 
-        const output = await db.get(current._id)
+        const output = await db.get<any>(current._id)
         expect(output.value).toBe(4)
         expect(output._rev).toBe(newRev)
 
@@ -107,7 +107,7 @@ describe("writethrough", () => {
         })
         expect(res.ok).toBe(true)
 
-        const output = await db.get(id)
+        const output = await db.get<any>(id)
         expect(output.value).toBe(3)
         expect(output._rev).toBe(res.rev)
       })
@@ -130,8 +130,8 @@ describe("writethrough", () => {
         const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
         expect(resp1.rev).toBeDefined()
         expect(resp2.rev).toBeDefined()
-        expect((await db.get("db1")).value).toBe("first")
-        expect((await db2.get("db1")).value).toBe("second")
+        expect((await db.get<any>("db1")).value).toBe("first")
+        expect((await db2.get<any>("db1")).value).toBe("second")
       })
     })
   })

packages/backend-core/src/constants/db.ts

@@ -1,5 +1,5 @@
-export const SEPARATOR = "_"
-export const UNICODE_MAX = "\ufff0"
+import { prefixed, DocumentType } from "@budibase/types"
+export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -14,8 +14,6 @@ export enum ViewName {
   USER_BY_APP = "by_app",
   USER_BY_EMAIL = "by_email2",
   BY_API_KEY = "by_api_key",
-  /** @deprecated - could be deleted */
-  USER_BY_BUILDERS = "by_builders",
   LINK = "by_link",
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
@@ -36,42 +34,6 @@ export enum InternalTable {
   USER_METADATA = "ta_users",
 }
 
-export enum DocumentType {
-  USER = "us",
-  GROUP = "gr",
-  WORKSPACE = "workspace",
-  CONFIG = "config",
-  TEMPLATE = "template",
-  APP = "app",
-  DEV = "dev",
-  APP_DEV = "app_dev",
-  APP_METADATA = "app_metadata",
-  ROLE = "role",
-  MIGRATIONS = "migrations",
-  DEV_INFO = "devinfo",
-  AUTOMATION_LOG = "log_au",
-  ACCOUNT_METADATA = "acc_metadata",
-  PLUGIN = "plg",
-  DATASOURCE = "datasource",
-  DATASOURCE_PLUS = "datasource_plus",
-  APP_BACKUP = "backup",
-  TABLE = "ta",
-  ROW = "ro",
-  AUTOMATION = "au",
-  LINK = "li",
-  WEBHOOK = "wh",
-  INSTANCE = "inst",
-  LAYOUT = "layout",
-  SCREEN = "screen",
-  QUERY = "query",
-  DEPLOYMENTS = "deployments",
-  METADATA = "metadata",
-  MEM_VIEW = "view",
-  USER_FLAG = "flag",
-  AUTOMATION_METADATA = "meta_au",
-  AUDIT_LOG = "al",
-}
-
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",
@@ -95,7 +57,7 @@ export const StaticDatabases = {
   },
 }
 
-export const APP_PREFIX = DocumentType.APP + SEPARATOR
-export const APP_DEV = DocumentType.APP_DEV + SEPARATOR
+export const APP_PREFIX = prefixed(DocumentType.APP)
+export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"

packages/backend-core/src/constants/misc.ts

@@ -20,6 +20,8 @@ export enum Header {
   TYPE = "x-budibase-type",
   PREVIEW_ROLE = "x-budibase-role",
   TENANT_ID = "x-budibase-tenant-id",
+  VERIFICATION_CODE = "x-budibase-verification-code",
+  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
   CORRELATION_ID = "x-budibase-correlation-id",

packages/backend-core/src/db/constants.ts

@@ -0,0 +1,10 @@
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+] as const
+
+export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -8,6 +8,7 @@ import {
   DatabasePutOpts,
   DatabaseCreateIndexOpts,
   DatabaseDeleteIndexOpts,
+  DocExistsResponse,
   Document,
   isDocument,
 } from "@budibase/types"
@@ -120,6 +121,19 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
+  async docExists(docId: string): Promise<DocExistsResponse> {
+    const db = await this.checkSetup()
+    let _rev, exists
+    try {
+      const { etag } = await db.head(docId)
+      _rev = etag
+      exists = true
+    } catch (err) {
+      exists = false
+    }
+    return { _rev, exists }
+  }
+
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string

packages/backend-core/src/db/couch/index.ts

@@ -2,3 +2,4 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
+export * from "../constants"

packages/backend-core/src/db/lucene.ts

@@ -1,7 +1,6 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
-import { SearchFilters, Row } from "@budibase/types"
-import { createUserIndex } from "./searchIndexes/searchIndexes"
+import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
@@ -65,6 +64,7 @@ export class QueryBuilder<T> {
     this.#index = index
     this.#query = {
       allOr: false,
+      onEmptyFilter: EmptyFilterOption.RETURN_ALL,
       string: {},
       fuzzy: {},
       range: {},
@@ -218,6 +218,10 @@ export class QueryBuilder<T> {
     this.#query.allOr = true
   }
 
+  setOnEmptyFilter(value: EmptyFilterOption) {
+    this.#query.onEmptyFilter = value
+  }
+
   handleSpaces(input: string) {
     if (this.#noEscaping) {
       return input
@@ -289,8 +293,9 @@ export class QueryBuilder<T> {
     const builder = this
     let allOr = this.#query && this.#query.allOr
     let query = allOr ? "" : "*:*"
+    let allFiltersEmpty = true
     const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
-    let tableId
+    let tableId: string = ""
     if (this.#query.equal!.tableId) {
       tableId = this.#query.equal!.tableId
       delete this.#query.equal!.tableId
@@ -305,7 +310,7 @@ export class QueryBuilder<T> {
     }
 
     const contains = (key: string, value: any, mode = "AND") => {
-      if (Array.isArray(value) && value.length === 0) {
+      if (!value || (Array.isArray(value) && value.length === 0)) {
         return null
       }
       if (!Array.isArray(value)) {
@@ -384,6 +389,12 @@ export class QueryBuilder<T> {
           built += ` ${mode} `
         }
         built += expression
+        if (
+          (typeof value !== "string" && value != null) ||
+          (typeof value === "string" && value !== tableId && value !== "")
+        ) {
+          allFiltersEmpty = false
+        }
       }
       if (opts?.returnBuilt) {
         return built
@@ -463,6 +474,13 @@ export class QueryBuilder<T> {
       allOr = false
       build({ tableId }, equal)
     }
+    if (allFiltersEmpty) {
+      if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
+        return ""
+      } else if (this.#query?.allOr) {
+        return query.replace("()", "(*:*)")
+      }
+    }
     return query
   }
 

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -1,6 +1,6 @@
 import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
-import { Database } from "@budibase/types"
+import { Database, EmptyFilterOption } from "@budibase/types"
 import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
 
 const INDEX_NAME = "main"
@@ -156,6 +156,76 @@ describe("lucene", () => {
       expect(resp.rows.length).toBe(2)
     })
 
+    describe("empty filters behaviour", () => {
+      it("should return all rows by default", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(3)
+      })
+
+      it("should return all rows when onEmptyFilter is ALL", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
+        builder.setAllOr()
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(3)
+      })
+
+      it("should return no rows when onEmptyFilter is NONE", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(0)
+      })
+
+      it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
+        builder.addEqual("property", "")
+        builder.addEqual("number", 1)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(1)
+      })
+    })
+
     describe("skip", () => {
       const skipDbName = `db-${newid()}`
       let docs: {

packages/backend-core/src/db/utils.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
 import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
-import { getAppMetadata } from "../cache/appMetadata"
+import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
 import { App, Database } from "@budibase/types"
 import { getStartEndKeyURL } from "../docIds"
@@ -101,7 +101,9 @@ export async function getAllApps({
     const response = await Promise.allSettled(appPromises)
     const apps = response
       .filter(
-        (result: any) => result.status === "fulfilled" && result.value != null
+        (result: any) =>
+          result.status === "fulfilled" &&
+          result.value?.state !== AppState.INVALID
       )
       .map(({ value }: any) => value)
     if (!all) {
@@ -126,7 +128,11 @@ export async function getAppsByIDs(appIds: string[]) {
   )
   // have to list the apps which exist, some may have been deleted
   return settled
-    .filter(promise => promise.status === "fulfilled")
+    .filter(
+      promise =>
+        promise.status === "fulfilled" &&
+        (promise.value as DeletedApp).state !== AppState.INVALID
+    )
     .map(promise => (promise as PromiseFulfilledResult<App>).value)
 }
 

packages/backend-core/src/db/views.ts

@@ -105,16 +105,6 @@ export const createApiKeyView = async () => {
   await createView(db, viewJs, ViewName.BY_API_KEY)
 }
 
-export const createUserBuildersView = async () => {
-  const db = getGlobalDB()
-  const viewJs = `function(doc) {
-    if (doc.builder && doc.builder.global === true) {
-      emit(doc._id, doc._id)
-    }
-  }`
-  await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -223,7 +213,6 @@ export const queryPlatformView = async <T>(
 const CreateFuncByName: any = {
   [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
   [ViewName.BY_API_KEY]: createApiKeyView,
-  [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
   [ViewName.USER_BY_APP]: createUserAppView,
 }
 

packages/backend-core/src/environment.ts

@@ -1,4 +1,5 @@
 import { existsSync, readFileSync } from "fs"
+import { ServiceType } from "@budibase/types"
 
 function isTest() {
   return isCypress() || isJest()
@@ -83,10 +84,20 @@ function getPackageJsonFields(): {
   }
 }
 
+function isWorker() {
+  return environment.SERVICE_TYPE === ServiceType.WORKER
+}
+
+function isApps() {
+  return environment.SERVICE_TYPE === ServiceType.APPS
+}
+
 const environment = {
   isTest,
   isJest,
   isDev,
+  isWorker,
+  isApps,
   isProd: () => {
     return !isDev()
   },
@@ -153,6 +164,7 @@ const environment = {
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING,
   BLACKLIST_IPS: process.env.BLACKLIST_IPS,
+  SERVICE_TYPE: "unknown",
   /**
    * Enable to allow an admin user to login using a password.
    * This can be useful to prevent lockout when configuring SSO.
@@ -163,6 +175,7 @@ const environment = {
     : false,
   ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
+  OFFLINE_MODE: process.env.OFFLINE_MODE,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/errors/errors.ts

@@ -55,6 +55,18 @@ export class HTTPError extends BudibaseError {
   }
 }
 
+export class NotFoundError extends HTTPError {
+  constructor(message: string) {
+    super(message, 404)
+  }
+}
+
+export class BadRequestError extends HTTPError {
+  constructor(message: string) {
+    super(message, 400)
+  }
+}
+
 // LICENSING
 
 export class UsageLimitError extends HTTPError {

packages/backend-core/src/events/identification.ts

@@ -21,6 +21,7 @@ import { processors } from "./processors"
 import { newid } from "../utils"
 import * as installation from "../installation"
 import * as configs from "../configs"
+import * as users from "../users"
 import { withCache, TTL, CacheKey } from "../cache/generic"
 
 /**
@@ -164,8 +165,8 @@ const identifyUser = async (
   const id = user._id as string
   const tenantId = await getEventTenantId(user.tenantId)
   const type = IdentityType.USER
-  let builder = user.builder?.global || false
-  let admin = user.admin?.global || false
+  let builder = users.hasBuilderPermissions(user)
+  let admin = users.hasAdminPermissions(user)
   let providerType
   if (isSSOUser(user)) {
     providerType = user.providerType
@@ -264,7 +265,7 @@ const getEventTenantId = async (tenantId: string): Promise<string> => {
   }
 }
 
-const getUniqueTenantId = async (tenantId: string): Promise<string> => {
+export const getUniqueTenantId = async (tenantId: string): Promise<string> => {
   // make sure this tenantId always matches the tenantId in context
   return context.doInTenant(tenantId, () => {
     return withCache(CacheKey.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {

packages/backend-core/src/features/index.ts

@@ -1,5 +1,6 @@
 import env from "../environment"
 import * as context from "../context"
+export * from "./installation"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.

packages/backend-core/src/features/installation.ts

@@ -0,0 +1,17 @@
+export function processFeatureEnvVar<T>(
+  fullList: string[],
+  featureList?: string
+) {
+  let list
+  if (!featureList) {
+    list = fullList
+  } else {
+    list = featureList.split(",")
+  }
+  for (let feature of list) {
+    if (!fullList.includes(feature)) {
+      throw new Error(`Feature: ${feature} is not an allowed option`)
+    }
+  }
+  return list as unknown as T[]
+}

packages/backend-core/src/index.ts

@@ -6,7 +6,8 @@ export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
 export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as featureFlags from "./featureFlags"
+export * as featureFlags from "./features"
+export * as features from "./features/installation"
 export * as sessions from "./security/sessions"
 export * as platform from "./platform"
 export * as auth from "./auth"

packages/backend-core/src/logging/index.ts

@@ -2,6 +2,3 @@ export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
 export * as system from "./system"
-
-// turn off or on context logging i.e. tenantId, appId etc
-export let LOG_CONTEXT = true

packages/backend-core/src/logging/pino/logger.ts

@@ -2,11 +2,9 @@ import pino, { LoggerOptions } from "pino"
 import pinoPretty from "pino-pretty"
 
 import { IdentityType } from "@budibase/types"
-
 import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-import { LOG_CONTEXT } from "../index"
 
 import { localFileDestination } from "../system"
 
@@ -14,29 +12,44 @@ import { localFileDestination } from "../system"
 
 let pinoInstance: pino.Logger | undefined
 if (!env.DISABLE_PINO_LOGGER) {
+  const level = env.LOG_LEVEL
   const pinoOptions: LoggerOptions = {
-    level: env.LOG_LEVEL,
+    level,
     formatters: {
-      level: label => {
-        return { level: label.toUpperCase() }
+      level: level => {
+        return { level: level.toUpperCase() }
       },
       bindings: () => {
+        if (env.SELF_HOSTED) {
+          // "service" is being injected in datadog using the pod names,
+          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
           return {
             service: env.SERVICE_NAME,
           }
+        } else {
+          return {}
+        }
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
-  const destinations: pino.DestinationStream[] = []
+  const destinations: pino.StreamEntry[] = []
 
-  if (env.isDev()) {
-    destinations.push(pinoPretty({ singleLine: true }))
+  destinations.push(
+    env.isDev()
+      ? {
+          stream: pinoPretty({ singleLine: true }),
+          level: level as pino.Level,
         }
+      : { stream: process.stdout, level: level as pino.Level }
+  )
 
   if (env.SELF_HOSTED) {
-    destinations.push(localFileDestination())
+    destinations.push({
+      stream: localFileDestination(),
+      level: level as pino.Level,
+    })
   }
 
   pinoInstance = destinations.length
@@ -93,7 +106,6 @@ if (!env.DISABLE_PINO_LOGGER) {
 
     let contextObject = {}
 
-    if (LOG_CONTEXT) {
     contextObject = {
       tenantId: getTenantId(),
       appId: getAppId(),
@@ -102,7 +114,6 @@ if (!env.DISABLE_PINO_LOGGER) {
       identityType: identity?.type,
       correlationId: correlation.getId(),
     }
-    }
 
     const mergingObject: any = {
       err: error,

packages/backend-core/src/middleware/adminOnly.ts

@@ -1,10 +1,8 @@
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
+import { isAdmin } from "../users"
 
-export default async (ctx: BBContext, next: any) => {
-  if (
-    !ctx.internal &&
-    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
-  ) {
+export default async (ctx: UserCtx, next: any) => {
+  if (!ctx.internal && !isAdmin(ctx.user)) {
     ctx.throw(403, "Admin user only endpoint.")
   }
   return next()

packages/backend-core/src/middleware/builderOnly.ts

@@ -1,10 +1,19 @@
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
+import { isBuilder, hasBuilderPermissions } from "../users"
+import { getAppId } from "../context"
+import env from "../environment"
 
-export default async (ctx: BBContext, next: any) => {
-  if (
-    !ctx.internal &&
-    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
-  ) {
+export default async (ctx: UserCtx, next: any) => {
+  const appId = getAppId()
+  const builderFn = env.isWorker()
+    ? hasBuilderPermissions
+    : env.isApps()
+    ? isBuilder
+    : undefined
+  if (!builderFn) {
+    throw new Error("Service name unknown - middleware inactive.")
+  }
+  if (!ctx.internal && !builderFn(ctx.user, appId)) {
     ctx.throw(403, "Builder user only endpoint.")
   }
   return next()

packages/backend-core/src/middleware/builderOrAdmin.ts

@@ -1,12 +1,20 @@
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
+import { isBuilder, isAdmin, hasBuilderPermissions } from "../users"
+import { getAppId } from "../context"
+import env from "../environment"
 
-export default async (ctx: BBContext, next: any) => {
-  if (
-    !ctx.internal &&
-    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&
-    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
-  ) {
-    ctx.throw(403, "Builder user only endpoint.")
+export default async (ctx: UserCtx, next: any) => {
+  const appId = getAppId()
+  const builderFn = env.isWorker()
+    ? hasBuilderPermissions
+    : env.isApps()
+    ? isBuilder
+    : undefined
+  if (!builderFn) {
+    throw new Error("Service name unknown - middleware inactive.")
+  }
+  if (!ctx.internal && !builderFn(ctx.user, appId) && !isAdmin(ctx.user)) {
+    ctx.throw(403, "Admin/Builder user only endpoint.")
   }
   return next()
 }

packages/backend-core/src/middleware/tests/builder.spec.ts

@@ -0,0 +1,180 @@
+import adminOnly from "../adminOnly"
+import builderOnly from "../builderOnly"
+import builderOrAdmin from "../builderOrAdmin"
+import { structures } from "../../../tests"
+import { ContextUser, ServiceType } from "@budibase/types"
+import { doInAppContext } from "../../context"
+import env from "../../environment"
+env._set("SERVICE_TYPE", ServiceType.APPS)
+
+const appId = "app_aaa"
+const basicUser = structures.users.user()
+const adminUser = structures.users.adminUser()
+const adminOnlyUser = structures.users.adminOnlyUser()
+const builderUser = structures.users.builderUser()
+const appBuilderUser = structures.users.appBuilderUser(appId)
+
+function buildUserCtx(user: ContextUser) {
+  return {
+    internal: false,
+    user,
+    throw: jest.fn(),
+  } as any
+}
+
+function passed(throwFn: jest.Func, nextFn: jest.Func) {
+  expect(throwFn).not.toBeCalled()
+  expect(nextFn).toBeCalled()
+}
+
+function threw(throwFn: jest.Func) {
+  // cant check next, the throw function doesn't actually throw - so it still continues
+  expect(throwFn).toBeCalled()
+}
+
+describe("adminOnly middleware", () => {
+  it("should allow admin user", () => {
+    const ctx = buildUserCtx(adminUser),
+      next = jest.fn()
+    adminOnly(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should not allow basic user", () => {
+    const ctx = buildUserCtx(basicUser),
+      next = jest.fn()
+    adminOnly(ctx, next)
+    threw(ctx.throw)
+  })
+
+  it("should not allow builder user", () => {
+    const ctx = buildUserCtx(builderUser),
+      next = jest.fn()
+    adminOnly(ctx, next)
+    threw(ctx.throw)
+  })
+})
+
+describe("builderOnly middleware", () => {
+  it("should allow builder user", () => {
+    const ctx = buildUserCtx(builderUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow app builder user", () => {
+    const ctx = buildUserCtx(appBuilderUser),
+      next = jest.fn()
+    doInAppContext(appId, () => {
+      builderOnly(ctx, next)
+    })
+    passed(ctx.throw, next)
+  })
+
+  it("should allow admin and builder user", () => {
+    const ctx = buildUserCtx(adminUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should not allow admin user", () => {
+    const ctx = buildUserCtx(adminOnlyUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    threw(ctx.throw)
+  })
+
+  it("should not allow app builder user to different app", () => {
+    const ctx = buildUserCtx(appBuilderUser),
+      next = jest.fn()
+    doInAppContext("app_bbb", () => {
+      builderOnly(ctx, next)
+    })
+    threw(ctx.throw)
+  })
+
+  it("should not allow basic user", () => {
+    const ctx = buildUserCtx(basicUser),
+      next = jest.fn()
+    builderOnly(ctx, next)
+    threw(ctx.throw)
+  })
+})
+
+describe("builderOrAdmin middleware", () => {
+  it("should allow builder user", () => {
+    const ctx = buildUserCtx(builderUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow builder and admin user", () => {
+    const ctx = buildUserCtx(adminUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow admin user", () => {
+    const ctx = buildUserCtx(adminOnlyUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    passed(ctx.throw, next)
+  })
+
+  it("should allow app builder user", () => {
+    const ctx = buildUserCtx(appBuilderUser),
+      next = jest.fn()
+    doInAppContext(appId, () => {
+      builderOrAdmin(ctx, next)
+    })
+    passed(ctx.throw, next)
+  })
+
+  it("should not allow basic user", () => {
+    const ctx = buildUserCtx(basicUser),
+      next = jest.fn()
+    builderOrAdmin(ctx, next)
+    threw(ctx.throw)
+  })
+})
+
+describe("check service difference", () => {
+  it("should not allow without app ID in apps", () => {
+    env._set("SERVICE_TYPE", ServiceType.APPS)
+    const appId = "app_a"
+    const ctx = buildUserCtx({
+      ...basicUser,
+      builder: {
+        apps: [appId],
+      },
+    })
+    const next = jest.fn()
+    doInAppContext(appId, () => {
+      builderOnly(ctx, next)
+    })
+    passed(ctx.throw, next)
+    doInAppContext("app_b", () => {
+      builderOnly(ctx, next)
+    })
+    threw(ctx.throw)
+  })
+
+  it("should allow without app ID in worker", () => {
+    env._set("SERVICE_TYPE", ServiceType.WORKER)
+    const ctx = buildUserCtx({
+      ...basicUser,
+      builder: {
+        apps: ["app_a"],
+      },
+    })
+    const next = jest.fn()
+    doInAppContext("app_b", () => {
+      builderOnly(ctx, next)
+    })
+    passed(ctx.throw, next)
+  })
+})

packages/backend-core/src/security/permissions.ts

@@ -1,29 +1,12 @@
-const { flatten } = require("lodash")
-const { cloneDeep } = require("lodash/fp")
+import { PermissionType, PermissionLevel } from "@budibase/types"
+export { PermissionType, PermissionLevel } from "@budibase/types"
+import flatten from "lodash/flatten"
+import cloneDeep from "lodash/fp/cloneDeep"
 
 export type RoleHierarchy = {
   permissionId: string
 }[]
 
-export enum PermissionLevel {
-  READ = "read",
-  WRITE = "write",
-  EXECUTE = "execute",
-  ADMIN = "admin",
-}
-
-// these are the global types, that govern the underlying default behaviour
-export enum PermissionType {
-  APP = "app",
-  TABLE = "table",
-  USER = "user",
-  AUTOMATION = "automation",
-  WEBHOOK = "webhook",
-  BUILDER = "builder",
-  VIEW = "view",
-  QUERY = "query",
-}
-
 export class Permission {
   type: PermissionType
   level: PermissionLevel
@@ -95,7 +78,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -104,7 +86,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
     ],
   },
@@ -115,7 +96,6 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.USER, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
     ],
   },
@@ -126,7 +106,6 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
       new Permission(PermissionType.USER, PermissionLevel.ADMIN),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
-      new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
     ],
@@ -173,3 +152,4 @@ export function isPermissionLevelHigherThanRead(level: PermissionLevel) {
 
 // utility as a lot of things need simply the builder permission
 export const BUILDER = PermissionType.BUILDER
+export const GLOBAL_BUILDER = PermissionType.GLOBAL_BUILDER

packages/backend-core/src/security/roles.ts

@@ -3,7 +3,7 @@ import { prefixRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db"
 import { getAppDB } from "../context"
 import { doWithDB } from "../db"
 import { Screen, Role as RoleDoc } from "@budibase/types"
-const { cloneDeep } = require("lodash/fp")
+import cloneDeep from "lodash/fp/cloneDeep"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",

packages/backend-core/src/security/tests/permissions.spec.ts

@@ -1,4 +1,4 @@
-import { cloneDeep } from "lodash"
+import cloneDeep from "lodash/cloneDeep"
 import * as permissions from "../permissions"
 import { BUILTIN_ROLE_IDS } from "../roles"
 

packages/backend-core/src/users/db.ts

@@ -0,0 +1,468 @@
+import env from "../environment"
+import * as eventHelpers from "./events"
+import * as accounts from "../accounts"
+import * as accountSdk from "../accounts"
+import * as cache from "../cache"
+import { getGlobalDB, getIdentity, getTenantId } from "../context"
+import * as dbUtils from "../db"
+import { EmailUnavailableError, HTTPError } from "../errors"
+import * as platform from "../platform"
+import * as sessions from "../security/sessions"
+import * as usersCore from "./users"
+import {
+  Account,
+  AllDocsResponse,
+  BulkUserCreated,
+  BulkUserDeleted,
+  isSSOAccount,
+  isSSOUser,
+  RowResponse,
+  SaveUserOpts,
+  User,
+  UserStatus,
+} from "@budibase/types"
+import {
+  getAccountHolderFromUserIds,
+  isAdmin,
+  validateUniqueUser,
+} from "./utils"
+import { searchExistingEmails } from "./lookup"
+import { hash } from "../utils"
+
+type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
+type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
+type FeatureFn = () => Promise<Boolean>
+type QuotaFns = { addUsers: QuotaUpdateFn; removeUsers: QuotaUpdateFn }
+type GroupFns = { addUsers: GroupUpdateFn }
+type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
+
+const bulkDeleteProcessing = async (dbUser: User) => {
+  const userId = dbUser._id as string
+  await platform.users.removeUser(dbUser)
+  await eventHelpers.handleDeleteEvents(dbUser)
+  await cache.user.invalidateUser(userId)
+  await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
+}
+
+export class UserDB {
+  static quotas: QuotaFns
+  static groups: GroupFns
+  static features: FeatureFns
+
+  static init(quotaFns: QuotaFns, groupFns: GroupFns, featureFns: FeatureFns) {
+    UserDB.quotas = quotaFns
+    UserDB.groups = groupFns
+    UserDB.features = featureFns
+  }
+
+  static async isPreventPasswordActions(user: User, account?: Account) {
+    // when in maintenance mode we allow sso users with the admin role
+    // to perform any password action - this prevents lockout
+    if (env.ENABLE_SSO_MAINTENANCE_MODE && isAdmin(user)) {
+      return false
+    }
+
+    // SSO is enforced for all users
+    if (await UserDB.features.isSSOEnforced()) {
+      return true
+    }
+
+    // Check local sso
+    if (isSSOUser(user)) {
+      return true
+    }
+
+    // Check account sso
+    if (!account) {
+      account = await accountSdk.getAccountByTenantId(getTenantId())
+    }
+    return !!(account && account.email === user.email && isSSOAccount(account))
+  }
+
+  static async buildUser(
+    user: User,
+    opts: SaveUserOpts = {
+      hashPassword: true,
+      requirePassword: true,
+    },
+    tenantId: string,
+    dbUser?: any,
+    account?: Account
+  ): Promise<User> {
+    let { password, _id } = user
+
+    // don't require a password if the db user doesn't already have one
+    if (dbUser && !dbUser.password) {
+      opts.requirePassword = false
+    }
+
+    let hashedPassword
+    if (password) {
+      if (await UserDB.isPreventPasswordActions(user, account)) {
+        throw new HTTPError("Password change is disabled for this user", 400)
+      }
+      hashedPassword = opts.hashPassword ? await hash(password) : password
+    } else if (dbUser) {
+      hashedPassword = dbUser.password
+    }
+
+    // passwords are never required if sso is enforced
+    const requirePasswords =
+      opts.requirePassword && !(await UserDB.features.isSSOEnforced())
+    if (!hashedPassword && requirePasswords) {
+      throw "Password must be specified."
+    }
+
+    _id = _id || dbUtils.generateGlobalUserID()
+
+    const fullUser = {
+      createdAt: Date.now(),
+      ...dbUser,
+      ...user,
+      _id,
+      password: hashedPassword,
+      tenantId,
+    }
+    // make sure the roles object is always present
+    if (!fullUser.roles) {
+      fullUser.roles = {}
+    }
+    // add the active status to a user if its not provided
+    if (fullUser.status == null) {
+      fullUser.status = UserStatus.ACTIVE
+    }
+
+    return fullUser
+  }
+
+  static async allUsers() {
+    const db = getGlobalDB()
+    const response = await db.allDocs(
+      dbUtils.getGlobalUserParams(null, {
+        include_docs: true,
+      })
+    )
+    return response.rows.map((row: any) => row.doc)
+  }
+
+  static async countUsersByApp(appId: string) {
+    let response: any = await usersCore.searchGlobalUsersByApp(appId, {})
+    return {
+      userCount: response.length,
+    }
+  }
+
+  static async getUsersByAppAccess(appId?: string) {
+    const opts: any = {
+      include_docs: true,
+      limit: 50,
+    }
+    let response: User[] = await usersCore.searchGlobalUsersByAppAccess(
+      appId,
+      opts
+    )
+    return response
+  }
+
+  static async getUserByEmail(email: string) {
+    return usersCore.getGlobalUserByEmail(email)
+  }
+
+  /**
+   * Gets a user by ID from the global database, based on the current tenancy.
+   */
+  static async getUser(userId: string) {
+    const user = await usersCore.getById(userId)
+    if (user) {
+      delete user.password
+    }
+    return user
+  }
+
+  static async bulkGet(userIds: string[]) {
+    return await usersCore.bulkGetGlobalUsersById(userIds)
+  }
+
+  static async bulkUpdate(users: User[]) {
+    return await usersCore.bulkUpdateGlobalUsers(users)
+  }
+
+  static async save(user: User, opts: SaveUserOpts = {}): Promise<User> {
+    // default booleans to true
+    if (opts.hashPassword == null) {
+      opts.hashPassword = true
+    }
+    if (opts.requirePassword == null) {
+      opts.requirePassword = true
+    }
+    const tenantId = getTenantId()
+    const db = getGlobalDB()
+
+    let { email, _id, userGroups = [], roles } = user
+
+    if (!email && !_id) {
+      throw new Error("_id or email is required")
+    }
+
+    if (
+      user.builder?.apps?.length &&
+      !(await UserDB.features.isAppBuildersEnabled())
+    ) {
+      throw new Error("Unable to update app builders, please check license")
+    }
+
+    let dbUser: User | undefined
+    if (_id) {
+      // try to get existing user from db
+      try {
+        dbUser = (await db.get(_id)) as User
+        if (email && dbUser.email !== email) {
+          throw "Email address cannot be changed"
+        }
+        email = dbUser.email
+      } catch (e: any) {
+        if (e.status === 404) {
+          // do nothing, save this new user with the id specified - required for SSO auth
+        } else {
+          throw e
+        }
+      }
+    }
+
+    if (!dbUser && email) {
+      // no id was specified - load from email instead
+      dbUser = await usersCore.getGlobalUserByEmail(email)
+      if (dbUser && dbUser._id !== _id) {
+        throw new EmailUnavailableError(email)
+      }
+    }
+
+    const change = dbUser ? 0 : 1 // no change if there is existing user
+    return UserDB.quotas.addUsers(change, async () => {
+      await validateUniqueUser(email, tenantId)
+
+      let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
+      // don't allow a user to update its own roles/perms
+      if (opts.currentUserId && opts.currentUserId === dbUser?._id) {
+        builtUser = usersCore.cleanseUserObject(builtUser, dbUser) as User
+      }
+
+      if (!dbUser && roles?.length) {
+        builtUser.roles = { ...roles }
+      }
+
+      // make sure we set the _id field for a new user
+      // Also if this is a new user, associate groups with them
+      let groupPromises = []
+      if (!_id) {
+        _id = builtUser._id!
+
+        if (userGroups.length > 0) {
+          for (let groupId of userGroups) {
+            groupPromises.push(UserDB.groups.addUsers(groupId, [_id!]))
+          }
+        }
+      }
+
+      try {
+        // save the user to db
+        let response = await db.put(builtUser)
+        builtUser._rev = response.rev
+
+        await eventHelpers.handleSaveEvents(builtUser, dbUser)
+        await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+        await cache.user.invalidateUser(response.id)
+
+        await Promise.all(groupPromises)
+
+        // finally returned the saved user from the db
+        return db.get(builtUser._id!)
+      } catch (err: any) {
+        if (err.status === 409) {
+          throw "User exists already"
+        } else {
+          throw err
+        }
+      }
+    })
+  }
+
+  static async bulkCreate(
+    newUsersRequested: User[],
+    groups: string[]
+  ): Promise<BulkUserCreated> {
+    const tenantId = getTenantId()
+
+    let usersToSave: any[] = []
+    let newUsers: any[] = []
+
+    const emails = newUsersRequested.map((user: User) => user.email)
+    const existingEmails = await searchExistingEmails(emails)
+    const unsuccessful: { email: string; reason: string }[] = []
+
+    for (const newUser of newUsersRequested) {
+      if (
+        newUsers.find(
+          (x: User) => x.email.toLowerCase() === newUser.email.toLowerCase()
+        ) ||
+        existingEmails.includes(newUser.email.toLowerCase())
+      ) {
+        unsuccessful.push({
+          email: newUser.email,
+          reason: `Unavailable`,
+        })
+        continue
+      }
+      newUser.userGroups = groups
+      newUsers.push(newUser)
+    }
+
+    const account = await accountSdk.getAccountByTenantId(tenantId)
+    return UserDB.quotas.addUsers(newUsers.length, async () => {
+      // create the promises array that will be called by bulkDocs
+      newUsers.forEach((user: any) => {
+        usersToSave.push(
+          UserDB.buildUser(
+            user,
+            {
+              hashPassword: true,
+              requirePassword: user.requirePassword,
+            },
+            tenantId,
+            undefined, // no dbUser
+            account
+          )
+        )
+      })
+
+      const usersToBulkSave = await Promise.all(usersToSave)
+      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+
+      // Post-processing of bulk added users, e.g. events and cache operations
+      for (const user of usersToBulkSave) {
+        // TODO: Refactor to bulk insert users into the info db
+        // instead of relying on looping tenant creation
+        await platform.users.addUser(tenantId, user._id, user.email)
+        await eventHelpers.handleSaveEvents(user, undefined)
+      }
+
+      const saved = usersToBulkSave.map(user => {
+        return {
+          _id: user._id,
+          email: user.email,
+        }
+      })
+
+      // now update the groups
+      if (Array.isArray(saved) && groups) {
+        const groupPromises = []
+        const createdUserIds = saved.map(user => user._id)
+        for (let groupId of groups) {
+          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+        }
+        await Promise.all(groupPromises)
+      }
+
+      return {
+        successful: saved,
+        unsuccessful,
+      }
+    })
+  }
+
+  static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
+    const db = getGlobalDB()
+
+    const response: BulkUserDeleted = {
+      successful: [],
+      unsuccessful: [],
+    }
+
+    // remove the account holder from the delete request if present
+    const account = await getAccountHolderFromUserIds(userIds)
+    if (account) {
+      userIds = userIds.filter(u => u !== account.budibaseUserId)
+      // mark user as unsuccessful
+      response.unsuccessful.push({
+        _id: account.budibaseUserId,
+        email: account.email,
+        reason: "Account holder cannot be deleted",
+      })
+    }
+
+    // Get users and delete
+    const allDocsResponse: AllDocsResponse<User> = await db.allDocs({
+      include_docs: true,
+      keys: userIds,
+    })
+    const usersToDelete: User[] = allDocsResponse.rows.map(
+      (user: RowResponse<User>) => {
+        return user.doc
+      }
+    )
+
+    // Delete from DB
+    const toDelete = usersToDelete.map(user => ({
+      ...user,
+      _deleted: true,
+    }))
+    const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+
+    await UserDB.quotas.removeUsers(toDelete.length)
+    for (let user of usersToDelete) {
+      await bulkDeleteProcessing(user)
+    }
+
+    // Build Response
+    // index users by id
+    const userIndex: { [key: string]: User } = {}
+    usersToDelete.reduce((prev, current) => {
+      prev[current._id!] = current
+      return prev
+    }, userIndex)
+
+    // add the successful and unsuccessful users to response
+    dbResponse.forEach(item => {
+      const email = userIndex[item.id].email
+      if (item.ok) {
+        response.successful.push({ _id: item.id, email })
+      } else {
+        response.unsuccessful.push({
+          _id: item.id,
+          email,
+          reason: "Database error",
+        })
+      }
+    })
+
+    return response
+  }
+
+  static async destroy(id: string) {
+    const db = getGlobalDB()
+    const dbUser = (await db.get(id)) as User
+    const userId = dbUser._id as string
+
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+      // root account holder can't be deleted from inside budibase
+      const email = dbUser.email
+      const account = await accounts.getAccount(email)
+      if (account) {
+        if (dbUser.userId === getIdentity()!._id) {
+          throw new HTTPError('Please visit "Account" to delete this user', 400)
+        } else {
+          throw new HTTPError("Account holder cannot be deleted", 400)
+        }
+      }
+    }
+
+    await platform.users.removeUser(dbUser)
+
+    await db.remove(userId, dbUser._rev)
+
+    await UserDB.quotas.removeUsers(1)
+    await eventHelpers.handleDeleteEvents(dbUser)
+    await cache.user.invalidateUser(userId)
+    await sessions.invalidateSessions(userId, { reason: "deletion" })
+  }
+}

packages/backend-core/src/users/events.ts

@@ -1,15 +1,18 @@
-import env from "../../environment"
-import { events, accounts, tenancy } from "@budibase/backend-core"
+import env from "../environment"
+import * as events from "../events"
+import * as accounts from "../accounts"
+import { getTenantId } from "../context"
 import { User, UserRoles, CloudAccount } from "@budibase/types"
+import { hasBuilderPermissions, hasAdminPermissions } from "./utils"
 
 export const handleDeleteEvents = async (user: any) => {
   await events.user.deleted(user)
 
-  if (isBuilder(user)) {
+  if (hasBuilderPermissions(user)) {
     await events.user.permissionBuilderRemoved(user)
   }
 
-  if (isAdmin(user)) {
+  if (hasAdminPermissions(user)) {
     await events.user.permissionAdminRemoved(user)
   }
 }
@@ -55,7 +58,7 @@ export const handleSaveEvents = async (
   user: User,
   existingUser: User | undefined
 ) => {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = getTenantId()
   let tenantAccount: CloudAccount | undefined
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     tenantAccount = await accounts.getAccountByTenantId(tenantId)
@@ -103,23 +106,20 @@ export const handleSaveEvents = async (
   await handleAppRoleEvents(user, existingUser)
 }
 
-const isBuilder = (user: any) => user.builder && user.builder.global
-const isAdmin = (user: any) => user.admin && user.admin.global
-
 export const isAddingBuilder = (user: any, existingUser: any) => {
-  return isAddingPermission(user, existingUser, isBuilder)
+  return isAddingPermission(user, existingUser, hasBuilderPermissions)
 }
 
 export const isRemovingBuilder = (user: any, existingUser: any) => {
-  return isRemovingPermission(user, existingUser, isBuilder)
+  return isRemovingPermission(user, existingUser, hasBuilderPermissions)
 }
 
 const isAddingAdmin = (user: any, existingUser: any) => {
-  return isAddingPermission(user, existingUser, isAdmin)
+  return isAddingPermission(user, existingUser, hasAdminPermissions)
 }
 
 const isRemovingAdmin = (user: any, existingUser: any) => {
-  return isRemovingPermission(user, existingUser, isAdmin)
+  return isRemovingPermission(user, existingUser, hasAdminPermissions)
 }
 
 const isOnboardingComplete = (user: any, existingUser: any) => {

packages/backend-core/src/users/index.ts

@@ -0,0 +1,4 @@
+export * from "./users"
+export * from "./utils"
+export * from "./lookup"
+export { UserDB } from "./db"

packages/backend-core/src/users/lookup.ts

@@ -0,0 +1,102 @@
+import {
+  AccountMetadata,
+  PlatformUser,
+  PlatformUserByEmail,
+  User,
+} from "@budibase/types"
+import * as dbUtils from "../db"
+import { ViewName } from "../constants"
+
+/**
+ * Apply a system-wide search on emails:
+ * - in tenant
+ * - cross tenant
+ * - accounts
+ * return an array of emails that match the supplied emails.
+ */
+export async function searchExistingEmails(emails: string[]) {
+  let matchedEmails: string[] = []
+
+  const existingTenantUsers = await getExistingTenantUsers(emails)
+  matchedEmails.push(...existingTenantUsers.map(user => user.email))
+
+  const existingPlatformUsers = await getExistingPlatformUsers(emails)
+  matchedEmails.push(...existingPlatformUsers.map(user => user._id!))
+
+  const existingAccounts = await getExistingAccounts(emails)
+  matchedEmails.push(...existingAccounts.map(account => account.email))
+
+  return [...new Set(matchedEmails.map(email => email.toLowerCase()))]
+}
+
+// lookup, could be email or userId, either will return a doc
+export async function getPlatformUser(
+  identifier: string
+): Promise<PlatformUser | null> {
+  // use the view here and allow to find anyone regardless of casing
+  // Use lowercase to ensure email login is case insensitive
+  return (await dbUtils.queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, {
+    keys: [identifier.toLowerCase()],
+    include_docs: true,
+  })) as PlatformUser
+}
+
+export async function getExistingTenantUsers(
+  emails: string[]
+): Promise<User[]> {
+  const lcEmails = emails.map(email => email.toLowerCase())
+  const params = {
+    keys: lcEmails,
+    include_docs: true,
+  }
+
+  const opts = {
+    arrayResponse: true,
+  }
+
+  return (await dbUtils.queryGlobalView(
+    ViewName.USER_BY_EMAIL,
+    params,
+    undefined,
+    opts
+  )) as User[]
+}
+
+export async function getExistingPlatformUsers(
+  emails: string[]
+): Promise<PlatformUserByEmail[]> {
+  const lcEmails = emails.map(email => email.toLowerCase())
+  const params = {
+    keys: lcEmails,
+    include_docs: true,
+  }
+
+  const opts = {
+    arrayResponse: true,
+  }
+  return (await dbUtils.queryPlatformView(
+    ViewName.PLATFORM_USERS_LOWERCASE,
+    params,
+    opts
+  )) as PlatformUserByEmail[]
+}
+
+export async function getExistingAccounts(
+  emails: string[]
+): Promise<AccountMetadata[]> {
+  const lcEmails = emails.map(email => email.toLowerCase())
+  const params = {
+    keys: lcEmails,
+    include_docs: true,
+  }
+
+  const opts = {
+    arrayResponse: true,
+  }
+
+  return (await dbUtils.queryPlatformView(
+    ViewName.ACCOUNT_BY_EMAIL,
+    params,
+    opts
+  )) as AccountMetadata[]
+}

packages/backend-core/src/users/users.ts

@@ -11,10 +11,16 @@ import {
   SEPARATOR,
   UNICODE_MAX,
   ViewName,
-} from "./db"
-import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types"
-import { getGlobalDB } from "./context"
-import * as context from "./context"
+} from "../db"
+import {
+  BulkDocsResponse,
+  SearchUsersRequest,
+  User,
+  ContextUser,
+} from "@budibase/types"
+import { getGlobalDB } from "../context"
+import * as context from "../context"
+import { user as userCache } from "../cache"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -178,7 +184,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
  * Performs a starts with search on the global email view.
  */
 export const searchGlobalUsersByEmail = async (
-  email: string,
+  email: string | unknown,
   opts: any,
   getOpts?: GetOpts
 ) => {
@@ -248,3 +254,23 @@ export async function getUserCount() {
   })
   return response.total_rows
 }
+
+// used to remove the builder/admin permissions, for processing the
+// user as an app user (they may have some specific role/group
+export function removePortalUserPermissions(user: User | ContextUser) {
+  delete user.admin
+  delete user.builder
+  return user
+}
+
+export function cleanseUserObject(user: User | ContextUser, base?: User) {
+  delete user.admin
+  delete user.builder
+  delete user.roles
+  if (base) {
+    user.admin = base.admin
+    user.builder = base.builder
+    user.roles = base.roles
+  }
+  return user
+}

packages/backend-core/src/users/utils.ts

@@ -0,0 +1,55 @@
+import { CloudAccount } from "@budibase/types"
+import * as accountSdk from "../accounts"
+import env from "../environment"
+import { getPlatformUser } from "./lookup"
+import { EmailUnavailableError } from "../errors"
+import { getTenantId } from "../context"
+import { sdk } from "@budibase/shared-core"
+import { getAccountByTenantId } from "../accounts"
+
+// extract from shared-core to make easily accessible from backend-core
+export const isBuilder = sdk.users.isBuilder
+export const isAdmin = sdk.users.isAdmin
+export const isGlobalBuilder = sdk.users.isGlobalBuilder
+export const isAdminOrBuilder = sdk.users.isAdminOrBuilder
+export const hasAdminPermissions = sdk.users.hasAdminPermissions
+export const hasBuilderPermissions = sdk.users.hasBuilderPermissions
+export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions
+
+export async function validateUniqueUser(email: string, tenantId: string) {
+  // check budibase users in other tenants
+  if (env.MULTI_TENANCY) {
+    const tenantUser = await getPlatformUser(email)
+    if (tenantUser != null && tenantUser.tenantId !== tenantId) {
+      throw new EmailUnavailableError(email)
+    }
+  }
+
+  // check root account users in account portal
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    const account = await accountSdk.getAccount(email)
+    if (account && account.verified && account.tenantId !== tenantId) {
+      throw new EmailUnavailableError(email)
+    }
+  }
+}
+
+/**
+ * For the given user id's, return the account holder if it is in the ids.
+ */
+export async function getAccountHolderFromUserIds(
+  userIds: string[]
+): Promise<CloudAccount | undefined> {
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    const tenantId = getTenantId()
+    const account = await getAccountByTenantId(tenantId)
+    if (!account) {
+      throw new Error(`Account not found for tenantId=${tenantId}`)
+    }
+
+    const budibaseUserId = account.budibaseUserId
+    if (userIds.includes(budibaseUserId)) {
+      return account
+    }
+  }
+}

packages/backend-core/tests/core/utilities/jestUtils.ts

@@ -1,3 +1,5 @@
+import { db } from "../../../src"
+
 export function expectFunctionWasCalledTimesWith(
   jestFunction: any,
   times: number,
@@ -7,3 +9,22 @@ export function expectFunctionWasCalledTimesWith(
     jestFunction.mock.calls.filter((call: any) => call[0] === argument).length
   ).toBe(times)
 }
+
+export const expectAnyInternalColsAttributes: {
+  [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any
+} = {
+  tableId: expect.anything(),
+  type: expect.anything(),
+  _id: expect.anything(),
+  _rev: expect.anything(),
+  createdAt: expect.anything(),
+  updatedAt: expect.anything(),
+}
+
+export const expectAnyExternalColsAttributes: {
+  [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
+} = {
+  tableId: expect.anything(),
+  _id: expect.anything(),
+  _rev: expect.anything(),
+}

packages/backend-core/tests/core/utilities/mocks/events.ts

@@ -1,5 +1,3 @@
-import * as events from "../../../../src/events"
-
 beforeAll(async () => {
   const processors = await import("../../../../src/events/processors")
   const events = await import("../../../../src/events")

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -1,5 +1,5 @@
 import { Feature, License, Quotas } from "@budibase/types"
-import _ from "lodash"
+import cloneDeep from "lodash/cloneDeep"
 
 let CLOUD_FREE_LICENSE: License
 let UNLIMITED_LICENSE: License
@@ -58,7 +58,7 @@ export const useCloudFree = () => {
 // FEATURES
 
 const useFeature = (feature: Feature) => {
-  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  const license = cloneDeep(UNLIMITED_LICENSE)
   const opts: UseLicenseOpts = {
     features: [feature],
   }
@@ -86,6 +86,10 @@ export const useAuditLogs = () => {
   return useFeature(Feature.AUDIT_LOGS)
 }
 
+export const usePublicApiUserRoles = () => {
+  return useFeature(Feature.USER_ROLE_PUBLIC_API)
+}
+
 export const useScimIntegration = () => {
   return useFeature(Feature.SCIM)
 }
@@ -94,10 +98,14 @@ export const useSyncAutomations = () => {
   return useFeature(Feature.SYNC_AUTOMATIONS)
 }
 
+export const useAppBuilders = () => {
+  return useFeature(Feature.APP_BUILDERS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {
-  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  const license = cloneDeep(UNLIMITED_LICENSE)
   license.quotas.constant.automationLogRetentionDays.value = value
   return useLicense(license)
 }

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -11,9 +11,9 @@ import {
   CreateAccount,
   CreatePassswordAccount,
 } from "@budibase/types"
-import _ from "lodash"
+import sample from "lodash/sample"
 
-export const account = (): Account => {
+export const account = (partial: Partial<Account> = {}): Account => {
   return {
     accountId: uuid(),
     tenantId: generator.word(),
@@ -29,6 +29,7 @@ export const account = (): Account => {
     size: "10+",
     profession: "Software Engineer",
     quotaUsage: quotas.usage(),
+    ...partial,
   }
 }
 
@@ -45,13 +46,11 @@ export const cloudAccount = (): CloudAccount => {
 }
 
 function providerType(): AccountSSOProviderType {
-  return _.sample(
-    Object.values(AccountSSOProviderType)
-  ) as AccountSSOProviderType
+  return sample(Object.values(AccountSSOProviderType)) as AccountSSOProviderType
 }
 
 function provider(): AccountSSOProvider {
-  return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
+  return sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
 }
 
 export function ssoAccount(account: Account = cloudAccount()): SSOAccount {

packages/backend-core/tests/core/utilities/structures/db.ts

@@ -1,4 +1,4 @@
-import { structures } from ".."
+import { generator } from "./generator"
 import { newid } from "../../../../src/docIds/newid"
 
 export function id() {
@@ -6,7 +6,7 @@ export function id() {
 }
 
 export function rev() {
-  return `${structures.generator.character({
+  return `${generator.character({
     numeric: true,
-  })}-${structures.uuid().replace(/-/, "")}`
+  })}-${generator.guid().replace(/-/, "")}`
 }

packages/backend-core/tests/core/utilities/structures/documents/index.ts

@@ -0,0 +1 @@
+export * from "./platform"

packages/backend-core/tests/core/utilities/structures/documents/platform/index.ts

@@ -0,0 +1 @@
+export * as installation from "./installation"

packages/backend-core/tests/core/utilities/structures/documents/platform/installation.ts

@@ -0,0 +1,12 @@
+import { generator } from "../../generator"
+import { Installation } from "@budibase/types"
+import * as db from "../../db"
+
+export function install(): Installation {
+  return {
+    _id: "install",
+    _rev: db.rev(),
+    installId: generator.guid(),
+    version: generator.string(),
+  }
+}

packages/backend-core/tests/core/utilities/structures/index.ts

@@ -2,6 +2,7 @@ export * from "./common"
 export * as accounts from "./accounts"
 export * as apps from "./apps"
 export * as db from "./db"
+export * as docs from "./documents"
 export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -3,6 +3,8 @@ import {
   Customer,
   Feature,
   License,
+  OfflineIdentifier,
+  OfflineLicense,
   PlanModel,
   PlanType,
   PriceDuration,
@@ -11,6 +13,7 @@ import {
   Quotas,
   Subscription,
 } from "@budibase/types"
+import { generator } from "./generator"
 
 export function price(): PurchasedPrice {
   return {
@@ -127,15 +130,15 @@ export function subscription(): Subscription {
   }
 }
 
-export const license = (
-  opts: {
+interface GenerateLicenseOpts {
   quotas?: Quotas
   plan?: PurchasedPlan
   planType?: PlanType
   features?: Feature[]
   billing?: Billing
-  } = {}
-): License => {
+}
+
+export const license = (opts: GenerateLicenseOpts = {}): License => {
   return {
     features: opts.features || [],
     quotas: opts.quotas || quotas(),
@@ -143,3 +146,22 @@ export const license = (
     billing: opts.billing || billing(),
   }
 }
+
+export function offlineLicense(opts: GenerateLicenseOpts = {}): OfflineLicense {
+  const base = license(opts)
+  return {
+    ...base,
+    expireAt: new Date().toISOString(),
+    identifier: offlineIdentifier(),
+  }
+}
+
+export function offlineIdentifier(
+  installId: string = generator.guid(),
+  tenantId: string = generator.guid()
+): OfflineIdentifier {
+  return {
+    installId,
+    tenantId,
+  }
+}

packages/backend-core/tests/core/utilities/structures/scim.ts

@@ -1,7 +1,6 @@
 import { ScimCreateGroupRequest, ScimCreateUserRequest } from "@budibase/types"
 import { uuid } from "./common"
 import { generator } from "./generator"
-import _ from "lodash"
 
 interface CreateUserRequestFields {
   externalId: string
@@ -20,10 +19,10 @@ export function createUserRequest(userData?: Partial<CreateUserRequestFields>) {
     username: generator.name(),
   }
 
-  const { externalId, email, firstName, lastName, username } = _.assign(
-    defaultValues,
-    userData
-  )
+  const { externalId, email, firstName, lastName, username } = {
+    ...defaultValues,
+    ...userData,
+  }
 
   let user: ScimCreateUserRequest = {
     schemas: [

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -15,7 +15,7 @@ import { generator } from "./generator"
 import { email, uuid } from "./common"
 import * as shared from "./shared"
 import { user } from "./shared"
-import _ from "lodash"
+import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
   return {
@@ -47,7 +47,7 @@ export function authDetails(userDoc?: User): SSOAuthDetails {
 }
 
 export function providerType(): SSOProviderType {
-  return _.sample(Object.values(SSOProviderType)) as SSOProviderType
+  return sample(Object.values(SSOProviderType)) as SSOProviderType
 }
 
 export function ssoProfile(user?: User): SSOProfile {

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -1,5 +1,6 @@
 import {
   AdminUser,
+  AdminOnlyUser,
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
@@ -21,6 +22,15 @@ export const adminUser = (userProps?: any): AdminUser => {
   }
 }
 
+export const adminOnlyUser = (userProps?: any): AdminOnlyUser => {
+  return {
+    ...user(userProps),
+    admin: {
+      global: true,
+    },
+  }
+}
+
 export const builderUser = (userProps?: any): BuilderUser => {
   return {
     ...user(userProps),
@@ -30,6 +40,15 @@ export const builderUser = (userProps?: any): BuilderUser => {
   }
 }
 
+export const appBuilderUser = (appId: string, userProps?: any): BuilderUser => {
+  return {
+    ...user(userProps),
+    builder: {
+      apps: [appId],
+    },
+  }
+}
+
 export function ssoUser(
   opts: { user?: any; details?: SSOAuthDetails } = {}
 ): SSOUser {

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -32,8 +32,8 @@ function getTestContainerSettings(
 ): string | null {
   const entry = Object.entries(global).find(
     ([k]) =>
-      k.includes(`_${serverName.toUpperCase()}`) &&
-      k.includes(`_${key.toUpperCase()}__`)
+      k.includes(`${serverName.toUpperCase()}`) &&
+      k.includes(`${key.toUpperCase()}`)
   )
   if (!entry) {
     return null
@@ -67,27 +67,14 @@ function getContainerInfo(containerName: string, port: number) {
 }
 
 function getCouchConfig() {
-  return getContainerInfo("couchdb-service", 5984)
-}
-
-function getMinioConfig() {
-  return getContainerInfo("minio-service", 9000)
-}
-
-function getRedisConfig() {
-  return getContainerInfo("redis-service", 6379)
+  return getContainerInfo("couchdb", 5984)
 }
 
 export function setupEnv(...envs: any[]) {
-  const couch = getCouchConfig(),
-    minio = getCouchConfig(),
-    redis = getRedisConfig()
+  const couch = getCouchConfig()
   const configs = [
     { key: "COUCH_DB_PORT", value: couch.port },
     { key: "COUCH_DB_URL", value: couch.url },
-    { key: "MINIO_PORT", value: minio.port },
-    { key: "MINIO_URL", value: minio.url },
-    { key: "REDIS_URL", value: redis.url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tsconfig.build.json

@@ -12,7 +12,11 @@
     "declaration": true,
     "types": ["node", "jest"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/shared-core": ["../shared-core/src"]
+    }
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [

packages/backend-core/tsconfig.json

@@ -1,12 +1,4 @@
 {
   "extends": "./tsconfig.build.json",
-  "compilerOptions": {
-    "composite": true,
-    "baseUrl": ".",
-    "paths": {
-      "@budibase/types": ["../types/src"]
-    }
-  },
-
   "exclude": ["node_modules", "dist"]
 }

packages/bbui/package.json

@@ -85,7 +85,8 @@
     "dayjs": "^1.10.4",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
-    "svelte-portal": "^1.0.0"
+    "svelte-portal": "^1.0.0",
+    "svelte-dnd-action": "^0.9.8"
   },
   "resolutions": {
     "loader-utils": "1.4.1"
@@ -96,13 +97,13 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/string-templates"
+              "@budibase/string-templates",
+              "@budibase/shared-core"
             ],
             "target": "build"
           }
         ]
       }
     }
-  },
-  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
+  }
 }

packages/bbui/src/Actions/position_dropdown.js

@@ -63,7 +63,7 @@ export default function positionDropdown(element, opts) {
 
     // Apply styles
     Object.entries(styles).forEach(([style, value]) => {
-      if (value) {
+      if (value != null) {
         element.style[style] = `${value.toFixed(0)}px`
       } else {
         element.style[style] = null

packages/bbui/src/Drawer/Drawer.svelte

@@ -30,6 +30,7 @@
   setContext("drawer-actions", {
     hide,
     show,
+    headless,
   })
 
   const easeInOutQuad = x => {

packages/bbui/src/Form/Core/CheckboxGroup.svelte

@@ -12,23 +12,24 @@
   export let getOptionValue = option => option
 
   const dispatch = createEventDispatcher()
+
   const onChange = e => {
-    let tempValue = value
-    let isChecked = e.target.checked
-    if (!tempValue.includes(e.target.value) && isChecked) {
-      tempValue.push(e.target.value)
-    }
-    value = tempValue
+    const optionValue = e.target.value
+    if (e.target.checked && !value.includes(optionValue)) {
+      dispatch("change", [...value, optionValue])
+    } else {
       dispatch(
         "change",
-      tempValue.filter(val => val !== e.target.value || isChecked)
+        value.filter(x => x !== optionValue)
       )
     }
+  }
 </script>
 
 <div class={`spectrum-FieldGroup spectrum-FieldGroup--${direction}`}>
   {#if options && Array.isArray(options)}
     {#each options as option}
+      {@const optionValue = getOptionValue(option)}
       <div
         title={getOptionLabel(option)}
         class="spectrum-Checkbox spectrum-FieldGroup-item"
@@ -39,11 +40,11 @@
         >
           <input
             on:change={onChange}
-            value={getOptionValue(option)}
             type="checkbox"
             class="spectrum-Checkbox-input"
+            value={optionValue}
+            checked={value.includes(optionValue)}
             {disabled}
-            checked={value.includes(getOptionValue(option))}
           />
           <span class="spectrum-Checkbox-box">
             <svg

packages/bbui/src/Icon/Icon.svelte

@@ -47,7 +47,7 @@
   </svg>
   {#if tooltip && showTooltip}
     <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
-      <Tooltip textWrapping direction="bottom" text={tooltip} />
+      <Tooltip textWrapping direction="top" text={tooltip} />
     </div>
   {/if}
 </div>
@@ -80,15 +80,14 @@
     position: absolute;
     pointer-events: none;
     left: 50%;
-    top: calc(100% + 4px);
-    width: 100vw;
-    max-width: 150px;
+    bottom: calc(100% + 4px);
     transform: translateX(-50%);
     text-align: center;
+    z-index: 1;
   }
 
   .spectrum-Icon--sizeXS {
-    width: 10px;
-    height: 10px;
+    width: var(--spectrum-global-dimension-size-150);
+    height: var(--spectrum-global-dimension-size-150);
   }
 </style>

packages/bbui/src/IconPicker/IconPicker.svelte

@@ -1,5 +1,4 @@
 <script>
-  //import { createEventDispatcher } from "svelte"
   import "@spectrum-css/popover/dist/index-vars.css"
   import clickOutside from "../Actions/click_outside"
   import { fly } from "svelte/transition"

packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte

@@ -0,0 +1,252 @@
+<script>
+  import { flip } from "svelte/animate"
+  import { dndzone } from "svelte-dnd-action"
+  import Icon from "../Icon/Icon.svelte"
+  import Popover from "../Popover/Popover.svelte"
+  import { onMount } from "svelte"
+  const flipDurationMs = 150
+
+  export let constraints
+  export let optionColors = {}
+  let options = []
+
+  let colorPopovers = []
+  let anchors = []
+
+  let colorsArray = [
+    "hsla(0, 90%, 75%, 0.3)",
+    "hsla(50, 80%, 75%, 0.3)",
+    "hsla(120, 90%, 75%, 0.3)",
+    "hsla(200, 90%, 75%, 0.3)",
+    "hsla(240, 90%, 75%, 0.3)",
+    "hsla(320, 90%, 75%, 0.3)",
+  ]
+  $: {
+    if (constraints.inclusion.length) {
+      options = constraints.inclusion.map(value => ({
+        name: value,
+        id: Math.random(),
+      }))
+    }
+  }
+  const removeInput = idx => {
+    delete optionColors[options[idx].name]
+    constraints.inclusion = constraints.inclusion.filter((e, i) => i !== idx)
+    options = options.filter((e, i) => i !== idx)
+    colorPopovers.pop(undefined)
+    anchors.pop(undefined)
+  }
+
+  const addNewInput = () => {
+    options = [
+      ...options,
+      { name: `Option ${constraints.inclusion.length + 1}`, id: Math.random() },
+    ]
+    constraints.inclusion = [
+      ...constraints.inclusion,
+      `Option ${constraints.inclusion.length + 1}`,
+    ]
+
+    colorPopovers.push(undefined)
+    anchors.push(undefined)
+  }
+
+  const handleDndConsider = e => {
+    options = e.detail.items
+  }
+  const handleDndFinalize = e => {
+    options = e.detail.items
+    constraints.inclusion = options.map(option => option.name)
+  }
+
+  const handleColorChange = (optionName, color, idx) => {
+    optionColors[optionName] = color
+    colorPopovers[idx].hide()
+  }
+
+  const handleNameChange = (optionName, idx, value) => {
+    constraints.inclusion[idx] = value
+    options[idx].name = value
+    optionColors[value] = optionColors[optionName]
+    delete optionColors[optionName]
+  }
+
+  const openColorPickerPopover = (optionIdx, target) => {
+    colorPopovers[optionIdx].show()
+    anchors[optionIdx] = target
+  }
+
+  onMount(() => {
+    // Initialize anchor arrays on mount, assuming 'options' is already populated
+    colorPopovers = constraints.inclusion.map(() => undefined)
+    anchors = constraints.inclusion.map(() => undefined)
+  })
+</script>
+
+<div>
+  <div
+    class="actions"
+    use:dndzone={{
+      items: options,
+      flipDurationMs,
+      dropTargetStyle: { outline: "none" },
+    }}
+    on:consider={handleDndConsider}
+    on:finalize={handleDndFinalize}
+  >
+    {#each options as option, idx (option.id)}
+      <div
+        class="no-border action-container"
+        animate:flip={{ duration: flipDurationMs }}
+      >
+        <div class="child drag-handle-spacing">
+          <Icon name="DragHandle" size="L" />
+        </div>
+        <div class="child color-picker">
+          <div
+            id="color-picker"
+            bind:this={anchors[idx]}
+            style="--color:{optionColors?.[option.name] ||
+              'hsla(0, 1%, 50%, 0.3)'}"
+            class="circle"
+            on:click={e => openColorPickerPopover(idx, e.target)}
+          >
+            <Popover
+              bind:this={colorPopovers[idx]}
+              anchor={anchors[idx]}
+              align="left"
+              offset={0}
+              style=""
+              popoverTarget={document.getElementById(`color-picker`)}
+              animate={false}
+            >
+              <div class="colors">
+                {#each colorsArray as color}
+                  <div
+                    on:click={() => handleColorChange(option.name, color, idx)}
+                    style="--color:{color};"
+                    class="circle circle-hover"
+                  />
+                {/each}
+              </div>
+            </Popover>
+          </div>
+        </div>
+        <div class="child">
+          <input
+            class="input-field"
+            type="text"
+            on:change={e => handleNameChange(option.name, idx, e.target.value)}
+            value={option.name}
+            placeholder="Option name"
+          />
+        </div>
+        <div class="child">
+          <Icon name="Close" hoverable size="S" on:click={removeInput(idx)} />
+        </div>
+      </div>
+    {/each}
+  </div>
+  <div on:click={addNewInput} class="add-option">
+    <Icon hoverable name="Add" />
+    <div>Add option</div>
+  </div>
+</div>
+
+<style>
+  .action-container {
+    background-color: var(--spectrum-alias-background-color-primary);
+    border-radius: 0px;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    transition: background-color 130ms ease-in-out, color 130ms ease-in-out,
+      border-color 130ms ease-in-out;
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+  }
+  .no-border {
+    border-bottom: none;
+  }
+
+  .action-container:last-child {
+    border-bottom: 1px solid var(--spectrum-global-color-gray-300) !important;
+  }
+
+  .child {
+    height: 30px;
+  }
+  .child:hover,
+  .child:focus {
+    background: var(--spectrum-global-color-gray-200);
+  }
+  .add-option {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    padding: var(--spacing-m);
+    gap: var(--spacing-m);
+    cursor: pointer;
+  }
+
+  .input-field {
+    border: none;
+    outline: none;
+    background-color: transparent;
+    width: 100%;
+    color: var(--text);
+  }
+
+  .child input[type="text"] {
+    padding-left: 10px;
+  }
+
+  .input-field:hover,
+  .input-field:focus {
+    background: var(--spectrum-global-color-gray-200);
+  }
+
+  .action-container > :nth-child(1) {
+    flex-grow: 1;
+    justify-content: center;
+    display: flex;
+  }
+
+  .action-container > :nth-child(2) {
+    flex-grow: 1;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+  }
+
+  .action-container > :nth-child(3) {
+    flex-grow: 4;
+    display: flex;
+  }
+  .action-container > :nth-child(4) {
+    flex-grow: 1;
+    justify-content: center;
+    display: flex;
+  }
+
+  .circle {
+    height: 20px;
+    width: 20px;
+    background-color: var(--color);
+    border-radius: 50%;
+    display: inline-block;
+    box-sizing: border-box;
+  }
+
+  .circle-hover:hover {
+    border: 1px solid var(--spectrum-global-color-blue-400);
+    cursor: pointer;
+  }
+
+  .colors {
+    display: grid;
+    grid-template-columns: 1fr 1fr 1fr;
+    gap: var(--spacing-xl);
+    justify-items: center;
+    margin: var(--spacing-m);
+  }
+</style>

packages/bbui/src/Popover/Popover.svelte

@@ -21,6 +21,7 @@
   export let offset = 5
   export let customHeight
   export let animate = true
+  export let customZindex
 
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
 
@@ -85,8 +86,9 @@
       }}
       on:keydown={handleEscape}
       class="spectrum-Popover is-open"
+      class:customZindex
       role="presentation"
-      style="height: {customHeight}"
+      style="height: {customHeight}; --customZindex: {customZindex};"
       transition:fly|local={{ y: -20, duration: animate ? 200 : 0 }}
     >
       <slot />
@@ -100,4 +102,8 @@
     border-color: var(--spectrum-global-color-gray-300);
     overflow: auto;
   }
+
+  .customZindex {
+    z-index: var(--customZindex) !important;
+  }
 </style>

packages/bbui/src/index.js

@@ -84,7 +84,7 @@ export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte
 export { default as Slider } from "./Form/Slider.svelte"
 export { default as Accordion } from "./Accordion/Accordion.svelte"
 export { default as File } from "./Form/File.svelte"
-
+export { default as OptionSelectDnD } from "./OptionSelectDnD/OptionSelectDnD.svelte"
 // Renderers
 export { default as BoldRenderer } from "./Table/BoldRenderer.svelte"
 export { default as CodeRenderer } from "./Table/CodeRenderer.svelte"

packages/builder/package.json

@@ -101,14 +101,14 @@
     "@rollup/plugin-replace": "^2.4.2",
     "@roxi/routify": "2.18.5",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
-    "@testing-library/jest-dom": "^5.11.10",
+    "@testing-library/jest-dom": "5.17.0",
     "@testing-library/svelte": "^3.2.2",
-    "babel-jest": "^26.6.3",
+    "babel-jest": "29.6.2",
     "cypress": "^9.3.1",
     "cypress-multi-reporters": "^1.6.0",
     "cypress-terminal-report": "^1.4.1",
     "identity-obj-proxy": "^3.0.0",
-    "jest": "^26.6.3",
+    "jest": "29.6.2",
     "jsdom": "^21.1.1",
     "mochawesome": "^7.1.3",
     "mochawesome-merge": "^4.2.1",
@@ -133,8 +133,17 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/string-templates",
-              "@budibase/shared-core"
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      },
+      "dev:builder": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates"
             ],
             "target": "build"
           }
@@ -144,7 +153,6 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
               "@budibase/string-templates"
             ],
             "target": "build"
@@ -152,6 +160,5 @@
         ]
       }
     }
-  },
-  "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
+  }
 }

packages/builder/src/builderStore/dataBinding.js

@@ -491,6 +491,7 @@ const getSelectedRowsBindings = asset => {
         readableBinding: `${table._instanceName}.Selected rows`,
         category: "Selected rows",
         icon: "ViewRow",
+        display: { name: table._instanceName },
       }))
     )
 
@@ -506,6 +507,7 @@ const getSelectedRowsBindings = asset => {
         )}.${makePropSafe("selectedRows")}`,
         readableBinding: `${block._instanceName}.Selected rows`,
         category: "Selected rows",
+        display: { name: block._instanceName },
       }))
     )
   }

packages/builder/src/builderStore/index.js

@@ -4,7 +4,7 @@ import { getTemporalStore } from "./store/temporal"
 import { getThemeStore } from "./store/theme"
 import { getUserStore } from "./store/users"
 import { getDeploymentStore } from "./store/deployments"
-import { derived } from "svelte/store"
+import { derived, writable } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
 import { createHistoryStore } from "builderStore/store/history"
@@ -61,6 +61,12 @@ export const selectedLayout = derived(store, $store => {
 export const selectedComponent = derived(
   [store, selectedScreen],
   ([$store, $selectedScreen]) => {
+    if (
+      $selectedScreen &&
+      $store.selectedComponentId?.startsWith(`${$selectedScreen._id}-`)
+    ) {
+      return $selectedScreen?.props
+    }
     if (!$selectedScreen || !$store.selectedComponentId) {
       return null
     }
@@ -141,3 +147,5 @@ export const userSelectedResourceMap = derived(userStore, $userStore => {
 export const isOnlyUser = derived(userStore, $userStore => {
   return $userStore.length < 2
 })
+
+export const screensHeight = writable("210px")

packages/builder/src/builderStore/store/frontend.js

@@ -225,7 +225,6 @@ export const getFrontendStore = () => {
         // Select new screen
         store.update(state => {
           state.selectedScreenId = screen._id
-          state.selectedComponentId = screen.props?._id
           return state
         })
       },
@@ -769,9 +768,13 @@ export const getFrontendStore = () => {
         else {
           await store.actions.screens.patch(screen => {
             // Find the selected component
+            let selectedComponentId = state.selectedComponentId
+            if (selectedComponentId.startsWith(`${screen._id}-`)) {
+              selectedComponentId = screen?.props._id
+            }
             const currentComponent = findComponent(
               screen.props,
-              state.selectedComponentId
+              selectedComponentId
             )
             if (!currentComponent) {
               return false
@@ -994,12 +997,20 @@ export const getFrontendStore = () => {
         const componentId = state.selectedComponentId
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
-
-        // Check we aren't right at the top of the tree
         const index = parent?._children.findIndex(x => x._id === componentId)
-        if (!parent || componentId === screen.props._id) {
+
+        // Check for screen and navigation component edge cases
+        const screenComponentId = `${screen._id}-screen`
+        const navComponentId = `${screen._id}-navigation`
+        if (componentId === screenComponentId) {
           return null
         }
+        if (componentId === navComponentId) {
+          return screenComponentId
+        }
+        if (parent._id === screen.props._id && index === 0) {
+          return navComponentId
+        }
 
         // If we have siblings above us, choose the sibling or a descendant
         if (index > 0) {
@@ -1021,12 +1032,20 @@ export const getFrontendStore = () => {
         return parent._id
       },
       getNext: () => {
+        const state = get(store)
         const component = get(selectedComponent)
         const componentId = component?._id
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
         const index = parent?._children.findIndex(x => x._id === componentId)
 
+        // Check for screen and navigation component edge cases
+        const screenComponentId = `${screen._id}-screen`
+        const navComponentId = `${screen._id}-navigation`
+        if (state.selectedComponentId === screenComponentId) {
+          return navComponentId
+        }
+
         // If we have children, select first child
         if (component._children?.length) {
           return component._children[0]._id

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -108,7 +108,10 @@
   /****************************************************/
 
   const getInputData = (testData, blockInputs) => {
-    let newInputData = testData || blockInputs
+    // Test data is not cloned for reactivity
+    let newInputData = testData || cloneDeep(blockInputs)
+
+    // Ensures the app action fields are populated
     if (block.event === "app:trigger" && !newInputData?.fields) {
       newInputData = cloneDeep(blockInputs)
     }

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -50,6 +50,7 @@
         type="string"
         {bindings}
         fillWidth={true}
+        updateOnChange={false}
       />
     {/each}
   </div>

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { datasources, tables } from "stores/backend"
+  import { datasources, tables, integrations } from "stores/backend"
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
@@ -27,6 +27,17 @@
   $: isUsersTable = id === TableNames.USERS
   $: isInternal = $tables.selected?.type !== "external"
 
+  $: datasource = $datasources.list.find(datasource => {
+    return datasource._id === $tables.selected?.sourceId
+  })
+
+  $: relationshipsEnabled = relationshipSupport(datasource)
+
+  const relationshipSupport = datasource => {
+    const integration = $integrations[datasource?.source]
+    return !isInternal && integration?.relationships !== false
+  }
+
   const handleGridTableUpdate = async e => {
     tables.replaceTable(id, e.detail)
 
@@ -53,12 +64,19 @@
     <svelte:fragment slot="filter">
       <GridFilterButton />
     </svelte:fragment>
+    <svelte:fragment slot="edit-column">
+      <GridEditColumnModal />
+    </svelte:fragment>
+    <svelte:fragment slot="add-column">
+      <GridAddColumnModal />
+    </svelte:fragment>
+
     <svelte:fragment slot="controls">
       {#if isInternal}
         <GridCreateViewButton />
       {/if}
       <GridManageAccessButton />
-      {#if !isInternal}
+      {#if relationshipsEnabled}
         <GridRelationshipButton />
       {/if}
       {#if isUsersTable}
@@ -66,9 +84,8 @@
       {:else}
         <GridImportButton />
       {/if}
+
       <GridExportButton />
-      <GridAddColumnModal />
-      <GridEditColumnModal />
       {#if isUsersTable}
         <GridEditUserModal />
       {:else}

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -109,6 +109,7 @@
         {disableSorting}
         {customPlaceholder}
         allowEditRows={allowEditing}
+        allowEditColumns={allowEditing}
         showAutoColumns={!hideAutocolumns}
         {allowClickRows}
         on:clickrelationship={e => selectRelationship(e.detail)}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -6,19 +6,21 @@
     Select,
     Toggle,
     RadioGroup,
+    Icon,
     DatePicker,
-    ModalContent,
-    Context,
     Modal,
     notifications,
+    OptionSelectDnD,
+    Layout,
+    AbsTooltip,
   } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
-    RelationshipTypes,
+    RelationshipType,
     ALLOWABLE_STRING_OPTIONS,
     ALLOWABLE_NUMBER_OPTIONS,
     ALLOWABLE_STRING_TYPES,
@@ -26,13 +28,12 @@
     SWITCHABLE_TYPES,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
-  import ValuesList from "components/common/ValuesList.svelte"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { truncate } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
-  import { getContext } from "svelte"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -44,11 +45,12 @@
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
-  const { hide } = getContext(Context.Modal)
-  let fieldDefinitions = cloneDeep(FIELDS)
+  const { dispatch: gridDispatch } = getContext("grid")
 
   export let field
 
+  let mounted = false
+  let fieldDefinitions = cloneDeep(FIELDS)
   let originalName
   let linkEditDisabled
   let primaryDisplay
@@ -60,11 +62,10 @@
   let savingColumn
   let deleteColName
   let jsonSchemaModal
-
+  let allowedTypes = []
   let editableColumn = {
     type: "string",
     constraints: fieldDefinitions.STRING.constraints,
-
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
@@ -82,7 +83,23 @@
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
+    } else if (!savingColumn) {
+      let highestNumber = 0
+      Object.keys(table.schema).forEach(columnName => {
+        const columnNumber = extractColumnNumber(columnName)
+        if (columnNumber > highestNumber) {
+          highestNumber = columnNumber
         }
+        return highestNumber
+      })
+
+      if (highestNumber >= 1) {
+        editableColumn.name = `Column 0${highestNumber + 1}`
+      } else {
+        editableColumn.name = "Column 01"
+      }
+    }
+    allowedTypes = getAllowedTypes()
   }
 
   $: initialiseField(field, savingColumn)
@@ -181,9 +198,11 @@
         indexes,
       })
       dispatch("updatecolumns")
+      gridDispatch("close-edit-column")
+
       if (
         saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipTypes.MANY_TO_MANY
+        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
       ) {
         // Fetching the new tables
         tables.fetch()
@@ -202,6 +221,7 @@
 
   function cancelEdit() {
     editableColumn.name = originalName
+    gridDispatch("close-edit-column")
   }
 
   async function deleteColumn() {
@@ -213,8 +233,8 @@
         await tables.deleteField(editableColumn)
         notifications.success(`Column ${editableColumn.name} deleted`)
         confirmDeleteDialog.hide()
-        hide()
         dispatch("updatecolumns")
+        gridDispatch("close-edit-column")
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
@@ -237,7 +257,7 @@
 
     // Default relationships many to many
     if (editableColumn.type === LINK_TYPE) {
-      editableColumn.relationshipType = RelationshipTypes.MANY_TO_MANY
+      editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     }
     if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
@@ -250,14 +270,6 @@
     required = req
   }
 
-  function onChangePrimaryDisplay(e) {
-    const isPrimary = e.detail
-    // primary display is always required
-    if (isPrimary) {
-      editableColumn.constraints.presence = { allowEmpty: false }
-    }
-  }
-
   function openJsonSchemaEditor() {
     jsonSchemaModal.show()
   }
@@ -271,6 +283,11 @@
     deleteColName = ""
   }
 
+  function extractColumnNumber(columnName) {
+    const match = columnName.match(/Column (\d+)/)
+    return match ? parseInt(match[1]) : 0
+  }
+
   function getRelationshipOptions(field) {
     if (!field || !field.tableId) {
       return null
@@ -285,17 +302,17 @@
       {
         name: `Many ${thisName} rows → many ${linkName} rows`,
         alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipTypes.MANY_TO_MANY,
+        value: RelationshipType.MANY_TO_MANY,
       },
       {
         name: `One ${linkName} row → many ${thisName} rows`,
         alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipTypes.ONE_TO_MANY,
+        value: RelationshipType.ONE_TO_MANY,
       },
       {
         name: `One ${thisName} row → many ${linkName} rows`,
         alt: `One ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipTypes.MANY_TO_ONE,
+        value: RelationshipType.MANY_TO_ONE,
       },
     ]
   }
@@ -375,7 +392,7 @@
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
-    } else if (fieldInfo.name && !fieldInfo.name.match(/^[_a-zA-Z0-9\s]*$/g)) {
+    } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
     } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(
@@ -399,31 +416,30 @@
     }
     return newError
   }
+
+  onMount(() => {
+    mounted = true
+  })
 </script>
 
-<ModalContent
-  title={originalName ? "Edit Column" : "Create Column"}
-  confirmText="Save Column"
-  onConfirm={saveColumn}
-  onCancel={cancelEdit}
-  disabled={invalid}
->
+<Layout noPadding gap="S">
+  {#if mounted}
     <Input
-    label="Name"
+      autofocus
       bind:value={editableColumn.name}
       disabled={uneditable ||
         (linkEditDisabled && editableColumn.type === LINK_TYPE)}
       error={errors?.name}
     />
-
+  {/if}
   <Select
     disabled={!typeEnabled}
-    label="Type"
     bind:value={editableColumn.type}
     on:change={handleTypeChange}
-    options={getAllowedTypes()}
+    options={allowedTypes}
     getOptionLabel={field => field.name}
     getOptionValue={field => field.type}
+    getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
       if (option.type == AUTO_TYPE) {
         return availableAutoColumnKeys?.length > 0
@@ -432,28 +448,6 @@
     }}
   />
 
-  {#if canBeRequired || canBeDisplay}
-    <div>
-      {#if canBeRequired}
-        <Toggle
-          value={required}
-          on:change={onChangeRequired}
-          disabled={primaryDisplay}
-          thin
-          text="Required"
-        />
-      {/if}
-      {#if canBeDisplay}
-        <Toggle
-          bind:value={primaryDisplay}
-          on:change={onChangePrimaryDisplay}
-          thin
-          text="Use as table display column"
-        />
-      {/if}
-    </div>
-  {/if}
-
   {#if editableColumn.type === "string"}
     <Input
       type="number"
@@ -461,46 +455,65 @@
       bind:value={editableColumn.constraints.length.maximum}
     />
   {:else if editableColumn.type === "options"}
-    <ValuesList
-      label="Options (one per line)"
-      bind:values={editableColumn.constraints.inclusion}
+    <OptionSelectDnD
+      bind:constraints={editableColumn.constraints}
+      bind:optionColors={editableColumn.optionColors}
     />
   {:else if editableColumn.type === "longform"}
     <div>
-      <Label
-        size="M"
-        tooltip="Rich text includes support for images, links, tables, lists and more"
+      <div class="tooltip-alignment">
+        <Label size="M">Formatting</Label>
+        <AbsTooltip
+          position="top"
+          type="info"
+          text={"Rich text includes support for images, link"}
         >
-        Formatting
-      </Label>
+          <Icon size="XS" name="InfoOutline" />
+        </AbsTooltip>
+      </div>
+
       <Toggle
         bind:value={editableColumn.useRichText}
         text="Enable rich text support (markdown)"
       />
     </div>
   {:else if editableColumn.type === "array"}
-    <ValuesList
-      label="Options (one per line)"
-      bind:values={editableColumn.constraints.inclusion}
+    <OptionSelectDnD
+      bind:constraints={editableColumn.constraints}
+      bind:optionColors={editableColumn.optionColors}
     />
   {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn}
-    <DatePicker
-      label="Earliest"
-      bind:value={editableColumn.constraints.datetime.earliest}
-    />
-    <DatePicker
-      label="Latest"
-      bind:value={editableColumn.constraints.datetime.latest}
-    />
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Earliest</Label>
+      </div>
+      <div class="input-length">
+        <DatePicker bind:value={editableColumn.constraints.datetime.earliest} />
+      </div>
+    </div>
+
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Latest</Label>
+      </div>
+      <div class="input-length">
+        <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
+      </div>
+    </div>
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
-        <Label
-          tooltip={isCreating
+        <div>
+          <Label>Time zones</Label>
+          <AbsTooltip
+            position="top"
+            type="info"
+            text={isCreating
               ? null
               : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
           >
-          Time zones
-        </Label>
+            <Icon size="XS" name="InfoOutline" />
+          </AbsTooltip>
+        </div>
         <Toggle
           bind:value={editableColumn.ignoreTimezones}
           text="Ignore time zones"
@@ -508,16 +521,30 @@
       </div>
     {/if}
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Max Value</Label>
+      </div>
+      <div class="input-length">
         <Input
           type="number"
-      label="Min Value"
-      bind:value={editableColumn.constraints.numericality.greaterThanOrEqualTo}
+          bind:value={editableColumn.constraints.numericality
+            .greaterThanOrEqualTo}
         />
+      </div>
+    </div>
+
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Max Value</Label>
+      </div>
+      <div class="input-length">
         <Input
           type="number"
-      label="Max Value"
           bind:value={editableColumn.constraints.numericality.lessThanOrEqualTo}
         />
+      </div>
+    </div>
   {:else if editableColumn.type === "link"}
     <Select
       label="Table"
@@ -546,8 +573,12 @@
     />
   {:else if editableColumn.type === FORMULA_TYPE}
     {#if !table.sql}
+      <div class="split-label">
+        <div class="label-length">
+          <Label size="M">Formula Type</Label>
+        </div>
+        <div class="input-length">
           <Select
-        label="Formula type"
             bind:value={editableColumn.formulaType}
             options={[
               { label: "Dynamic", value: "dynamic" },
@@ -558,10 +589,16 @@
             tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,
          while static formula are calculated when the row is saved."
           />
+        </div>
+      </div>
     {/if}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Formula</Label>
+      </div>
+      <div class="input-length">
         <ModalBindableInput
           title="Formula"
-      label="Formula"
           value={editableColumn.formula}
           on:change={e => {
             editableColumn = {
@@ -572,6 +609,8 @@
           bindings={getBindings({ table })}
           allowJS
         />
+      </div>
+    </div>
   {:else if editableColumn.type === JSON_TYPE}
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
@@ -590,12 +629,28 @@
     />
   {/if}
 
-  <div slot="footer">
-    {#if !uneditable && originalName != null}
-      <Button warning text on:click={confirmDelete}>Delete</Button>
+  {#if canBeRequired || canBeDisplay}
+    <div>
+      {#if canBeRequired}
+        <Toggle
+          value={required}
+          on:change={onChangeRequired}
+          disabled={primaryDisplay}
+          thin
+          text="Required"
+        />
       {/if}
     </div>
-</ModalContent>
+  {/if}
+</Layout>
+
+<div class="action-buttons">
+  {#if !uneditable && originalName != null}
+    <Button quiet warning text on:click={confirmDelete}>Delete</Button>
+  {/if}
+  <Button secondary newStyles on:click={cancelEdit}>Cancel</Button>
+  <Button disabled={invalid} newStyles cta on:click={saveColumn}>Save</Button>
+</div>
 <Modal bind:this={jsonSchemaModal}>
   <JSONSchemaModal
     schema={editableColumn.schema}
@@ -606,6 +661,7 @@
     }}
   />
 </Modal>
+
 <ConfirmDialog
   bind:this={confirmDeleteDialog}
   okText="Delete Column"
@@ -621,3 +677,30 @@
   </p>
   <Input bind:value={deleteColName} placeholder={originalName} />
 </ConfirmDialog>
+
+<style>
+  .action-buttons {
+    display: flex;
+    justify-content: flex-end;
+    margin-top: var(--spacing-s);
+    gap: var(--spacing-l);
+  }
+  .split-label {
+    display: flex;
+    align-items: center;
+  }
+
+  .tooltip-alignment {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-xs);
+  }
+
+  .label-length {
+    flex-basis: 40%;
+  }
+
+  .input-length {
+    flex-grow: 1;
+  }
+</style>

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -95,9 +95,9 @@
   {#if !creating}
     <div>
       A user's email, role, first and last names cannot be changed from within
-      the app builder. Please go to the <Link
-        on:click={$goto("/builder/portal/manage/users")}>user portal</Link
-      > to do this.
+      the app builder. Please go to the
+      <Link on:click={$goto("/builder/portal/users/users")}>user portal</Link>
+      to do this.
     </div>
   {/if}
   <RowFieldControl

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte

@@ -1,15 +1,8 @@
 <script>
-  import { getContext, onMount } from "svelte"
-  import { Modal } from "@budibase/bbui"
+  import { getContext } from "svelte"
   import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
 
-  const { rows, subscribe } = getContext("grid")
-
-  let modal
-
-  onMount(() => subscribe("add-column", modal.show))
+  const { rows } = getContext("grid")
 </script>
 
-<Modal bind:this={modal}>
-  <CreateEditColumn on:updatecolumns={rows.actions.refreshTableDefinition} />
-</Modal>
+<CreateEditColumn on:updatecolumns={rows.actions.refreshTableDefinition} />

packages/builder/src/components/backend/DataTable/modals/grid/GridEditColumnModal.svelte

@@ -1,24 +1,19 @@
 <script>
   import { getContext, onMount } from "svelte"
-  import { Modal } from "@budibase/bbui"
   import CreateEditColumn from "../CreateEditColumn.svelte"
 
   const { rows, subscribe } = getContext("grid")
 
   let editableColumn
-  let editColumnModal
 
   const editColumn = column => {
     editableColumn = column
-    editColumnModal.show()
   }
 
   onMount(() => subscribe("edit-column", editColumn))
 </script>
 
-<Modal bind:this={editColumnModal}>
-  <CreateEditColumn
+<CreateEditColumn
   field={editableColumn}
   on:updatecolumns={rows.actions.refreshData}
-  />
-</Modal>
+/>

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { RelationshipTypes } from "constants/backend"
+  import { RelationshipType } from "constants/backend"
   import {
     keepOpen,
     Button,
@@ -25,11 +25,11 @@
   const relationshipTypes = [
     {
       label: "One to Many",
-      value: RelationshipTypes.MANY_TO_ONE,
+      value: RelationshipType.MANY_TO_ONE,
     },
     {
       label: "Many to Many",
-      value: RelationshipTypes.MANY_TO_MANY,
+      value: RelationshipType.MANY_TO_MANY,
     },
   ]
 
@@ -58,8 +58,8 @@
     value: table._id,
   }))
   $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
-  $: isManyToMany = relationshipType === RelationshipTypes.MANY_TO_MANY
-  $: isManyToOne = relationshipType === RelationshipTypes.MANY_TO_ONE
+  $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
+  $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
 
   function getTable(id) {
     return plusTables.find(table => table._id === id)
@@ -116,7 +116,7 @@
 
   function allRequiredAttributesSet() {
     const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
-    if (relationshipType === RelationshipTypes.MANY_TO_ONE) {
+    if (relationshipType === RelationshipType.MANY_TO_ONE) {
       return base && fromPrimary && fromForeign
     } else {
       return base && getTable(throughId) && throughFromKey && throughToKey
@@ -181,12 +181,12 @@
   }
 
   function otherRelationshipType(type) {
-    if (type === RelationshipTypes.MANY_TO_ONE) {
-      return RelationshipTypes.ONE_TO_MANY
-    } else if (type === RelationshipTypes.ONE_TO_MANY) {
-      return RelationshipTypes.MANY_TO_ONE
-    } else if (type === RelationshipTypes.MANY_TO_MANY) {
-      return RelationshipTypes.MANY_TO_MANY
+    if (type === RelationshipType.MANY_TO_ONE) {
+      return RelationshipType.ONE_TO_MANY
+    } else if (type === RelationshipType.ONE_TO_MANY) {
+      return RelationshipType.MANY_TO_ONE
+    } else if (type === RelationshipType.MANY_TO_MANY) {
+      return RelationshipType.MANY_TO_MANY
     }
   }
 
@@ -218,7 +218,7 @@
 
     // if any to many only need to check from
     const manyToMany =
-      relateFrom.relationshipType === RelationshipTypes.MANY_TO_MANY
+      relateFrom.relationshipType === RelationshipType.MANY_TO_MANY
 
     if (!manyToMany) {
       delete relateFrom.through
@@ -253,7 +253,7 @@
       }
       relateTo = {
         ...relateTo,
-        relationshipType: RelationshipTypes.ONE_TO_MANY,
+        relationshipType: RelationshipType.ONE_TO_MANY,
         foreignKey: relateFrom.fieldName,
         fieldName: fromPrimary,
       }
@@ -321,7 +321,7 @@
       fromColumn = toRelationship.name
     }
     relationshipType =
-      fromRelationship.relationshipType || RelationshipTypes.MANY_TO_ONE
+      fromRelationship.relationshipType || RelationshipType.MANY_TO_ONE
     if (selectedFromTable) {
       fromId = selectedFromTable._id
       fromColumn = selectedFromTable.name

packages/builder/src/components/backend/Datasources/relationshipErrors.js

@@ -1,4 +1,4 @@
-import { RelationshipTypes } from "constants/backend"
+import { RelationshipType } from "constants/backend"
 
 const typeMismatch = "Column type of the foreign key must match the primary key"
 const columnBeingUsed = "Column name cannot be an existing column"
@@ -40,7 +40,7 @@ export class RelationshipErrorChecker {
   }
 
   isMany() {
-    return this.type === RelationshipTypes.MANY_TO_MANY
+    return this.type === RelationshipType.MANY_TO_MANY
   }
 
   relationshipTypeSet(type) {

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,17 +1,9 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Icon } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
-  let fileInput
-  let error = null
-  let fileName = null
-
-  let loading = false
-  let validation = {}
-  let validateHash = ""
-
   export let rows = []
   export let schema = {}
   export let allValid = true
@@ -49,6 +41,27 @@
     },
   ]
 
+  let fileInput
+  let error = null
+  let fileName = null
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let errors = {}
+
+  $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
+    return validation[column]
+  })
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+    validateHash = newValidateHash
+  }
+  $: openFileUpload(promptUpload, fileInput)
+
   async function handleFile(e) {
     loading = true
     error = null
@@ -67,34 +80,23 @@
 
   async function validate(rows, schema) {
     loading = true
-    error = null
-    validation = {}
-    allValid = false
-
     try {
       if (rows.length > 0) {
         const response = await API.validateNewTableImport({ rows, schema })
         validation = response.schemaValidation
         allValid = response.allValid
+        errors = response.errors
+        error = null
       }
     } catch (e) {
       error = e.message
+      validation = {}
+      allValid = false
+      errors = {}
     }
-
     loading = false
   }
 
-  $: {
-    // binding in consumer is causing double renders here
-    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
-
-    if (newValidateHash !== validateHash) {
-      validate(rows, schema)
-    }
-
-    validateHash = newValidateHash
-  }
-
   const handleChange = (name, e) => {
     schema[name].type = e.detail
     schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
@@ -106,7 +108,13 @@
     }
   }
 
-  $: openFileUpload(promptUpload, fileInput)
+  const deleteColumn = name => {
+    if (loading) {
+      return
+    }
+    delete schema[name]
+    schema = schema
+  }
 </script>
 
 <div class="dropzone">
@@ -119,10 +127,8 @@
     on:change={handleFile}
   />
   <label for="file-upload" class:uploaded={rows.length > 0}>
-    {#if loading}
-      loading...
-    {:else if error}
-      error: {error}
+    {#if error}
+      Error: {error}
     {:else if fileName}
       {fileName}
     {:else}
@@ -142,23 +148,26 @@
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={loading}
         />
         <span
-          class={loading || validation[column.name]
+          class={validation[column.name]
             ? "fieldStatusSuccess"
             : "fieldStatusFailure"}
         >
-          {validation[column.name] ? "Success" : "Failure"}
+          {#if validation[column.name]}
+            Success
+          {:else}
+            Failure
+            {#if errors[column.name]}
+              <Icon name="Help" tooltip={errors[column.name]} />
+            {/if}
+          {/if}
         </span>
-        <i
-          class={`omit-button ri-close-circle-fill ${
-            loading ? "omit-button-disabled" : ""
-          }`}
-          on:click={() => {
-            delete schema[column.name]
-            schema = schema
-          }}
+        <Icon
+          size="S"
+          name="Close"
+          hoverable
+          on:click={() => deleteColumn(column.name)}
         />
       </div>
     {/each}
@@ -167,7 +176,7 @@
     <Select
       label="Display Column"
       bind:value={displayColumn}
-      options={Object.keys(schema)}
+      options={displayColumnOptions}
       sort
     />
   </div>
@@ -235,23 +244,16 @@
     justify-self: center;
     font-weight: 600;
   }
-
   .fieldStatusFailure {
     color: var(--red);
     justify-self: center;
     font-weight: 600;
+    display: flex;
+    align-items: center;
+    gap: 4px;
   }
-
-  .omit-button {
-    font-size: 1.2em;
-    color: var(--grey-7);
-    cursor: pointer;
-    justify-self: flex-end;
-  }
-
-  .omit-button-disabled {
-    pointer-events: none;
-    opacity: 70%;
+  .fieldStatusFailure :global(.spectrum-Icon) {
+    width: 12px;
   }
 
   .display-column {