Merge branch 'develop' into bug/sev5/make-query-name-safe

.github/workflows/deploy-cloud.yaml

@@ -38,17 +38,6 @@ jobs:
           fi
           echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
       
-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:prod
-          docker tag proxy-service budibase/proxy:$PROD_TAG
-          docker push budibase/proxy:$PROD_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          PROD_TAG: k8s
-      
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

.github/workflows/deploy-preprod.yml

@@ -28,17 +28,6 @@ jobs:
           release_version=$(cat lerna.json | jq -r '.version')
           echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
 
-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:preprod
-          docker tag proxy-service budibase/proxy:$PREPROD_TAG
-          docker push budibase/proxy:$PREPROD_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          PREPROD_TAG: k8s-preprod
-
       - name: Pull values.yaml from budibase-infra
         run: | 
           curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \

.github/workflows/deploy-release.yml

@@ -29,17 +29,6 @@ jobs:
           release_version=$(cat lerna.json | jq -r '.version')
           echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
 
-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:release
-          docker tag proxy-service budibase/proxy:$RELEASE_TAG
-          docker push budibase/proxy:$RELEASE_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          RELEASE_TAG: k8s-release
-
       - name: Pull values.yaml from budibase-infra
         run: | 
           curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \

.github/workflows/release-develop.yml

@@ -26,7 +26,7 @@ env:
   FEATURE_PREVIEW_URL: https://budirelease.live
       
 jobs:
-  release:
+  release-images:
     runs-on: ubuntu-latest 
 
     steps:
@@ -44,19 +44,12 @@ jobs:
         run: yarn install:pro develop
 
       - run: yarn 
-      - run: yarn bootstrap 
+      - run: yarn bootstrap
       - run: yarn lint 
       - run: yarn build
       - run: yarn build:sdk
       - run: yarn test
 
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-1
-
       - name: Publish budibase packages to NPM
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
@@ -76,22 +69,25 @@ jobs:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
 
-      - name: Get the latest budibase release version
+  deploy-to-release-env:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get the current budibase release version
         id: version
-        run: | 
+        run: |
           release_version=$(cat lerna.json | jq -r '.version')
           echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
 
-      - name: Tag and release Proxy service docker image
+      - name: Configure AWS Credentials
-        run: | 
+        uses: aws-actions/configure-aws-credentials@v1
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+        with:
-          yarn build:docker:proxy:release
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          docker tag proxy-service budibase/proxy:$RELEASE_TAG
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          docker push budibase/proxy:$RELEASE_TAG
+          aws-region: eu-west-1
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          RELEASE_TAG: k8s-release
 
       - name: Pull values.yaml from budibase-infra
         run: | 
@@ -149,3 +145,54 @@ jobs:
           webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
           content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
           embed-title: ${{ env.RELEASE_VERSION }}
+
+  release-helm-chart:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Helm
+        uses: azure/setup-helm@v1
+        id: helm-install
+
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
+      - name: Build and release helm chart
+        run: |
+          git config user.name "Budibase Helm Bot"
+          git config user.email "<>"
+          git reset --hard
+          git pull
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync
+          echo "Packaging successful"
+          git checkout gh-pages
+          echo "Indexing helm repo"         
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
+          git add -A
+          git commit -m "Helm Release: develop"
+          git push
+
+  trigger-deploy-to-qa-env:
+    needs: [release-helm-chart]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the current budibase release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
+        with:
+          repository: budibase/budibase-deploys
+          event: deploy-develop-to-qa
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-selfhost.yml

@@ -67,16 +67,24 @@ jobs:
         uses: azure/setup-helm@v1
         id: helm-install
 
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
       - name: Build and release helm chart
         run: | 
           git config user.name "Budibase Helm Bot"
           git config user.email "<>"
           git reset --hard
           git pull
-          helm package charts/budibase
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync
+          echo "Packaging successful"
           git checkout gh-pages
-          mv *.tgz docs
+          echo "Indexing helm repo"
-          helm repo index docs
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
           git add -A
           git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
           git push

.github/workflows/release.yml

@@ -98,17 +98,6 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-1
 
-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:preprod
-          docker tag proxy-service budibase/proxy:$PREPROD_TAG
-          docker push budibase/proxy:$PREPROD_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          PREPROD_TAG: k8s-preprod
-
       - name: Pull values.yaml from budibase-infra
         run: | 
           curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \

.gitignore

@@ -4,6 +4,7 @@ builder/*
 packages/server/runtime_apps/
 .idea/
 bb-airgapped.tar.gz
+*.iml
 
 # Logs
 logs
@@ -65,8 +66,6 @@ typings/
 .env
 !qa-core/.env
 !hosting/.env
-hosting/.generated-nginx.dev.conf
-hosting/proxy/.generated-nginx.prod.conf
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -104,5 +103,9 @@ stats.html
 
 # TypeScript cache
 *.tsbuildinfo
+
+# plugins
 budibase-component
 budibase-datasource
+
+*.iml

.vscode/settings.json

@@ -8,7 +8,7 @@
         "editor.defaultFormatter": "vscode.json-language-features"
     },
     "[javascript]": {
-        "editor.defaultFormatter": "esbenp.prettier-vscode"
+        "editor.defaultFormatter": "vscode.typescript-language-features"
     },
     "debug.javascript.terminalOptions": {
         "skipFiles": [
@@ -16,4 +16,7 @@
             "<node_internals>/**"
         ]
     },
+    "[typescript]": {
+        "editor.defaultFormatter": "vscode.typescript-language-features"
+    },
 }

charts/budibase/Chart.yaml

@@ -11,11 +11,13 @@ sources:
   - https://github.com/Budibase/budibase
   - https://budibase.com
 type: application
-version: 0.2.11
+# populates on packaging
-appVersion: 1.0.214
+version: 0.0.0
+# populates on packaging
+appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 3.6.1
+    version: 3.3.4
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled
   - name: ingress-nginx

charts/budibase/templates/app-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: app-service
@@ -23,6 +20,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.apps.annotations }}
+{{- toYaml .Values.services.apps.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: app-service
@@ -67,6 +67,8 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}
         {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
         - name: MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -77,13 +79,19 @@ spec:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
         - name: MINIO_URL
           value: {{ .Values.services.objectStore.url }}
         - name: PLUGIN_BUCKET_NAME
           value: {{ .Values.services.objectStore.pluginBucketName | quote }}
         - name: APPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
@@ -159,6 +167,18 @@ spec:
         - name: ELASTIC_APM_SERVER_URL
           value: {{ .Values.globals.elasticApmServerUrl | quote }}
         {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
         - name: CDN_URL
           value: {{ .Values.globals.cdnUrl }}
         {{ if .Values.services.tlsRejectUnauthorized }}

charts/budibase/templates/proxy-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
@@ -23,16 +20,34 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.proxy.annotations }}
+{{- toYaml .Values.services.proxy.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         app.kubernetes.io/name: budibase-proxy
     spec:
       containers:
-      - image: budibase/proxy:{{ .Values.services.proxy.tag | default "k8s" }}
+      - image: budibase/proxy:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         name: proxy-service
         ports:
         - containerPort: {{ .Values.services.proxy.port }}
+        env:
+        - name: APPS_UPSTREAM_URL
+          value: {{ tpl .Values.services.proxy.upstreams.apps . | quote }}
+        - name: WORKER_UPSTREAM_URL
+          value: {{ tpl .Values.services.proxy.upstreams.worker . | quote }}
+        - name: MINIO_UPSTREAM_URL
+          value: {{ tpl .Values.services.proxy.upstreams.minio . | quote }}
+        - name: COUCHDB_UPSTREAM_URL
+          value: {{ .Values.services.couchdb.url | default (tpl .Values.services.proxy.upstreams.couchdb .) | quote }}
+        - name: RESOLVER
+          {{ if .Values.services.proxy.resolver }}
+          value: {{ .Values.services.proxy.resolver }}
+          {{ else }}
+          value: kube-dns.kube-system.svc.{{ .Values.services.dns }}
+          {{ end }}
         {{ with .Values.services.proxy.resources }}
         resources:
         {{- toYaml . | nindent 10 }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: worker-service
@@ -24,6 +21,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.worker.annotations }}
+{{- toYaml .Values.services.worker.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: worker-service
@@ -68,6 +68,8 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}
         {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
         - name: MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -80,11 +82,17 @@ spec:
               key: objectStoreSecret
         - name: MINIO_URL
           value: {{ .Values.services.objectStore.url }}
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
         - name: PLUGIN_BUCKET_NAME
           value: {{ .Values.services.objectStore.pluginBucketName | quote }}
         - name: APPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
@@ -150,6 +158,18 @@ spec:
         - name: ELASTIC_APM_SERVER_URL
           value: {{ .Values.globals.elasticApmServerUrl | quote }}
         {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
         - name: CDN_URL
           value: {{ .Values.globals.cdnUrl }}
         {{ if .Values.services.tlsRejectUnauthorized }}

charts/budibase/values.yaml

@@ -22,12 +22,6 @@ serviceAccount:
 
 podAnnotations: {}
 
-# logAnnotations:
-#   co.elastic.logs/multiline.type: pattern
-#   co.elastic.logs/multiline.pattern: '^[[:space:]]'
-#   co.elastic.logs/multiline.negate: false
-#   co.elastic.logs/multiline.match: after
-
 podSecurityContext:
   {}
   # fsGroup: 2000
@@ -112,6 +106,9 @@ globals:
 #  elasticApmEnabled:
 #  elasticApmSecretToken:
 #  elasticApmServerUrl:
+#  globalAgentHttpProxy:
+#  globalAgentHttpsProxy:
+#  globalAgentNoProxy:
 
 services:
   budibaseVersion: latest
@@ -121,7 +118,16 @@ services:
   proxy:
     port: 10000
     replicaCount: 1
+    upstreams:
+      apps: 'http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}'
+      worker: 'http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}'
+      minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}'
+      couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}'
     resources: {}
+#    annotations:
+#      co.elastic.logs/module: nginx
+#      co.elastic.logs/fileset.stdout: access
+#      co.elastic.logs/fileset.stderr: error
 
   apps:
     port: 4002
@@ -129,11 +135,20 @@ services:
     logLevel: info
     resources: {}
 #    nodeDebug: "" # set the value of NODE_DEBUG
-
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
   worker:
     port: 4003
     replicaCount: 1
     resources: {}
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
 
   couchdb:
     enabled: true
@@ -164,6 +179,7 @@ services:
     resources: {}
 
   objectStore:
+    # Set to false if using another object store such as S3
     minio: true
     browser: true
     port: 9000
@@ -179,6 +195,13 @@ services:
     ##   set, choosing the default provisioner.
     storageClass: ""
     resources: {}
+    cloudfront:
+      # Set the url of a distribution to enable cloudfront
+      cdn: ""
+      # ID of public key stored in cloudfront
+      publicKeyId: ""
+      # Base64 encoded private key for the above public key
+      privateKey64: ""
 
 # Override values in couchDB subchart
 couchdb:

hosting/docker-compose.dev.yaml

@@ -25,9 +25,9 @@ services:
   proxy-service:
     container_name: budi-nginx-dev
     restart: on-failure
-    image: nginx:latest
+    image: budibase/proxy:latest
     volumes:
-      - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf
+      - ./nginx.dev.conf:/etc/nginx/templates/nginx.conf.template
       - ./proxy/error.html:/usr/share/nginx/html/error.html
     ports:
       - "${MAIN_PORT}:10000"
@@ -36,6 +36,8 @@ services:
       - couchdb-service
     extra_hosts:
       - "host.docker.internal:host-gateway"
+    environment:
+      - PROXY_ADDRESS=host.docker.internal
 
   couchdb-service:
     # platform: linux/amd64

hosting/docker-compose.yaml

@@ -82,6 +82,12 @@ services:
     environment:
       - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
       - PROXY_RATE_LIMIT_API_PER_SECOND=20
+      - APPS_UPSTREAM_URL=http://app-service:4002
+      - WORKER_UPSTREAM_URL=http://worker-service:4003
+      - MINIO_UPSTREAM_URL=http://minio-service:9000
+      - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+      - RESOLVER=127.0.0.11
     depends_on:
       - minio-service
       - worker-service

hosting/nginx.dev.conf

@@ -25,17 +25,17 @@ http {
   }
 
   upstream app-service {
-    server {{address}}:4001;
+    server ${PROXY_ADDRESS}:4001;
     keepalive 32;
   }
 
   upstream worker-service {
-    server {{address}}:4002;
+    server ${PROXY_ADDRESS}:4002;
     keepalive 32;
   }
 
   upstream builder {
-    server {{address}}:3000;
+    server ${PROXY_ADDRESS}:3000;
     keepalive 32;
   }
 
@@ -186,6 +186,26 @@ http {
       proxy_pass      http://minio-service:9000;
     }
 
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://minio-service:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

hosting/proxy/Dockerfile

@@ -4,7 +4,7 @@ FROM nginx:latest
 # use the default nginx behaviour for *.template files which are processed with envsubst
 # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
 ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
-COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
+COPY nginx.prod.conf /etc/nginx/templates/nginx.conf.template
 
 # IPv6 removal needs to happen after envsubst
 RUN rm -rf /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
@@ -16,4 +16,11 @@ COPY error.html /usr/share/nginx/html/error.html
 
 # Default environment
 ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
-ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
+ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
+# Use docker-compose values as defaults for backwards compatibility
+ENV APPS_UPSTREAM_URL=http://app-service:4002
+ENV WORKER_UPSTREAM_URL=http://worker-service:4003
+ENV MINIO_UPSTREAM_URL=http://minio-service:9000
+ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+ENV RESOLVER=127.0.0.11

hosting/proxy/nginx.prod.conf

@@ -23,7 +23,7 @@ http {
   tcp_nodelay on;
   server_tokens off;
   types_hash_max_size 2048;
-  resolver {{ resolver }} valid=10s ipv6=off;
+  resolver ${RESOLVER} valid=10s ipv6=off;
 
   # buffering
   client_header_buffer_size 1k;
@@ -76,27 +76,23 @@ http {
     add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
 
     # upstreams
-    set $apps {{ apps }};
+    set $apps ${APPS_UPSTREAM_URL};
-    set $worker {{ worker }};
+    set $worker ${WORKER_UPSTREAM_URL};
-    set $minio {{ minio }};
+    set $minio ${MINIO_UPSTREAM_URL};
-    set $couchdb {{ couchdb }};
+    set $couchdb ${COUCHDB_UPSTREAM_URL};
-    {{#if watchtower}}
+    set $watchtower ${WATCHTOWER_UPSTREAM_URL};
-    set $watchtower {{ watchtower }};
-    {{/if}}
 
     location /app {
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
     }
 
     location = / {
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
     }
 
-    {{#if watchtower}}
     location = /v1/update {
-      proxy_pass      http://$watchtower:8080;
+      proxy_pass      $watchtower;
     }
-    {{/if}}
 
     location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
@@ -107,19 +103,17 @@ http {
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
       proxy_set_header    Host                $host;
 
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
     }
 
     location ~ ^/api/(system|admin|global)/ {
       proxy_set_header    Host                $host;
-
+      proxy_pass      $worker;
-      proxy_pass      http://$worker:4003;
     }
 
     location /worker/ {
       proxy_set_header    Host                $host;
-
+      proxy_pass      $worker;
-      proxy_pass      http://$worker:4003;
       rewrite ^/worker/(.*)$ /$1 break;
     }
 
@@ -138,7 +132,7 @@ http {
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
 
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
     }
 
     location /api/ {
@@ -157,7 +151,7 @@ http {
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
       proxy_set_header    Host                $host;
 
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
     }
 
     location /api/webhooks/ {
@@ -177,11 +171,11 @@ http {
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
       proxy_set_header    Host                $host;
 
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
     }
 
     location /db/ {
-      proxy_pass      http://$couchdb:5984;
+      proxy_pass      $couchdb;
       rewrite ^/db/(.*)$ /$1 break;
     }
 
@@ -191,7 +185,7 @@ http {
       proxy_set_header    Connection  'upgrade';
       proxy_set_header    Host        $host;
       proxy_cache_bypass  $http_upgrade;
-      proxy_pass          http://$apps:4002;
+      proxy_pass      $apps;
     }
 
     location / {
@@ -205,7 +199,27 @@ http {
       proxy_set_header Connection "";
       chunked_transfer_encoding off;
 
-      proxy_pass      http://$minio:9000;
+      proxy_pass      $minio;
+    }
+
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      $minio;
+      rewrite ^/files/signed/(.*)$ /$1 break;
     }
 
     client_header_timeout 60;

hosting/single/nginx/nginx-default-site.conf

@@ -95,15 +95,37 @@ server {
     }
 
     location / {
-        proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
 
-        proxy_connect_timeout 300;
+      proxy_connect_timeout 300;
-        proxy_http_version 1.1;
+      proxy_http_version 1.1;
-        proxy_set_header Connection "";
+      proxy_set_header Connection "";
-        chunked_transfer_encoding off;
+      chunked_transfer_encoding off;
-        proxy_pass      http://127.0.0.1:9000;
+
+      proxy_pass      http://127.0.0.1:9000;
+    }
+
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://127.0.0.1:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
     }
 
     client_header_timeout 60;

hosting/single/runner.sh

@@ -78,7 +78,7 @@ mkdir -p ${DATA_DIR}/search
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
-/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.1.43-alpha.15",
+  "version": "2.2.12-alpha.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -25,6 +25,7 @@
     "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
     "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+    "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli",
     "build:sdk": "lerna run build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
@@ -44,8 +45,8 @@
     "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test && yarn test:pro",
     "test:pro": "bash scripts/pro/test.sh",
-    "lint:eslint": "eslint packages",
+    "lint:eslint": "eslint packages && eslint qa-core",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
@@ -55,15 +56,11 @@
     "test:e2e:ci:record": "lerna run cy:ci:record --stream",
     "test:e2e:ci:notify": "lerna run cy:ci:notify",
     "build:specs": "lerna run specs",
-    "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
     "build:docker:pre": "lerna run build && lerna run predocker",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
-    "build:docker:proxy:compose": "node scripts/proxy/generateProxyConfig compose && npm run build:docker:proxy",
-    "build:docker:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod && npm run build:docker:proxy",
-    "build:docker:proxy:release": "node scripts/proxy/generateProxyConfig release && npm run build:docker:proxy",
-    "build:docker:proxy:prod": "node scripts/proxy/generateProxyConfig prod && npm run build:docker:proxy",
     "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
+    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
     "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",

packages/backend-core/__mocks__/aws-sdk.ts

@@ -3,7 +3,10 @@ const mockS3 = {
   deleteObject: jest.fn().mockReturnThis(),
   deleteObjects: jest.fn().mockReturnThis(),
   createBucket: jest.fn().mockReturnThis(),
-  listObjects: jest.fn().mockReturnThis(),
+  listObject: jest.fn().mockReturnThis(),
+  getSignedUrl: jest.fn((operation: string, params: any) => {
+    return `http://s3.example.com/${params.Bucket}/${params.Key}`
+  }),
   promise: jest.fn().mockReturnThis(),
   catch: jest.fn(),
 }

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.1.43-alpha.15",
+  "version": "2.2.12-alpha.4",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -15,14 +15,18 @@
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
+    "build:pro": "../../scripts/pro/build.sh",
+    "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "2.1.43-alpha.15",
+    "@budibase/nano": "10.1.1",
+    "@budibase/types": "2.2.12-alpha.4",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
+    "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",
     "bcryptjs": "2.4.3",
@@ -31,11 +35,10 @@
     "emitter-listener": "1.1.2",
     "ioredis": "4.28.0",
     "joi": "17.6.0",
-    "jsonwebtoken": "8.5.1",
+    "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "lodash": "4.17.21",
     "lodash.isarguments": "3.1.0",
-    "nano": "^10.1.0",
     "node-fetch": "2.6.7",
     "passport-google-oauth": "2.0.0",
     "passport-jwt": "4.0.0",

packages/backend-core/plugins.ts

@@ -0,0 +1 @@
+export * from "./src/plugin"

packages/backend-core/src/context/mainContext.ts

@@ -2,7 +2,7 @@
 // store an app ID to pretend there is a context
 import env from "../environment"
 import Context from "./Context"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
+import * as conversions from "../db/conversions"
 import { getDB } from "../db/db"
 import {
   DocumentType,
@@ -181,6 +181,14 @@ export function getAppId(): string | undefined {
   }
 }
 
+export const getProdAppId = () => {
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Could not get appId")
+  }
+  return conversions.getProdAppID(appId)
+}
+
 export function updateTenantId(tenantId?: string) {
   let context: ContextMap = updateContext({
     tenantId,
@@ -229,7 +237,7 @@ export function getProdAppDB(opts?: any): Database {
   if (!appId) {
     throw new Error("Unable to retrieve prod DB - no app ID.")
   }
-  return getDB(getProdAppID(appId), opts)
+  return getDB(conversions.getProdAppID(appId), opts)
 }
 
 /**
@@ -241,5 +249,5 @@ export function getDevAppDB(opts?: any): Database {
   if (!appId) {
     throw new Error("Unable to retrieve dev DB - no app ID.")
   }
-  return getDB(getDevelopmentAppID(appId), opts)
+  return getDB(conversions.getDevelopmentAppID(appId), opts)
 }

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -1,4 +1,4 @@
-import Nano from "nano"
+import Nano from "@budibase/nano"
 import {
   AllDocsResponse,
   AnyDocument,

packages/backend-core/src/db/utils.ts

@@ -14,7 +14,7 @@ import { doWithDB, allDbs, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import * as events from "../events"
-import { App, Database, ConfigType } from "@budibase/types"
+import { App, Database, ConfigType, isSettingsConfig } from "@budibase/types"
 
 /**
  * Generates a new app ID.
@@ -489,18 +489,12 @@ export const getScopedFullConfig = async function (
 
   // custom logic for settings doc
   if (type === ConfigType.SETTINGS) {
-    if (scopedConfig && scopedConfig.doc) {
+    if (!scopedConfig || !scopedConfig.doc) {
-      // overrides affected by environment variables
-      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
-        tenantAware: true,
-      })
-      scopedConfig.doc.config.analyticsEnabled =
-        await events.analytics.enabled()
-    } else {
       // defaults
       scopedConfig = {
         doc: {
           _id: generateConfigID({ type, user, workspace }),
+          type: ConfigType.SETTINGS,
           config: {
             platformUrl: await getPlatformUrl({ tenantAware: true }),
             analyticsEnabled: await events.analytics.enabled(),
@@ -508,6 +502,16 @@ export const getScopedFullConfig = async function (
         },
       }
     }
+
+    // will always be true - use assertion function to get type access
+    if (isSettingsConfig(scopedConfig.doc)) {
+      // overrides affected by environment
+      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
+        tenantAware: true,
+      })
+      scopedConfig.doc.config.analyticsEnabled =
+        await events.analytics.enabled()
+    }
   }
 
   return scopedConfig && scopedConfig.doc

packages/backend-core/src/environment.ts

@@ -1,9 +1,13 @@
 function isTest() {
-  return (
+  return isCypress() || isJest()
-    process.env.NODE_ENV === "jest" ||
+}
-    process.env.NODE_ENV === "cypress" ||
+
-    process.env.JEST_WORKER_ID != null
+function isJest() {
-  )
+  return !!(process.env.NODE_ENV === "jest" || process.env.JEST_WORKER_ID)
+}
+
+function isCypress() {
+  return process.env.NODE_ENV === "cypress"
 }
 
 function isDev() {
@@ -21,13 +25,16 @@ const DefaultBucketName = {
   APPS: "prod-budi-app-assets",
   TEMPLATES: "templates",
   GLOBAL: "global",
-  CLOUD: "prod-budi-tenant-uploads",
   PLUGINS: "plugins",
 }
 
 const environment = {
   isTest,
+  isJest,
   isDev,
+  isProd: () => {
+    return !isDev()
+  },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
@@ -42,6 +49,7 @@ const environment = {
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
   MINIO_URL: process.env.MINIO_URL,
+  MINIO_ENABLED: process.env.MINIO_ENABLED || 1,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL:
@@ -54,6 +62,9 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
+  CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
+  CLOUDFRONT_PRIVATE_KEY_64: process.env.CLOUDFRONT_PRIVATE_KEY_64,
+  CLOUDFRONT_PUBLIC_KEY_ID: process.env.CLOUDFRONT_PUBLIC_KEY_ID,
   BACKUPS_BUCKET_NAME:
     process.env.BACKUPS_BUCKET_NAME || DefaultBucketName.BACKUPS,
   APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || DefaultBucketName.APPS,
@@ -61,12 +72,9 @@ const environment = {
     process.env.TEMPLATES_BUCKET_NAME || DefaultBucketName.TEMPLATES,
   GLOBAL_BUCKET_NAME:
     process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
-  GLOBAL_CLOUD_BUCKET_NAME:
-    process.env.GLOBAL_CLOUD_BUCKET_NAME || DefaultBucketName.CLOUD,
   PLUGIN_BUCKET_NAME:
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
   USE_COUCH: process.env.USE_COUCH || true,
-  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
   LOG_LEVEL: process.env.LOG_LEVEL,
@@ -87,6 +95,11 @@ for (let [key, value] of Object.entries(environment)) {
     // @ts-ignore
     environment[key] = 0
   }
+  // handle the edge case of "false" to disable an environment variable
+  if (value === "false") {
+    // @ts-ignore
+    environment[key] = 0
+  }
 }
 
 export = environment

packages/backend-core/src/objectStore/buckets/app.ts

@@ -0,0 +1,40 @@
+import env from "../../environment"
+import * as objectStore from "../objectStore"
+import * as cloudfront from "../cloudfront"
+
+/**
+ * In production the client library is stored in the object store, however in development
+ * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * via a specific endpoint (under /api/assets/client).
+ * @param {string} appId In production we need the appId to look up the correct bucket, as the
+ * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
+ * @return {string} The URL to be inserted into appPackage response or server rendered
+ * app index file.
+ */
+export const clientLibraryUrl = (appId: string, version: string) => {
+  if (env.isProd()) {
+    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+    if (env.CLOUDFRONT_CDN) {
+      // append app version to bust the cache
+      if (version) {
+        file += `?v=${version}`
+      }
+      // don't need to use presigned for client with cloudfront
+      // file is public
+      return cloudfront.getUrl(file)
+    } else {
+      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+    }
+  } else {
+    return `/api/assets/client`
+  }
+}
+
+export const getAppFileUrl = (s3Key: string) => {
+  if (env.CLOUDFRONT_CDN) {
+    return cloudfront.getPresignedUrl(s3Key)
+  } else {
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
+  }
+}

packages/backend-core/src/objectStore/buckets/global.ts

@@ -0,0 +1,29 @@
+import env from "../../environment"
+import * as tenancy from "../../tenancy"
+import * as objectStore from "../objectStore"
+import * as cloudfront from "../cloudfront"
+
+// URLs
+
+export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
+  let file = getGlobalFileS3Key(type, name)
+  if (env.CLOUDFRONT_CDN) {
+    if (etag) {
+      file = `${file}?etag=${etag}`
+    }
+    return cloudfront.getPresignedUrl(file)
+  } else {
+    return objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
+  }
+}
+
+// KEYS
+
+export const getGlobalFileS3Key = (type: string, name: string) => {
+  let file = `${type}/${name}`
+  if (env.MULTI_TENANCY) {
+    const tenantId = tenancy.getTenantId()
+    file = `${tenantId}/${file}`
+  }
+  return file
+}

packages/backend-core/src/objectStore/buckets/index.ts

@@ -0,0 +1,3 @@
+export * from "./app"
+export * from "./global"
+export * from "./plugins"

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -0,0 +1,71 @@
+import env from "../../environment"
+import * as objectStore from "../objectStore"
+import * as tenancy from "../../tenancy"
+import * as cloudfront from "../cloudfront"
+import { Plugin } from "@budibase/types"
+
+// URLS
+
+export const enrichPluginURLs = (plugins: Plugin[]) => {
+  if (!plugins || !plugins.length) {
+    return []
+  }
+  return plugins.map(plugin => {
+    const jsUrl = getPluginJSUrl(plugin)
+    const iconUrl = getPluginIconUrl(plugin)
+    return { ...plugin, jsUrl, iconUrl }
+  })
+}
+
+const getPluginJSUrl = (plugin: Plugin) => {
+  const s3Key = getPluginJSKey(plugin)
+  return getPluginUrl(s3Key)
+}
+
+const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+  const s3Key = getPluginIconKey(plugin)
+  if (!s3Key) {
+    return
+  }
+  return getPluginUrl(s3Key)
+}
+
+const getPluginUrl = (s3Key: string) => {
+  if (env.CLOUDFRONT_CDN) {
+    return cloudfront.getPresignedUrl(s3Key)
+  } else {
+    return objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
+  }
+}
+
+// S3 KEYS
+
+export const getPluginJSKey = (plugin: Plugin) => {
+  return getPluginS3Key(plugin, "plugin.min.js")
+}
+
+export const getPluginIconKey = (plugin: Plugin) => {
+  // stored iconUrl is deprecated - hardcode to icon.svg in this case
+  const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
+  if (!iconFileName) {
+    return
+  }
+  return getPluginS3Key(plugin, iconFileName)
+}
+
+const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+  const s3Key = getPluginS3Dir(plugin.name)
+  return `${s3Key}/${fileName}`
+}
+
+export const getPluginS3Dir = (pluginName: string) => {
+  let s3Key = `${pluginName}`
+  if (env.MULTI_TENANCY) {
+    const tenantId = tenancy.getTenantId()
+    s3Key = `${tenantId}/${s3Key}`
+  }
+  if (env.CLOUDFRONT_CDN) {
+    s3Key = `plugins/${s3Key}`
+  }
+  return s3Key
+}

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -0,0 +1,171 @@
+import * as app from "../app"
+import { getAppFileUrl } from "../app"
+import { testEnv } from "../../../../tests"
+
+describe("app", () => {
+  beforeEach(() => {
+    testEnv.nodeJest()
+  })
+
+  describe("clientLibraryUrl", () => {
+    function getClientUrl() {
+      return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url in dev", () => {
+        testEnv.nodeDev()
+        const url = getClientUrl()
+        expect(url).toBe("/api/assets/client")
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+        )
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url in dev", async () => {
+        testEnv.nodeDev()
+        await testEnv.withTenant(tenantId => {
+          const url = getClientUrl()
+          expect(url).toBe("/api/assets/client")
+        })
+      })
+
+      it("gets url with embedded minio", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withMinio()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withS3()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withCloudfront()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+          )
+        })
+      })
+    })
+  })
+
+  describe("getAppFileUrl", () => {
+    function getAppFileUrl() {
+      return app.getAppFileUrl("app_123/attachments/image.jpeg")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getAppFileUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getAppFileUrl()
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getAppFileUrl()
+        // omit rest of signed params
+        expect(
+          url.includes("http://cf.example.com/app_123/attachments/image.jpeg?")
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          // omit rest of signed params
+          expect(
+            url.includes(
+              "http://cf.example.com/app_123/attachments/image.jpeg?"
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/tests/global.spec.ts

@@ -0,0 +1,74 @@
+import * as global from "../global"
+import { testEnv } from "../../../../tests"
+
+describe("global", () => {
+  describe("getGlobalFileUrl", () => {
+    function getGlobalFileUrl() {
+      return global.getGlobalFileUrl("settings", "logoUrl", "etag")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getGlobalFileUrl()
+        expect(url).toBe("/files/signed/global/settings/logoUrl")
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getGlobalFileUrl()
+        expect(url).toBe("http://s3.example.com/global/settings/logoUrl")
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getGlobalFileUrl()
+        // omit rest of signed params
+        expect(
+          url.includes("http://cf.example.com/settings/logoUrl?etag=etag&")
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          expect(url).toBe(`/files/signed/global/${tenantId}/settings/logoUrl`)
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          expect(url).toBe(
+            `http://s3.example.com/global/${tenantId}/settings/logoUrl`
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          // omit rest of signed params
+          expect(
+            url.includes(
+              `http://cf.example.com/${tenantId}/settings/logoUrl?etag=etag&`
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts

@@ -0,0 +1,110 @@
+import * as plugins from "../plugins"
+import { structures, testEnv } from "../../../../tests"
+
+describe("plugins", () => {
+  describe("enrichPluginURLs", () => {
+    const plugin = structures.plugins.plugin()
+
+    function getEnrichedPluginUrls() {
+      const enriched = plugins.enrichPluginURLs([plugin])[0]
+      return {
+        jsUrl: enriched.jsUrl!,
+        iconUrl: enriched.iconUrl!,
+      }
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const urls = getEnrichedPluginUrls()
+        expect(urls.jsUrl).toBe(
+          `/files/signed/plugins/${plugin.name}/plugin.min.js`
+        )
+        expect(urls.iconUrl).toBe(
+          `/files/signed/plugins/${plugin.name}/icon.svg`
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const urls = getEnrichedPluginUrls()
+        expect(urls.jsUrl).toBe(
+          `http://s3.example.com/plugins/${plugin.name}/plugin.min.js`
+        )
+        expect(urls.iconUrl).toBe(
+          `http://s3.example.com/plugins/${plugin.name}/icon.svg`
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const urls = getEnrichedPluginUrls()
+        // omit rest of signed params
+        expect(
+          urls.jsUrl.includes(
+            `http://cf.example.com/plugins/${plugin.name}/plugin.min.js?`
+          )
+        ).toBe(true)
+        expect(
+          urls.iconUrl.includes(
+            `http://cf.example.com/plugins/${plugin.name}/icon.svg?`
+          )
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          expect(urls.jsUrl).toBe(
+            `/files/signed/plugins/${tenantId}/${plugin.name}/plugin.min.js`
+          )
+          expect(urls.iconUrl).toBe(
+            `/files/signed/plugins/${tenantId}/${plugin.name}/icon.svg`
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          expect(urls.jsUrl).toBe(
+            `http://s3.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js`
+          )
+          expect(urls.iconUrl).toBe(
+            `http://s3.example.com/plugins/${tenantId}/${plugin.name}/icon.svg`
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          // omit rest of signed params
+          expect(
+            urls.jsUrl.includes(
+              `http://cf.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js?`
+            )
+          ).toBe(true)
+          expect(
+            urls.iconUrl.includes(
+              `http://cf.example.com/plugins/${tenantId}/${plugin.name}/icon.svg?`
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/cloudfront.ts

@@ -0,0 +1,41 @@
+import env from "../environment"
+const cfsign = require("aws-cloudfront-sign")
+
+let PRIVATE_KEY: string | undefined
+
+function getPrivateKey() {
+  if (!env.CLOUDFRONT_PRIVATE_KEY_64) {
+    throw new Error("CLOUDFRONT_PRIVATE_KEY_64 is not set")
+  }
+
+  if (PRIVATE_KEY) {
+    return PRIVATE_KEY
+  }
+
+  PRIVATE_KEY = Buffer.from(env.CLOUDFRONT_PRIVATE_KEY_64, "base64").toString(
+    "utf-8"
+  )
+
+  return PRIVATE_KEY
+}
+
+const getCloudfrontSignParams = () => {
+  return {
+    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID,
+    privateKeyString: getPrivateKey(),
+    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+  }
+}
+
+export const getPresignedUrl = (s3Key: string) => {
+  const url = getUrl(s3Key)
+  return cfsign.getSignedUrl(url, getCloudfrontSignParams())
+}
+
+export const getUrl = (s3Key: string) => {
+  let prefix = "/"
+  if (s3Key.startsWith("/")) {
+    prefix = ""
+  }
+  return `${env.CLOUDFRONT_CDN}${prefix}${s3Key}`
+}

packages/backend-core/src/objectStore/index.ts

@@ -1,2 +1,3 @@
 export * from "./objectStore"
 export * from "./utils"
+export * from "./buckets"

packages/backend-core/src/objectStore/objectStore.ts

@@ -8,7 +8,7 @@ import { promisify } from "util"
 import { join } from "path"
 import fs from "fs"
 import env from "../environment"
-import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
+import { budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 
@@ -26,7 +26,7 @@ type UploadParams = {
   bucket: string
   filename: string
   path: string
-  type?: string
+  type?: string | null
   // can be undefined, we will remove it
   metadata?: {
     [key: string]: string | undefined
@@ -41,6 +41,7 @@ const CONTENT_TYPE_MAP: any = {
   json: "application/json",
   gz: "application/gzip",
 }
+
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
   CONTENT_TYPE_MAP.css,
@@ -58,35 +59,17 @@ export function sanitizeBucket(input: string) {
   return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
 }
 
-function publicPolicy(bucketName: string) {
-  return {
-    Version: "2012-10-17",
-    Statement: [
-      {
-        Effect: "Allow",
-        Principal: {
-          AWS: ["*"],
-        },
-        Action: "s3:GetObject",
-        Resource: [`arn:aws:s3:::${bucketName}/*`],
-      },
-    ],
-  }
-}
-
-const PUBLIC_BUCKETS = [
-  ObjectStoreBuckets.APPS,
-  ObjectStoreBuckets.GLOBAL,
-  ObjectStoreBuckets.PLUGINS,
-]
-
 /**
  * Gets a connection to the object store using the S3 SDK.
  * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @param {object} opts configuration for the object store.
  * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (bucket: string) => {
+export const ObjectStore = (
+  bucket: string,
+  opts: { presigning: boolean } = { presigning: false }
+) => {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -100,9 +83,20 @@ export const ObjectStore = (bucket: string) => {
       Bucket: sanitizeBucket(bucket),
     }
   }
+
+  // custom S3 is in use i.e. minio
   if (env.MINIO_URL) {
-    config.endpoint = env.MINIO_URL
+    if (opts.presigning && env.MINIO_ENABLED) {
+      // IMPORTANT: Signed urls will inspect the host header of the request.
+      // Normally a signed url will need to be generated with a specified host in mind.
+      // To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      // use a predefined host. The host 'minio-service' is also forwarded to minio requests via nginx
+      config.endpoint = "minio-service"
+    } else {
+      config.endpoint = env.MINIO_URL
+    }
   }
+
   return new AWS.S3(config)
 }
 
@@ -135,16 +129,6 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
         await promises[bucketName]
         delete promises[bucketName]
       }
-      // public buckets are quite hidden in the system, make sure
-      // no bucket is set accidentally
-      if (PUBLIC_BUCKETS.includes(bucketName)) {
-        await client
-          .putBucketPolicy({
-            Bucket: bucketName,
-            Policy: JSON.stringify(publicPolicy(bucketName)),
-          })
-          .promise()
-      }
     } else {
       throw new Error("Unable to write to object store bucket.")
     }
@@ -274,6 +258,36 @@ export const listAllObjects = async (bucketName: string, path: string) => {
   return objects
 }
 
+/**
+ * Generate a presigned url with a default TTL of 1 hour
+ */
+export const getPresignedUrl = (
+  bucketName: string,
+  key: string,
+  durationSeconds: number = 3600
+) => {
+  const objectStore = ObjectStore(bucketName, { presigning: true })
+  const params = {
+    Bucket: sanitizeBucket(bucketName),
+    Key: sanitizeKey(key),
+    Expires: durationSeconds,
+  }
+  const url = objectStore.getSignedUrl("getObject", params)
+
+  if (!env.MINIO_ENABLED) {
+    // return the full URL to the client
+    return url
+  } else {
+    // return the path only to the client
+    // use the presigned url route to ensure the static
+    // hostname will be used in the request
+    const signedUrl = new URL(url)
+    const path = signedUrl.pathname
+    const query = signedUrl.search
+    return `/files/signed${path}${query}`
+  }
+}
+
 /**
  * Same as retrieval function but puts to a temporary file.
  */
@@ -315,9 +329,9 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
-    Key: filepath,
+    Key: sanitizeKey(filepath),
   }
-  return objectStore.deleteObject(params)
+  return objectStore.deleteObject(params).promise()
 }
 
 export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
@@ -326,7 +340,7 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
   const params = {
     Bucket: bucketName,
     Delete: {
-      Objects: filepaths.map((path: any) => ({ Key: path })),
+      Objects: filepaths.map((path: any) => ({ Key: sanitizeKey(path) })),
     },
   }
   return objectStore.deleteObjects(params).promise()

packages/backend-core/src/objectStore/utils.ts

@@ -14,7 +14,6 @@ export const ObjectStoreBuckets = {
   APPS: env.APPS_BUCKET_NAME,
   TEMPLATES: env.TEMPLATES_BUCKET_NAME,
   GLOBAL: env.GLOBAL_BUCKET_NAME,
-  GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
   PLUGINS: env.PLUGIN_BUCKET_NAME,
 }
 

packages/backend-core/src/redis/redlock.ts

@@ -13,6 +13,18 @@ const getClient = async (type: LockType): Promise<Redlock> => {
       }
       return noRetryRedlock
     }
+    case LockType.DEFAULT: {
+      if (!noRetryRedlock) {
+        noRetryRedlock = await newRedlock(OPTIONS.DEFAULT)
+      }
+      return noRetryRedlock
+    }
+    case LockType.DELAY_500: {
+      if (!noRetryRedlock) {
+        noRetryRedlock = await newRedlock(OPTIONS.DELAY_500)
+      }
+      return noRetryRedlock
+    }
     default: {
       throw new Error(`Could not get redlock client: ${type}`)
     }
@@ -41,6 +53,9 @@ export const OPTIONS = {
     // see https://www.awsarchitectureblog.com/2015/03/backoff.html
     retryJitter: 100, // time in ms
   },
+  DELAY_500: {
+    retryDelay: 500,
+  },
 }
 
 export const newRedlock = async (opts: Options = {}) => {
@@ -55,19 +70,17 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
   let lock
   try {
     // aquire lock
-    let name: string
+    let name: string = `lock:${tenancy.getTenantId()}_${opts.name}`
-    if (opts.systemLock) {
-      name = opts.name
-    } else {
-      name = `${tenancy.getTenantId()}_${opts.name}`
-    }
     if (opts.nameSuffix) {
       name = name + `_${opts.nameSuffix}`
     }
     lock = await redlock.lock(name, opts.ttl)
     // perform locked task
-    return task()
+    // need to await to ensure completion before unlocking
+    const result = await task()
+    return result
   } catch (e: any) {
+    console.log("lock error")
     // lock limit exceeded
     if (e.name === "LockError") {
       if (opts.type === LockType.TRY_ONCE) {

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,4 +1,4 @@
-import { doWithDB, queryPlatformView, getGlobalDBName } from "../db"
+import { doWithDB, getGlobalDBName } from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -8,11 +8,10 @@ import {
 import env from "../environment"
 import {
   BBContext,
-  PlatformUser,
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header, StaticDatabases, ViewName } from "../constants"
+import { Header, StaticDatabases } from "../constants"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@@ -111,27 +110,7 @@ export async function lookupTenantId(userId: string) {
   })
 }
 
-// lookup, could be email or userId, either will return a doc
+export const isUserInAppTenant = (appId: string, user?: any) => {
-export async function getTenantUser(
-  identifier: string
-): Promise<PlatformUser | undefined> {
-  // use the view here and allow to find anyone regardless of casing
-  // Use lowercase to ensure email login is case-insensitive
-  const users = await queryPlatformView<PlatformUser>(
-    ViewName.PLATFORM_USERS_LOWERCASE,
-    {
-      keys: [identifier.toLowerCase()],
-      include_docs: true,
-    }
-  )
-  if (Array.isArray(users)) {
-    return users[0]
-  } else {
-    return users
-  }
-}
-
-export function isUserInAppTenant(appId: string, user?: any) {
   let userTenantId
   if (user) {
     userTenantId = user.tenantId || DEFAULT_TENANT_ID

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,8 +1,8 @@
-const { structures } = require("../../tests")
+import { structures } from "../../../tests"
-const utils = require("../utils")
+import * as utils from "../../utils"
-const events = require("../events")
+import * as events from "../../events"
-const { DEFAULT_TENANT_ID } = require("../constants")
+import { DEFAULT_TENANT_ID } from "../../constants"
-const { doInTenant } = require("../context")
+import { doInTenant } from "../../context"
 
 describe("utils", () => {
   describe("platformLogout", () => {
@@ -14,4 +14,4 @@ describe("utils", () => {
       })
     })
   })
-})
+})

packages/backend-core/src/utils/utils.ts

@@ -1,6 +1,13 @@
 import { getAllApps, queryGlobalView } from "../db"
 import { options } from "../middleware/passport/jwt"
-import { Header, Cookie, MAX_VALID_DATE } from "../constants"
+import {
+  Header,
+  Cookie,
+  MAX_VALID_DATE,
+  DocumentType,
+  SEPARATOR,
+  ViewName,
+} from "../constants"
 import env from "../environment"
 import * as userCache from "../cache/user"
 import { getSessionsForUser, invalidateSessions } from "../security/sessions"
@@ -8,12 +15,11 @@ import * as events from "../events"
 import * as tenancy from "../tenancy"
 import {
   App,
-  BBContext,
+  Ctx,
   PlatformLogoutOpts,
   TenantResolutionStrategy,
 } from "@budibase/types"
 import { SetOption } from "cookies"
-import { DocumentType, SEPARATOR, ViewName } from "../constants"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
@@ -25,7 +31,7 @@ function confirmAppId(possibleAppId: string | undefined) {
     : undefined
 }
 
-async function resolveAppUrl(ctx: BBContext) {
+async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
@@ -50,7 +56,7 @@ async function resolveAppUrl(ctx: BBContext) {
   return app && app.appId ? app.appId : undefined
 }
 
-export function isServingApp(ctx: BBContext) {
+export function isServingApp(ctx: Ctx) {
   // dev app
   if (ctx.path.startsWith(`/${APP_PREFIX}`)) {
     return true
@@ -67,7 +73,7 @@ export function isServingApp(ctx: BBContext) {
  * @param {object} ctx The main request body to look through.
  * @returns {string|undefined} If an appId was found it will be returned.
  */
-export async function getAppIdFromCtx(ctx: BBContext) {
+export async function getAppIdFromCtx(ctx: Ctx) {
   // look in headers
   const options = [ctx.headers[Header.APP_ID]]
   let appId
@@ -83,12 +89,16 @@ export async function getAppIdFromCtx(ctx: BBContext) {
     appId = confirmAppId(ctx.request.body.appId)
   }
 
-  // look in the url - dev app
+  // look in the path
-  let appPath =
+  const pathId = parseAppIdFromUrl(ctx.path)
-    ctx.request.headers.referrer ||
+  if (!appId && pathId) {
-    ctx.path.split("/").filter(subPath => subPath.startsWith(APP_PREFIX))
+    appId = confirmAppId(pathId)
-  if (!appId && appPath.length) {
+  }
-    appId = confirmAppId(appPath[0])
+
+  // look in the referer
+  const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
+  if (!appId && refererId) {
+    appId = confirmAppId(refererId)
   }
 
   // look in the url - prod app
@@ -99,6 +109,13 @@ export async function getAppIdFromCtx(ctx: BBContext) {
   return appId
 }
 
+function parseAppIdFromUrl(url?: string) {
+  if (!url) {
+    return
+  }
+  return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX))
+}
+
 /**
  * opens the contents of the specified encrypted JWT.
  * @return {object} the contents of the token.
@@ -115,7 +132,7 @@ export function openJwt(token: string) {
  * @param {object} ctx The request which is to be manipulated.
  * @param {string} name The name of the cookie to get.
  */
-export function getCookie(ctx: BBContext, name: string) {
+export function getCookie(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
@@ -133,7 +150,7 @@ export function getCookie(ctx: BBContext, name: string) {
  * @param {object} opts options like whether to sign.
  */
 export function setCookie(
-  ctx: BBContext,
+  ctx: Ctx,
   value: any,
   name = "builder",
   opts = { sign: true }
@@ -159,7 +176,7 @@ export function setCookie(
 /**
  * Utility function, simply calls setCookie with an empty string for value
  */
-export function clearCookie(ctx: BBContext, name: string) {
+export function clearCookie(ctx: Ctx, name: string) {
   setCookie(ctx, null, name)
 }
 
@@ -169,7 +186,7 @@ export function clearCookie(ctx: BBContext, name: string) {
  * @param {object} ctx The koa context object to be tested.
  * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
  */
-export function isClient(ctx: BBContext) {
+export function isClient(ctx: Ctx) {
   return ctx.headers[Header.TYPE] === "client"
 }
 

packages/backend-core/tests/jestSetup.ts

@@ -17,7 +17,9 @@ env._set("MINIO_URL", "http://localhost")
 env._set("MINIO_ACCESS_KEY", "test")
 env._set("MINIO_SECRET_KEY", "test")
 
-global.console.log = jest.fn() // console.log are ignored in tests
+if (!process.env.DEBUG) {
+  global.console.log = jest.fn() // console.log are ignored in tests
+}
 
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging

packages/backend-core/tests/utilities/index.ts

@@ -1,6 +1,7 @@
 export * as mocks from "./mocks"
 export * as structures from "./structures"
 export { generator } from "./structures"
+export * as testEnv from "./testEnv"
 
 import * as dbConfig from "./db"
 dbConfig.init()

packages/backend-core/tests/utilities/mocks/events.ts

@@ -117,3 +117,7 @@ jest.spyOn(events.view, "filterDeleted")
 jest.spyOn(events.view, "calculationCreated")
 jest.spyOn(events.view, "calculationUpdated")
 jest.spyOn(events.view, "calculationDeleted")
+
+jest.spyOn(events.plugin, "init")
+jest.spyOn(events.plugin, "imported")
+jest.spyOn(events.plugin, "deleted")

packages/backend-core/tests/utilities/mocks/index.ts

@@ -1,5 +1,6 @@
-import "./posthog"
-import "./events"
 export * as accounts from "./accounts"
 export * as date from "./date"
+export * as licenses from "./licenses"
 export { default as fetch } from "./fetch"
+import "./posthog"
+import "./events"

packages/backend-core/tests/utilities/mocks/licenses.ts

@@ -0,0 +1,83 @@
+import { Feature, License, Quotas } from "@budibase/types"
+import _ from "lodash"
+
+let CLOUD_FREE_LICENSE: License
+let UNLIMITED_LICENSE: License
+let getCachedLicense: any
+
+// init for the packages other than pro
+export function init(proPkg: any) {
+  initInternal({
+    CLOUD_FREE_LICENSE: proPkg.constants.licenses.CLOUD_FREE_LICENSE,
+    UNLIMITED_LICENSE: proPkg.constants.licenses.UNLIMITED_LICENSE,
+    getCachedLicense: proPkg.licensing.cache.getCachedLicense,
+  })
+}
+
+// init for the pro package
+export function initInternal(opts: {
+  CLOUD_FREE_LICENSE: License
+  UNLIMITED_LICENSE: License
+  getCachedLicense: any
+}) {
+  CLOUD_FREE_LICENSE = opts.CLOUD_FREE_LICENSE
+  UNLIMITED_LICENSE = opts.UNLIMITED_LICENSE
+  getCachedLicense = opts.getCachedLicense
+}
+
+export interface UseLicenseOpts {
+  features?: Feature[]
+  quotas?: Quotas
+}
+
+// LICENSES
+
+export const useLicense = (license: License, opts?: UseLicenseOpts) => {
+  if (opts) {
+    if (opts.features) {
+      license.features.push(...opts.features)
+    }
+    if (opts.quotas) {
+      license.quotas = opts.quotas
+    }
+  }
+
+  getCachedLicense.mockReturnValue(license)
+
+  return license
+}
+
+export const useUnlimited = (opts?: UseLicenseOpts) => {
+  return useLicense(UNLIMITED_LICENSE, opts)
+}
+
+export const useCloudFree = () => {
+  return useLicense(CLOUD_FREE_LICENSE)
+}
+
+// FEATURES
+
+const useFeature = (feature: Feature) => {
+  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  const opts: UseLicenseOpts = {
+    features: [feature],
+  }
+
+  return useLicense(license, opts)
+}
+
+export const useBackups = () => {
+  return useFeature(Feature.APP_BACKUPS)
+}
+
+export const useGroups = () => {
+  return useFeature(Feature.USER_GROUPS)
+}
+
+// QUOTAS
+
+export const setAutomationLogsQuota = (value: number) => {
+  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  license.quotas.constant.automationLogRetentionDays.value = value
+  return useLicense(license)
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -6,3 +6,4 @@ export const generator = new Chance()
 export * as koa from "./koa"
 export * as accounts from "./accounts"
 export * as licenses from "./licenses"
+export * as plugins from "./plugins"

packages/backend-core/tests/utilities/structures/plugins.ts

@@ -0,0 +1,19 @@
+import { generator } from "."
+import { Plugin, PluginSource, PluginType } from "@budibase/types"
+
+export function plugin(): Plugin {
+  return {
+    description: generator.word(),
+    name: generator.word(),
+    version: "1.0.0",
+    source: PluginSource.FILE,
+    package: {
+      name: generator.word,
+    },
+    hash: generator.hash(),
+    schema: {
+      type: PluginType.DATASOURCE,
+    },
+    iconFileName: "icon.svg",
+  }
+}

packages/backend-core/tests/utilities/testEnv.ts

@@ -0,0 +1,87 @@
+import env from "../../src/environment"
+import * as tenancy from "../../src/tenancy"
+import { newid } from "../../src/utils"
+
+// TENANCY
+
+export async function withTenant(task: (tenantId: string) => any) {
+  const tenantId = newid()
+  return tenancy.doInTenant(tenantId, async () => {
+    await task(tenantId)
+  })
+}
+
+export function singleTenant() {
+  env._set("MULTI_TENANCY", 0)
+}
+
+export function multiTenant() {
+  env._set("MULTI_TENANCY", 1)
+}
+
+// NODE
+
+export function nodeDev() {
+  env._set("NODE_ENV", "dev")
+}
+
+export function nodeJest() {
+  env._set("NODE_ENV", "jest")
+}
+
+// FILES
+
+export function withS3() {
+  env._set("NODE_ENV", "production")
+  env._set("MINIO_ENABLED", 0)
+  env._set("MINIO_URL", "http://s3.example.com")
+  env._set("CLOUDFRONT_CDN", undefined)
+}
+
+const CLOUDFRONT_TEST_KEY =
+  "-----BEGIN RSA PRIVATE KEY-----\n" +
+  "MIIEpAIBAAKCAQEAqXRsir/0Qba1xEnybUs7d7QEAE02GRc+4H7HD5l5VnAxkV1m\n" +
+  "tNTXTmoYkaIhLdebV1EwQs3T9knxoyd4cVcrDkDfDLZErfYWJsuE3/QYNknnZs4/\n" +
+  "Ai0cg+v9ZX3gcizvpYg9GQI3INM0uRG8lJwGP7FQ/kknhA2yVFVCSxX6kkNtOUh5\n" +
+  "dKSG7m6IwswcSwD++Z/94vsFkoZIGY0e1CD/drFJ6+1TFY2YgbDKT5wDFLJ9vHFx\n" +
+  "/5o4POwn3gz/ru2Db9jbRdfEAqRdy46nRKQgBGUmupAgSK1+BJEzafexp8RmCGb0\n" +
+  "WUffxOtj8/jNCeCF0JBgVHAe3crOQ8ySrtoaHQIDAQABAoIBAA+ipW07/u6dTDI7\n" +
+  "XHoHKgqGeqQIe8he47dVG0ruL0rxeTFfe92NkfwzP+cYHZWcQkIRRLG1Six8cCZM\n" +
+  "uwlCML/U7n++xaGDhlG4D5+WZzGDKi3LM/cgcHQfrzbRIYeHa+lLI9AN60ZFFqVI\n" +
+  "5KyVpOH1m3KLD3FYzi6H22EQOxmJpqWlt2uArny5LxlPJKmmGSFjvneb4N2ZAKGQ\n" +
+  "QfClJGz9tRjceWUUdJrpqmTmBQIosKmLPq8PEviUNAVG+6m4r8jiRbf8OKkAm+3L\n" +
+  "LVIsN8HfYB9jEuERYPnbuXdX0kDEkg0xEyTH5YbNZvfm5ptCU9Xn+Jz1trF+wCHD\n" +
+  "2RlxdQUCgYEA3U0nCf6NTmmeMCsAX6gvaPuM0iUfUfS3b3G57I6u46lLGNLsfJw6\n" +
+  "MTpVc164lKYQK9czw/ijKzb8e3mcyzbPorVkajMjUCNWGrMK+vFbOGmqQkhUi30U\n" +
+  "IJuuTktMd+21D/SpLlev4MLria23vUIKEqNenYpV6wkGLt/mKtISaPMCgYEAxAYx\n" +
+  "j+xJLTK9eN+rpekwjYE78hD9VoBkBnr/NBiGV302AsJRuq2+L4zcBnAsH+SidFim\n" +
+  "cwqoj3jeVT8ZQFXlK3fGVaEJsCXd6GWk8ZIWUTn9JZwi2KcCvCU/YiHfx8c7y7Gl\n" +
+  "SiPXUPsvvkcw6RRh2u4J5tHLIqJe3W58ENoBNK8CgYEApxTBDMKrXTBQxn0w4wfQ\n" +
+  "A6soPuDYLMBeXj226eswD6KZmDxnYA1zwgcQzPIO2ewm+XKZGrR2PQJezbqbrrHL\n" +
+  "QkVBcwz49GA5eh8Dg0MGZCki6rhBXK8qqxPfHi2rpkBKG6nUsbBykXeY7XHC75kU\n" +
+  "kc3WeYsgIzvE908EMAA69hECgYEAinbpiYVZh1DBH+G26MIYZswz4OB5YyHcBevZ\n" +
+  "2x27v48VmMtUWe4iWopAXVfdA0ZILrD0Gm0b9gRl4IdqudQyxgqcEZ5oLoIBBwjN\n" +
+  "g0oy83tnwqpQvwLx3p7c79+HqCGmrlK0s/MvQ+e6qMi21t1r5e6hFed5euSA6B8E\n" +
+  "Cg9ELMcCgYB9bGwlNAE+iuzMIhKev1s7h3TzqKtGw37TtHXvxcTQs3uawJQksQ2s\n" +
+  "K0Zy1Ta7vybbwAA5m+LxoMT04WUdJO7Cr8/3rBMrbKKO3H7IgC3G+nXnOBdshzn5\n" +
+  "ifMbhZslFThC/osD5ZV7snXZgTWyPexaINJhHmdrAWpmW1h+UFoiMw==\n" +
+  "-----END RSA PRIVATE KEY-----\n"
+
+const CLOUDFRONT_TEST_KEY_64 = Buffer.from(
+  CLOUDFRONT_TEST_KEY,
+  "utf-8"
+).toString("base64")
+
+export function withCloudfront() {
+  withS3()
+  env._set("CLOUDFRONT_CDN", "http://cf.example.com")
+  env._set("CLOUDFRONT_PUBLIC_KEY_ID", "keypair_123")
+  env._set("CLOUDFRONT_PRIVATE_KEY_64", CLOUDFRONT_TEST_KEY_64)
+}
+
+export function withMinio() {
+  env._set("NODE_ENV", "production")
+  env._set("MINIO_ENABLED", 1)
+  env._set("MINIO_URL", "http://minio.example.com")
+  env._set("CLOUDFRONT_CDN", undefined)
+}

packages/backend-core/tsconfig.json

@@ -8,6 +8,10 @@
     }
   },
   "references": [
-    { "path": "../types" },
+    { "path": "../types" }
+  ],
+  "exclude": [
+    "node_modules",
+    "dist"
   ]
 }

packages/backend-core/yarn.lock

@@ -470,6 +470,23 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/nano@10.1.1":
+  version "10.1.1"
+  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.1.tgz#36ccda4d9bb64b5ee14dd2b27a295b40739b1038"
+  integrity sha512-kbMIzMkjVtl+xI0UPwVU0/pn8/ccxTyfzwBz6Z+ZiN2oUSb0fJCe0qwA6o8dxwSa8nZu4MbGAeMJl3CJndmWtA==
+  dependencies:
+    "@types/tough-cookie" "^4.0.2"
+    axios "^1.1.3"
+    http-cookie-agent "^4.0.2"
+    node-abort-controller "^3.0.1"
+    qs "^6.11.0"
+    tough-cookie "^4.1.2"
+
+"@budibase/types@2.2.12-alpha.2":
+  version "2.2.12-alpha.2"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.2.tgz#390959555517f11dc95147518e89f400c59dd520"
+  integrity sha512-Dz/3kdSkBRyf+AQyaPbAwq6hxD1kQuHht+4C3pyebbHUL440ZhZhJ1o2AAFc8SF1/9SqAk/NFaktAY2vCyRlOQ==
+
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
@@ -1526,6 +1543,13 @@ asynckit@^0.4.0:
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
   integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
 
+aws-cloudfront-sign@2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/aws-cloudfront-sign/-/aws-cloudfront-sign-2.2.0.tgz#3910f5a6d0d90fec07f2b4ef8ab07f3eefb5625d"
+  integrity sha512-qG+rwZMP3KRTPPbVmWY8DlrT56AkA4iVOeo23vkdK2EXeW/brJFN2haSNKzVz+oYhFMEIzVVloeAcrEzuRkuVQ==
+  dependencies:
+    lodash "^3.6.0"
+
 aws-sdk@2.1030.0:
   version "2.1030.0"
   resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1030.0.tgz#24a856af3d2b8b37c14a8f59974993661c66fd82"
@@ -3527,11 +3551,21 @@ json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
   integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
 
 json5@^2.2.1:
-  version "2.2.1"
+  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
-  integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==
+  integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
 
-jsonwebtoken@8.5.1, jsonwebtoken@^8.2.0:
+jsonwebtoken@9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
+  integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
+  dependencies:
+    jws "^3.2.2"
+    lodash "^4.17.21"
+    ms "^2.1.1"
+    semver "^7.3.8"
+
+jsonwebtoken@^8.2.0:
   version "8.5.1"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
   integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==
@@ -3827,6 +3861,11 @@ lodash@4.17.21, lodash@^4.17.21:
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
 
+lodash@^3.6.0:
+  version "3.10.1"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
+  integrity sha512-9mDDwqVIma6OZX79ZlDACZl8sBm0TEnkf99zV3iMA4GzkIT/9hiqP5mY0HoT1iNLCrKc/R1HByV+yJfRWVJryQ==
+
 lowercase-keys@^1.0.0, lowercase-keys@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f"
@@ -3850,9 +3889,9 @@ ltgt@2.2.1, ltgt@^2.1.2, ltgt@~2.2.0:
   integrity sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==
 
 luxon@^3.0.1:
-  version "3.0.4"
+  version "3.2.1"
-  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.2.1.tgz#14f1af209188ad61212578ea7e3d518d18cee45f"
-  integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw==
+  integrity sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==
 
 make-dir@^3.0.0, make-dir@^3.1.0:
   version "3.1.0"
@@ -4022,18 +4061,6 @@ msgpackr@^1.5.2:
   optionalDependencies:
     msgpackr-extract "^2.1.2"
 
-nano@^10.1.0:
-  version "10.1.0"
-  resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.0.tgz#afdd5a7440e62f09a8e23f41fcea328d27383922"
-  integrity sha512-COeN2TpLcHuSN44QLnPmfZCoCsKAg8/aelPOVqqm/2/MvRHDEA11/Kld5C4sLzDlWlhFZ3SO2WGJGevCsvcEzQ==
-  dependencies:
-    "@types/tough-cookie" "^4.0.2"
-    axios "^1.1.3"
-    http-cookie-agent "^4.0.2"
-    node-abort-controller "^3.0.1"
-    qs "^6.11.0"
-    tough-cookie "^4.1.2"
-
 napi-macros@~2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b"
@@ -4990,13 +5017,20 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-semver@7.3.7, semver@7.x, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5:
+semver@7.3.7:
   version "7.3.7"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
   integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==
   dependencies:
     lru-cache "^6.0.0"
 
+semver@7.x, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.8:
+  version "7.3.8"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.8.tgz#07a78feafb3f7b32347d725e33de7e2a2df67798"
+  integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==
+  dependencies:
+    lru-cache "^6.0.0"
+
 semver@^5.6.0, semver@^5.7.1:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.1.43-alpha.15",
+  "version": "2.2.12-alpha.4",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -37,8 +37,8 @@
     "dist"
   ],
   "dependencies": {
-    "@adobe/spectrum-css-workflow-icons": "^1.2.1",
+    "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.1.43-alpha.15",
+    "@budibase/string-templates": "2.2.12-alpha.4",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",
     "@spectrum-css/avatar": "3.0.2",

packages/bbui/src/Badge/Badge.svelte

@@ -10,10 +10,13 @@
   export let green = false
   export let active = false
   export let inactive = false
+  export let hoverable = false
 </script>
 
 <span
+  on:click
   class="spectrum-Label"
+  class:hoverable
   class:spectrum-Label--small={size === "S"}
   class:spectrum-Label--large={size === "L"}
   class:spectrum-Label--grey={grey}
@@ -27,3 +30,13 @@
 >
   <slot />
 </span>
+
+<style>
+  .spectrum-Label--grey {
+    background-color: var(--spectrum-global-color-gray-500);
+    font-weight: 600;
+  }
+  .hoverable:hover {
+    cursor: pointer;
+  }
+</style>

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -15,6 +15,7 @@
   export let value = []
   export let id = null
   export let disabled = false
+  export let compact = false
   export let fileSizeLimit = BYTES_IN_MB * 20
   export let processFiles = null
   export let deleteAttachments = null
@@ -239,70 +240,72 @@
           bind:this={fileInput}
           on:change={handleFile}
         />
-        <svg
+        {#if !compact}
-          class="spectrum-IllustratedMessage-illustration"
+          <svg
-          width="125"
+            class="spectrum-IllustratedMessage-illustration"
-          height="60"
+            width="125"
-          viewBox="0 0 199 97.7"
+            height="60"
-        >
+            viewBox="0 0 199 97.7"
-          <defs>
+          >
-            <style>
+            <defs>
-              .cls-1,
+              <style>
-              .cls-2 {
+                .cls-1,
-                fill: none;
+                .cls-2 {
-                stroke-linecap: round;
+                  fill: none;
-                stroke-linejoin: round;
+                  stroke-linecap: round;
-              }
+                  stroke-linejoin: round;
-              .cls-1 {
+                }
-                stroke-width: 3px;
+                .cls-1 {
-              }
+                  stroke-width: 3px;
-              .cls-2 {
+                }
-                stroke-width: 2px;
+                .cls-2 {
-              }
+                  stroke-width: 2px;
-            </style>
+                }
-          </defs>
+              </style>
-          <path
+            </defs>
-            class="cls-1"
+            <path
-            d="M110.53,85.66,100.26,95.89a1.09,1.09,0,0,1-1.52,0L88.47,85.66"
+              class="cls-1"
-          />
+              d="M110.53,85.66,100.26,95.89a1.09,1.09,0,0,1-1.52,0L88.47,85.66"
-          <line class="cls-1" x1="99.5" y1="95.5" x2="99.5" y2="58.5" />
+            />
-          <path class="cls-1" d="M105.5,73.5h19a2,2,0,0,0,2-2v-43" />
+            <line class="cls-1" x1="99.5" y1="95.5" x2="99.5" y2="58.5" />
-          <path
+            <path class="cls-1" d="M105.5,73.5h19a2,2,0,0,0,2-2v-43" />
-            class="cls-1"
+            <path
-            d="M126.5,22.5h-19a2,2,0,0,1-2-2V1.5h-31a2,2,0,0,0-2,2v68a2,2,0,0,0,2,2h19"
+              class="cls-1"
-          />
+              d="M126.5,22.5h-19a2,2,0,0,1-2-2V1.5h-31a2,2,0,0,0-2,2v68a2,2,0,0,0,2,2h19"
-          <line class="cls-1" x1="105.5" y1="1.5" x2="126.5" y2="22.5" />
+            />
-          <path
+            <line class="cls-1" x1="105.5" y1="1.5" x2="126.5" y2="22.5" />
-            class="cls-2"
+            <path
-            d="M47.93,50.49a5,5,0,1,0-4.83-5A4.93,4.93,0,0,0,47.93,50.49Z"
+              class="cls-2"
-          />
+              d="M47.93,50.49a5,5,0,1,0-4.83-5A4.93,4.93,0,0,0,47.93,50.49Z"
-          <path
+            />
-            class="cls-2"
+            <path
-            d="M36.6,65.93,42.05,60A2.06,2.06,0,0,1,45,60l12.68,13.2"
+              class="cls-2"
-          />
+              d="M36.6,65.93,42.05,60A2.06,2.06,0,0,1,45,60l12.68,13.2"
-          <path
+            />
-            class="cls-2"
+            <path
-            d="M3.14,73.23,22.42,53.76a1.65,1.65,0,0,1,2.38,0l19.05,19.7"
+              class="cls-2"
-          />
+              d="M3.14,73.23,22.42,53.76a1.65,1.65,0,0,1,2.38,0l19.05,19.7"
-          <path
+            />
-            class="cls-1"
+            <path
-            d="M139.5,36.5H196A1.49,1.49,0,0,1,197.5,38V72A1.49,1.49,0,0,1,196,73.5H141A1.49,1.49,0,0,1,139.5,72V32A1.49,1.49,0,0,1,141,30.5H154a2.43,2.43,0,0,1,1.67.66l6,5.66"
+              class="cls-1"
-          />
+              d="M139.5,36.5H196A1.49,1.49,0,0,1,197.5,38V72A1.49,1.49,0,0,1,196,73.5H141A1.49,1.49,0,0,1,139.5,72V32A1.49,1.49,0,0,1,141,30.5H154a2.43,2.43,0,0,1,1.67.66l6,5.66"
-          <rect
+            />
-            class="cls-1"
+            <rect
-            x="1.5"
+              class="cls-1"
-            y="34.5"
+              x="1.5"
-            width="58"
+              y="34.5"
-            height="39"
+              width="58"
-            rx="2"
+              height="39"
-            ry="2"
+              rx="2"
-          />
+              ry="2"
-        </svg>
+            />
-        <h2
+          </svg>
-          class="spectrum-Heading spectrum-Heading--sizeL spectrum-Heading--light spectrum-IllustratedMessage-heading"
+          <h2
-        >
+            class="spectrum-Heading spectrum-Heading--sizeL spectrum-Heading--light spectrum-IllustratedMessage-heading"
-          Drag and drop your file
+          >
-        </h2>
+            Drag and drop your file
+          </h2>
+        {/if}
         {#if !disabled}
           <p
             class="spectrum-Body spectrum-Body--sizeS spectrum-IllustratedMessage-description"
@@ -310,8 +313,10 @@
             <label for={fieldId} class="spectrum-Link">
               Select a file to upload
             </label>
-            <br />
+            {#if !compact}
-            from your computer
+              <br />
+              from your computer
+            {/if}
           </p>
           {#if fileTags.length}
             <Tags>

packages/bbui/src/Table/RelationshipRenderer.svelte

@@ -1,6 +1,7 @@
 <script>
   import "@spectrum-css/label/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
+  import Badge from "../Badge/Badge.svelte"
 
   export let row
   export let value
@@ -24,17 +25,11 @@
 
 {#each relationships as relationship}
   {#if relationship?.primaryDisplay}
-    <span class="spectrum-Label spectrum-Label--grey" on:click={onClick}>
+    <Badge hoverable grey on:click={onClick}>
       {relationship.primaryDisplay}
-    </span>
+    </Badge>
   {/if}
 {/each}
 {#if leftover}
   <div>+{leftover} more</div>
 {/if}
-
-<style>
-  span:hover {
-    cursor: pointer;
-  }
-</style>

packages/builder/cypress/integration/adminAndManagement/accountPortals.spec.js

@@ -2,18 +2,18 @@ import filterTests from "../../support/filterTests"
 const interact = require('../../support/interact')
 
 filterTests(["smoke", "all"], () => {
-  context("Account Portals", () => {
+  xcontext("Account Portals", () => {
-    
+
     const bbUserEmail = "bbuser@test.com"
-    
+
     before(() => {
       cy.login()
       cy.deleteApp("Cypress Tests")
       cy.createApp("Cypress Tests", false)
-      
+
       // Create new user
       cy.wait(500)
-      cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000})
+      cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.createUser(bbUserEmail)
       cy.contains("bbuser").click()
       cy.wait(500)
@@ -25,18 +25,18 @@ filterTests(["smoke", "all"], () => {
       cy.get(interact.SPECTRUM_MENU).within(() => {
         cy.get(interact.SPECTRUM_MENU_ITEM).contains("Force password reset").click({ force: true })
       })
-      
+
       cy.get(interact.SPECTRUM_DIALOG_GRID)
-      .find(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').as('pwd')
+        .find(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').as('pwd')
 
       cy.get(interact.SPECTRUM_BUTTON).contains("Reset password").click({ force: true })
-      
+
       // Login as new user and set password
       cy.logOut()
       cy.get('@pwd').then((pwd) => {
         cy.login(bbUserEmail, pwd)
       })
-      
+
       for (let i = 0; i < 2; i++) {
         cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).eq(i).type("test")
       }
@@ -44,7 +44,7 @@ filterTests(["smoke", "all"], () => {
       //cy.logoutNoAppGrid()
     })
 
-    it("should verify Standard Portal", () => {
+    xit("should verify Standard Portal", () => {
       // Development access should be disabled (Admin access is already disabled)
       cy.login()
       cy.setUserRole("bbuser", "App User")
@@ -58,15 +58,15 @@ filterTests(["smoke", "all"], () => {
 
       cy.logoutNoAppGrid()
     })
-    
-    it("should verify Admin Portal", () => {
-        cy.login()
-        // Configure user role
-        cy.setUserRole("bbuser", "Admin")
-        bbUserLogin()
 
-        // Verify available options for Admin portal
+    xit("should verify Admin Portal", () => {
-        cy.get(interact.SPECTRUM_SIDENAV)
+      cy.login()
+      // Configure user role
+      cy.setUserRole("bbuser", "Admin")
+      bbUserLogin()
+
+      // Verify available options for Admin portal
+      cy.get(interact.SPECTRUM_SIDENAV)
         .should('contain', 'Apps')
         //.and('contain', 'Usage')
         .and('contain', 'Users')
@@ -75,12 +75,12 @@ filterTests(["smoke", "all"], () => {
         .and('contain', 'Organisation')
         .and('contain', 'Theming')
         .and('contain', 'Update')
-        //.and('contain', 'Upgrade')
+      //.and('contain', 'Upgrade')
 
-        cy.logOut()
+      cy.logOut()
     })
 
-    it("should verify Development Portal", () => {
+    xit("should verify Development Portal", () => {
       // Only Development access should be enabled
       cy.login()
       cy.setUserRole("bbuser", "Developer")
@@ -98,7 +98,7 @@ filterTests(["smoke", "all"], () => {
         .and('not.contain', 'Update')
         .and('not.contain', 'Upgrade')
 
-        cy.logOut()
+      cy.logOut()
     })
 
     const bbUserLogin = () => {

packages/builder/cypress/integration/appOverview.spec.js

@@ -9,7 +9,7 @@ filterTests(["all"], () => {
       cy.createApp("Cypress Tests")
     })
 
-    it("Should be accessible from the applications list", () => {
+    xit("Should be accessible from the applications list", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .title")
         .eq(0)
@@ -27,7 +27,7 @@ filterTests(["all"], () => {
     })
 
     // Find a more suitable place for this.
-    it("Should allow unlocking in the app list", () => {
+    xit("Should allow unlocking in the app list", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(".appTable .lock-status").eq(0).contains("Locked by you").click()
@@ -38,7 +38,7 @@ filterTests(["all"], () => {
       cy.get(".lock-status").should("not.be.visible")
     })
 
-    it("Should allow unlocking in the app overview screen", () => {
+    xit("Should allow unlocking in the app overview screen", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(".appTable .app-row-actions button")
@@ -58,7 +58,7 @@ filterTests(["all"], () => {
       cy.get(".lock-status").should("not.be.visible")
     })
 
-    it("Should reflect the deploy state of an app that hasn't been published.", () => {
+    xit("Should reflect the deploy state of an app that hasn't been published.", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(".appTable .app-row-actions button")
@@ -81,7 +81,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect the app deployment state", () => {
+    xit("Should reflect the app deployment state", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.get(".appTable .app-row-actions button")
         .contains("Edit")
@@ -117,7 +117,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect an application that has been unpublished", () => {
+    xit("Should reflect an application that has been unpublished", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Edit")
@@ -154,7 +154,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should allow the editing of the application icon and colour", () => {
+    xit("Should allow the editing of the application icon and colour", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -196,7 +196,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect the last time the application was edited", () => {
+    xit("Should reflect the last time the application was edited", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -221,7 +221,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect application version is up-to-date", () => {
+    xit("Should reflect application version is up-to-date", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -302,7 +302,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should allow editing of the app details.", () => {
+    xit("Should allow editing of the app details.", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -373,13 +373,13 @@ filterTests(["all"], () => {
             .contains("Copy App ID")
             .click({ force: true })
         })
-      
+
       cy.get(".spectrum-Toast-content")
-      .contains("App ID copied to clipboard.")
+        .contains("App ID copied to clipboard.")
-      .should("be.visible")
+        .should("be.visible")
     })
 
-    it("Should allow unpublishing of the application via the Unpublish link", () => {
+    xit("Should allow unpublishing of the application via the Unpublish link", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -388,7 +388,7 @@ filterTests(["all"], () => {
 
       cy.get(`[data-cy="app-status"]`).within(() => {
         cy.contains("Unpublish").click({ force: true })
-        })
+      })
 
       cy.get("[data-cy='unpublish-modal']")
         .should("be.visible")
@@ -399,11 +399,11 @@ filterTests(["all"], () => {
       cy.get(".overview-tab [data-cy='app-status']").within(() => {
         cy.get(".status-display").contains("Unpublished")
         cy.get(".status-display .icon svg[aria-label='GlobeStrike']")
-        .should("exist")
+          .should("exist")
       })
     })
 
-    it("Should allow deleting of the application", () => {
+    xit("Should allow deleting of the application", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")

packages/builder/cypress/integration/appPublishWorkflow.spec.js

@@ -3,100 +3,101 @@ import { APP_TABLE_APP_NAME, DEPLOY_SUCCESS_MODAL } from "../support/interact";
 const interact = require('../support/interact')
 
 filterTests(['all'], () => {
-  context("Publish Application Workflow", () => {
+  xcontext("Publish Application Workflow", () => {
     before(() => {
       cy.login()
       cy.deleteAllApps()
       cy.createApp("Cypress Tests", false)
     })
 
-    it("Should reflect the unpublished status correctly", () => {
+    xit("Should reflect the unpublished status correctly", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
 
       cy.get(interact.APP_TABLE_STATUS, { timeout: 3000 }).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.contains("Unpublished")
+          cy.contains("Unpublished")
-        cy.get(interact.GLOBESTRIKE).should("exist")
+          cy.get(interact.GLOBESTRIKE).should("exist")
-      })
+        })
 
       cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Edit").click({ force: true })
+          cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Edit").click({ force: true })
-      })
+        })
 
       cy.get(interact.DEPLOYMENT_TOP_NAV_GLOBESTRIKE).should("exist")
       cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("not.exist")
     })
 
-    it("Should publish an application and correctly reflect that", () => {
+    xit("Should publish an application and correctly reflect that", () => {
       //Assuming the previous test was run and the unpublished app is open in edit mode.
-      cy.get(interact.TOPRIGHTNAV_BUTTON_SPECTRUM).contains("Publish").click({ force : true })
+      cy.get(interact.TOPRIGHTNAV_BUTTON_SPECTRUM).contains("Publish").click({ force: true })
 
       cy.get(interact.DEPLOY_APP_MODAL).should("be.visible")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.SPECTRUM_BUTTON).contains("Publish").click({ force : true })
+          cy.get(interact.SPECTRUM_BUTTON).contains("Publish").click({ force: true })
-      });
+        });
 
       //Verify that the app url is presented correctly to the user
       cy.get(interact.DEPLOY_SUCCESS_MODAL, { timeout: 1000 })
-      .should("be.visible")
+        .should("be.visible")
-      .within(() => {
+        .within(() => {
-        let appUrl = Cypress.config().baseUrl + '/app/cypress-tests'
+          let appUrl = Cypress.config().baseUrl + '/app/cypress-tests'
-        cy.get(interact.DEPLOY_APP_URL_INPUT).should('have.value', appUrl)
+          cy.get(interact.DEPLOY_APP_URL_INPUT).should('have.value', appUrl)
-        cy.get(interact.SPECTRUM_BUTTON).contains("Done").click({ force: true })
+          cy.get(interact.SPECTRUM_BUTTON).contains("Done").click({ force: true })
-      })
+        })
 
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(interact.APP_TABLE_STATUS, { timeout: 3000 }).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.contains("Published")
+          cy.contains("Published")
-        cy.get(interact.GLOBE).should("exist")
+          cy.get(interact.GLOBE).should("exist")
-      })
+        })
 
       cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.get(interact.SPECTRUM_BUTTON).contains("Manage")
+          cy.get(interact.SPECTRUM_BUTTON).contains("Manage")
-        cy.get(interact.SPECTRUM_BUTTON).contains("Edit").click({ force: true })
+          cy.get(interact.SPECTRUM_BUTTON).contains("Edit").click({ force: true })
-      })
+        })
 
       cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist").click({ force: true })
 
       cy.get(interact.PUBLISH_POPOVER_MENU).should("be.visible")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.PUBLISH_POPOVER_ACTION).should("exist")
+          cy.get(interact.PUBLISH_POPOVER_ACTION).should("exist")
-        cy.get("button").contains("View app").should("exist")
+          cy.get("button").contains("View app").should("exist")
-        cy.get(interact.PUBLISH_POPOVER_MESSAGE).should("have.text", "Last published a few seconds ago")
+          cy.get(interact.PUBLISH_POPOVER_MESSAGE).should("have.text", "Last published a few seconds ago")
-      })
+        })
     })
 
-    it("Should unpublish an application using the link and reflect the status change", () => {
+    xit("Should unpublish an application using the link and reflect the status change", () => {
       //Assuming the previous test app exists and is published
 
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
 
       cy.get(interact.APP_TABLE_STATUS).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.contains("Published")
+          cy.contains("Published")
-        cy.get("svg[aria-label='Globe']").should("exist")
+          cy.get("svg[aria-label='Globe']").should("exist")
-      })
+        })
 
       cy.get(interact.APP_TABLE).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.get(interact.APP_TABLE_APP_NAME).click({ force: true })
+          cy.get(interact.APP_TABLE_APP_NAME).click({ force: true })
-      })
+        })
-      
+
       cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist").click({ force: true })
-      
+
       cy.get("[data-cy='publish-popover-menu']")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.PUBLISH_POPOVER_ACTION).click({ force: true })
+          cy.get(interact.PUBLISH_POPOVER_ACTION).click({ force: true })
-      })
+        })
 
       cy.get(interact.UNPUBLISH_MODAL).should("be.visible")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.CONFIRM_WRAP_BUTTON).click({ force: true }
+          cy.get(interact.CONFIRM_WRAP_BUTTON).click({ force: true }
-      )})
+          )
+        })
 
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 6000 })
       cy.wait(500)

packages/builder/cypress/integration/createAutomation.spec.js

@@ -2,13 +2,13 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
-  context("Create a automation", () => {
+  xcontext("Create a automation", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
     })
 
-    it("should create a automation", () => {
+    xit("should create a automation", () => {
       cy.createTestTableWithData()
       cy.wait(2000)
       cy.contains("Automate").click()

packages/builder/cypress/integration/createView.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
-  context("Create a View", () => {
+  xcontext("Create a View", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -20,23 +20,25 @@ filterTests(['smoke', 'all'], () => {
       cy.addRow(["Teachers", 36, 3])
     })
 
-    it("creates a view", () => {
+    xit("creates a view", () => {
       cy.contains("Create view").click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.get("input").type("Test View")
         cy.get("button").contains("Create View").click({ force: true })
       })
-      cy.get(interact.TABLE_TITLE_H1).contains("Test View")
+      cy.contains(interact.TABLE_TITLE_H1, "Test View", { timeout: 10000 })
-      cy.get(interact.TITLE).then($headers => {
+      cy.get(".table-wrapper").within(() => {
-        expect($headers).to.have.length(3)
+        cy.get(interact.TITLE).then($headers => {
-        const headers = Array.from($headers).map(header =>
+          expect($headers).to.have.length(3)
-          header.textContent.trim()
+          const headers = Array.from($headers).map(header =>
-        )
+            header.textContent.trim()
-        expect(removeSpacing(headers)).to.deep.eq(["group", "age", "rating"])
+          )
+          expect(removeSpacing(headers)).to.deep.eq(["group", "age", "rating"])
+        })
       })
     })
 
-    it("filters the view by age over 10", () => {
+    xit("filters the view by age over 10", () => {
       cy.contains("Filter").click()
       cy.contains("Add Filter").click()
 
@@ -56,7 +58,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("creates a stats calculation view based on age", () => {
+    xit("creates a stats calculation view based on age", () => {
       cy.wait(1000)
       cy.contains("Calculate").click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
@@ -70,20 +72,22 @@ filterTests(['smoke', 'all'], () => {
       })
 
       cy.wait(1000)
-      cy.get(interact.TITLE).then($headers => {
+      cy.get(".table-wrapper").within(() => {
-        expect($headers).to.have.length(7)
+        cy.get(interact.TITLE).then($headers => {
-        const headers = Array.from($headers).map(header =>
+          expect($headers).to.have.length(7)
-          header.textContent.trim()
+          const headers = Array.from($headers).map(header =>
-        )
+            header.textContent.trim()
-        expect(removeSpacing(headers)).to.deep.eq([
+          )
-          "field",
+          expect(removeSpacing(headers)).to.deep.eq([
-          "sum",
+            "field",
-          "min",
+            "sum",
-          "max",
+            "min",
-          "count",
+            "max",
-          "sumsqr",
+            "count",
-          "avg",
+            "sumsqr",
-        ])
+            "avg",
+          ])
+        })
       })
       cy.get(interact.SPECTRUM_TABLE_CELL).then($values => {
         let values = Array.from($values).map(header => header.textContent.trim())
@@ -91,7 +95,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("groups the view by group", () => {
+    xit("groups the view by group", () => {
       cy.contains("Group by").click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.get(interact.SPECTRUM_PICKER_LABEL).eq(0).click()
@@ -123,7 +127,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("renames a view", () => {
+    xit("renames a view", () => {
       cy.contains(interact.NAV_ITEM, "Test View")
         .find(".actions .icon.open-popover")
         .click({ force: true })

packages/builder/cypress/integration/queryLevelTransformers.spec.js

@@ -14,11 +14,13 @@ filterTests(["smoke", "all"], () => {
       const restUrl = "https://api.openbrewerydb.org/breweries"
       cy.selectExternalDatasource(datasource)
       cy.createRestQuery("GET", restUrl, "/breweries")
-      cy.get(interact.SPECTRUM_TABS_ITEM).contains("Transformer").click()
+      cy.reload()
+      cy.contains(".nav-item-content", "/breweries", { timeout: 20000 }).click()
+      cy.contains(interact.SPECTRUM_TABS_ITEM, "Transformer", { timeout: 5000 }).click({ force: true })
       // Get Transformer Function from file
       cy.readFile("cypress/support/queryLevelTransformerFunction.js").then(
         transformerFunction => {
-          cy.get(interact.CODEMIRROR_TEXTAREA)
+          cy.get(interact.CODEMIRROR_TEXTAREA, { timeout: 5000 })
             // Highlight current text and overwrite with file contents
             .type(Cypress.platform === "darwin" ? "{cmd}a" : "{ctrl}a", {
               force: true,
@@ -28,6 +30,7 @@ filterTests(["smoke", "all"], () => {
       )
       // Send Query
       cy.intercept("**/queries/preview").as("query")
+      cy.get(interact.SPECTRUM_BUTTON).contains("Save").click({ force: true })
       cy.get(interact.SPECTRUM_BUTTON).contains("Send").click({ force: true })
       cy.wait("@query")
       // Assert against Status Code, body, & body rows
@@ -42,7 +45,9 @@ filterTests(["smoke", "all"], () => {
       const restUrl = "https://api.openbrewerydb.org/breweries"
       cy.selectExternalDatasource(datasource)
       cy.createRestQuery("GET", restUrl, "/breweries")
-      cy.get(interact.SPECTRUM_TABS_ITEM).contains("Transformer").click()
+      cy.reload()
+      cy.contains(".nav-item-content", "/breweries", { timeout: 2000 }).click()
+      cy.contains(interact.SPECTRUM_TABS_ITEM, "Transformer", { timeout: 5000 }).click({ force: true })
       // Get Transformer Function with Data from file
       cy.readFile(
         "cypress/support/queryLevelTransformerFunctionWithData.js"
@@ -71,7 +76,9 @@ filterTests(["smoke", "all"], () => {
       const restUrl = "https://api.openbrewerydb.org/breweries"
       cy.selectExternalDatasource(datasource)
       cy.createRestQuery("GET", restUrl, "/breweries")
-      cy.get(interact.SPECTRUM_TABS_ITEM).contains("Transformer").click()
+      cy.reload()
+      cy.contains(".nav-item-content", "/breweries", { timeout: 2000 }).click()
+      cy.contains(interact.SPECTRUM_TABS_ITEM, "Transformer", { timeout: 5000 }).click({ force: true })
       // Clear the code box and add "test"
       cy.get(interact.CODEMIRROR_TEXTAREA)
         .type(Cypress.platform === "darwin" ? "{cmd}a" : "{ctrl}a", {

packages/builder/cypress/support/commands.js

@@ -413,7 +413,7 @@ Cypress.Commands.add("searchForApplication", appName => {
 // Assumes there are no others
 Cypress.Commands.add("applicationInAppTable", appName => {
   cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 30000 })
-  cy.get(".appTable", { timeout: 5000 }).within(() => {
+  cy.get(".appTable", { timeout: 30000 }).within(() => {
     cy.get(".title").contains(appName).should("exist")
   })
 })
@@ -441,7 +441,7 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
   if (!initialTable) {
     cy.navigateToDataSection()
   }
-  cy.get(`[data-cy="new-datasource"]`, { timeout: 2000 }).click()
+  cy.get(`[data-cy="new-datasource"]`, { timeout: 20000 }).click()
   cy.wait(2000)
   cy.get(".item", { timeout: 2000 })
     .contains("Budibase DB")
@@ -461,10 +461,7 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
   cy.get(".nav-item", { timeout: 2000 })
     .contains("Budibase DB")
     .click({ force: true })
-  cy.get(".spectrum-Tabs-content", { timeout: 2000 }).should(
+  cy.get(".nav-item-content", { timeout: 2000 }).should("contain", tableName)
-    "contain",
-    tableName
-  )
 })
 
 Cypress.Commands.add("createTestTableWithData", () => {
@@ -483,7 +480,7 @@ Cypress.Commands.add(
 
     // Configure column
     cy.get(".spectrum-Modal").within(() => {
-      cy.get("input").first().type(columnName).blur()
+      cy.get("input").first().type(columnName)
 
       // Unset table display column
       cy.contains("display column").click({ force: true })
@@ -795,7 +792,7 @@ Cypress.Commands.add("selectExternalDatasource", datasourceName => {
   // Navigates to Data Section
   cy.navigateToDataSection()
   // Open Datasource modal
-  cy.get(".nav").within(() => {
+  cy.get(".container").within(() => {
     cy.get("[data-cy='new-datasource']").click()
   })
   // Clicks specified datasource & continue

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.1.43-alpha.15",
+  "version": "2.2.12-alpha.4",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.43-alpha.15",
+    "@budibase/bbui": "2.2.12-alpha.4",
-    "@budibase/client": "2.1.43-alpha.15",
+    "@budibase/client": "2.2.12-alpha.4",
-    "@budibase/frontend-core": "2.1.43-alpha.15",
+    "@budibase/frontend-core": "2.2.12-alpha.4",
-    "@budibase/string-templates": "2.1.43-alpha.15",
+    "@budibase/string-templates": "2.2.12-alpha.4",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -180,7 +180,7 @@
               onSelect(block)
             }}
           >
-            <Icon name={showLooping ? "ChevronDown" : "ChevronUp"} />
+            <Icon name={showLooping ? "ChevronUp" : "ChevronDown"} />
           </div>
         </div>
       </div>

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -232,6 +232,7 @@
             {filters}
             {bindings}
             {schemaFields}
+            datasource={{ type: "table", tableId }}
             panel={AutomationBindingPanel}
             fillWidth
             on:change={e => (tempFilters = e.detail)}

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -120,6 +120,12 @@
   const onUpdateRows = () => {
     fetch.refresh()
   }
+
+  // When importing new rows it is better to reinitialise request/paging data.
+  // Not doing so causes inconsistency in paging behaviour and content.
+  const onImportData = () => {
+    fetch.getInitialData()
+  }
 </script>
 
 <div>
@@ -172,7 +178,7 @@
         <ImportButton
           disabled={$tables.selected?._id === "ta_users"}
           tableId={$tables.selected?._id}
-          on:updaterows={onUpdateRows}
+          on:importrows={onImportData}
         />
         <ExportButton
           disabled={!hasRows || !hasCols}
@@ -184,6 +190,7 @@
             {filters}
             on:change={onFilter}
             disabled={!hasCols}
+            tableId={id}
           />
         {/key}
       </div>

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -216,7 +216,6 @@
     flex-direction: row;
     justify-content: flex-start;
     align-items: center;
-    margin-top: var(--spacing-m);
   }
   .table-title > div {
     margin-left: var(--spacing-xs);

packages/builder/src/components/backend/DataTable/buttons/ExistingRelationshipButton.svelte

@@ -1,7 +1,7 @@
 <script>
   import { ActionButton, Modal, notifications } from "@budibase/bbui"
   import CreateEditRelationship from "../../Datasources/CreateEditRelationship.svelte"
-  import { datasources, tables } from "../../../../stores/backend"
+  import { datasources } from "../../../../stores/backend"
   import { createEventDispatcher } from "svelte"
 
   export let table
@@ -21,8 +21,6 @@
       // Create datasource
       await datasources.save(datasource)
       notifications.success(`Relationship information saved.`)
-      const tableList = await tables.fetch()
-      await tables.select(tableList.find(tbl => tbl._id === table._id))
       dispatch("updatecolumns")
     } catch (err) {
       notifications.error(`Error saving relationship info: ${err}`)

packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte

@@ -12,5 +12,5 @@
   Import
 </ActionButton>
 <Modal bind:this={modal}>
-  <ImportModal {tableId} on:updaterows />
+  <ImportModal {tableId} on:importrows />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -6,6 +6,7 @@
   export let schema
   export let filters
   export let disabled = false
+  export let tableId
 
   const dispatch = createEventDispatcher()
 
@@ -37,6 +38,7 @@
         allowBindings={false}
         {filters}
         {schemaFields}
+        datasource={{ type: "table", tableId }}
         on:change={e => (tempValue = e.detail)}
       />
     </div>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -102,7 +102,7 @@
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
     opt =>
-      opt._id !== $tables.draft._id &&
+      opt._id !== $tables.selected._id &&
       opt.type === table.type &&
       table.sourceId === opt.sourceId
   )
@@ -112,7 +112,7 @@
 
   async function saveColumn() {
     if (field.type === AUTO_TYPE) {
-      field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
+      field = buildAutoColumn($tables.selected.name, field.name, field.subtype)
     }
     if (field.type !== LINK_TYPE) {
       delete field.fieldName
@@ -310,7 +310,7 @@
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(
         ", "
       )} are not allowed as column names`
-    } else if (inUse($tables.draft, fieldInfo.name, originalName)) {
+    } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
     if (fieldInfo.fieldName && fieldInfo.tableId) {

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { tables, rows } from "stores/backend"
+  import { tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
   import { API } from "api"
@@ -25,7 +25,6 @@
     try {
       await API.saveRow({ ...row, tableId: table._id })
       notifications.success("Row saved successfully")
-      rows.save()
       dispatch("updaterows")
     } catch (error) {
       if (error.handled) {

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { tables, rows } from "stores/backend"
+  import { tables } from "stores/backend"
   import { roles } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
@@ -57,7 +57,6 @@
     try {
       await API.saveRow({ ...row, tableId: table._id })
       notifications.success("User saved successfully")
-      rows.save()
       dispatch("updaterows")
     } catch (error) {
       if (error.handled) {

packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte

@@ -9,19 +9,20 @@
 
   $: views = $tables.list.flatMap(table => Object.keys(table.views || {}))
 
-  function saveView() {
+  const saveView = async () => {
+    name = name?.trim()
     if (views.includes(name)) {
       notifications.error(`View exists with name ${name}`)
       return
     }
     try {
-      viewsStore.save({
+      await viewsStore.save({
         name,
         tableId: $tables.selected._id,
         field,
       })
       notifications.success(`View ${name} created`)
-      $goto(`../../view/${name}`)
+      $goto(`../../view/${encodeURIComponent(name)}`)
     } catch (error) {
       notifications.error("Error creating view")
     }

packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte

@@ -29,7 +29,7 @@
     }
 
     // Always refresh rows just to be sure
-    dispatch("updaterows")
+    dispatch("importrows")
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,7 +1,5 @@
 <script>
-  import { onMount } from "svelte"
+  import { goto, isActive, params } from "@roxi/routify"
-  import { get } from "svelte/store"
-  import { goto, params } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
   import { database, datasources, queries, tables, views } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
@@ -14,40 +12,61 @@
     customQueryText,
   } from "helpers/data/utils"
   import IntegrationIcon from "./IntegrationIcon.svelte"
-  import { notifications } from "@budibase/bbui"
 
   let openDataSources = []
-  $: enrichedDataSources = Array.isArray($datasources.list)
+  $: enrichedDataSources = enrichDatasources(
-    ? $datasources.list.map(datasource => {
+    $datasources,
-        const selected = $datasources.selected === datasource._id
+    $params,
-        const open = openDataSources.includes(datasource._id)
+    $isActive,
-        const containsSelected = containsActiveEntity(datasource)
+    $tables,
-        const onlySource = $datasources.list.length === 1
+    $queries,
-        return {
+    $views
-          ...datasource,
+  )
-          selected,
-          open: selected || open || containsSelected || onlySource,
-        }
-      })
-    : []
   $: openDataSource = enrichedDataSources.find(x => x.open)
   $: {
-    // Ensure the open datasource is always included in the list of open
+    // Ensure the open datasource is always actually open
-    // datasources
     if (openDataSource) {
       openNode(openDataSource)
     }
   }
 
-  function selectDatasource(datasource) {
+  const enrichDatasources = (
-    openNode(datasource)
+    datasources,
-    datasources.select(datasource._id)
+    params,
-    $goto(`./datasource/${datasource._id}`)
+    isActive,
+    tables,
+    queries,
+    views
+  ) => {
+    if (!datasources?.list?.length) {
+      return []
+    }
+    return datasources.list.map(datasource => {
+      const selected =
+        isActive("./datasource") &&
+        datasources.selectedDatasourceId === datasource._id
+      const open = openDataSources.includes(datasource._id)
+      const containsSelected = containsActiveEntity(
+        datasource,
+        params,
+        isActive,
+        tables,
+        queries,
+        views
+      )
+      const onlySource = datasources.list.length === 1
+      return {
+        ...datasource,
+        selected,
+        containsSelected,
+        open: selected || open || containsSelected || onlySource,
+      }
+    })
   }
 
-  function onClickQuery(query) {
+  function selectDatasource(datasource) {
-    queries.select(query)
+    openNode(datasource)
-    $goto(`./datasource/${query.datasourceId}/${query._id}`)
+    $goto(`./datasource/${datasource._id}`)
   }
 
   function closeNode(datasource) {
@@ -69,21 +88,39 @@
     }
   }
 
-  onMount(async () => {
+  const containsActiveEntity = (
-    try {
+    datasource,
-      await datasources.fetch()
+    params,
-      await queries.fetch()
+    isActive,
-    } catch (error) {
+    tables,
-      notifications.error("Error fetching datasources and queries")
+    queries,
-    }
+    views
-  })
+  ) => {
-
+    // Check for being on a datasource page
-  const containsActiveEntity = datasource => {
+    if (params.datasourceId === datasource._id) {
-    // If we're view a query then the datasource ID is in the URL
-    if ($params.selectedDatasource === datasource._id) {
       return true
     }
 
+    // Check for hardcoded datasource edge cases
+    if (
+      isActive("./datasource/bb_internal") &&
+      datasource._id === "bb_internal"
+    ) {
+      return true
+    }
+    if (
+      isActive("./datasource/datasource_internal_bb_default") &&
+      datasource._id === "datasource_internal_bb_default"
+    ) {
+      return true
+    }
+
+    // Check for a matching query
+    if (params.queryId) {
+      const query = queries.list?.find(q => q._id === params.queryId)
+      return datasource._id === query?.datasourceId
+    }
+
     // If there are no entities it can't contain anything
     if (!datasource.entities) {
       return false
@@ -96,13 +133,13 @@
     }
 
     // Check for a matching table
-    if ($params.selectedTable) {
+    if (params.tableId) {
-      const selectedTable = get(tables).selected?._id
+      const selectedTable = tables.selected?._id
       return options.find(x => x._id === selectedTable) != null
     }
 
     // Check for a matching view
-    const selectedView = get(views).selected?.name
+    const selectedView = views.selected?.name
     const table = options.find(table => {
       return table.views?.[selectedView] != null
     })
@@ -117,7 +154,7 @@
         border={idx > 0}
         text={datasource.name}
         opened={datasource.open}
-        selected={datasource.selected}
+        selected={$isActive("./datasource") && datasource.selected}
         withArrow={true}
         on:click={() => selectDatasource(datasource)}
         on:iconClick={() => toggleNode(datasource)}
@@ -143,11 +180,11 @@
             iconText={customQueryIconText(datasource, query)}
             iconColor={customQueryIconColor(datasource, query)}
             text={customQueryText(datasource, query)}
-            opened={$queries.selected === query._id}
+            selected={$isActive("./query/:queryId") &&
-            selected={$queries.selected === query._id}
+              $queries.selectedQueryId === query._id}
-            on:click={() => onClickQuery(query)}
+            on:click={() => $goto(`./query/${query._id}`)}
           >
-            <EditQueryPopover {query} {onClickQuery} />
+            <EditQueryPopover {query} />
           </NavItem>
         {/each}
       {/if}
@@ -156,6 +193,9 @@
 {/if}
 
 <style>
+  .hierarchy-items-container {
+    margin: 0 calc(-1 * var(--spacing-xl));
+  }
   .datasource-icon {
     display: grid;
     place-items: center;

packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte

@@ -1,18 +1,15 @@
 <script>
   import { getIcon } from "./icons"
   import CustomSVG from "components/common/CustomSVG.svelte"
-  import { admin } from "stores/portal"
 
   export let integrationType
   export let schema
   export let size = "18"
 
-  $: objectStoreUrl = $admin.cloud ? "https://cdn.budi.live" : ""
-  $: pluginsUrl = `${objectStoreUrl}/plugins`
   $: iconInfo = getIcon(integrationType, schema)
 
   async function getSvgFromUrl(info) {
-    const url = `${pluginsUrl}/${info.url}`
+    const url = `${info.url}`
     const resp = await fetch(url, {
       headers: {
         ["pragma"]: "no-cache",

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -104,7 +104,6 @@
   }
 
   function onClickTable(table) {
-    tables.select(table)
     $goto(`../../table/${table._id}`)
   }
 

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -11,7 +11,7 @@
   import { onMount } from "svelte"
   import ICONS from "../icons"
   import { API } from "api"
-  import { IntegrationTypes } from "constants/backend"
+  import { IntegrationTypes, DatasourceTypes } from "constants/backend"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
   import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
   import GoogleDatasourceConfigModal from "components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte"
@@ -31,6 +31,7 @@
   $: customIntegrations = Object.entries(integrations).filter(
     entry => entry[1].custom
   )
+  $: sortedIntegrations = sortIntegrations(integrations)
 
   checkShowImport()
 
@@ -99,6 +100,29 @@
     }
     integrations = newIntegrations
   }
+
+  function sortIntegrations(integrations) {
+    let integrationsArray = Object.entries(integrations)
+    function getTypeOrder(schema) {
+      if (schema.type === DatasourceTypes.API) {
+        return 1
+      }
+      if (schema.type === DatasourceTypes.RELATIONAL) {
+        return 2
+      }
+      return schema.type?.charCodeAt(0)
+    }
+
+    integrationsArray.sort((a, b) => {
+      let typeOrderA = getTypeOrder(a[1])
+      let typeOrderB = getTypeOrder(b[1])
+      if (typeOrderA === typeOrderB) {
+        return a[1].friendlyName?.localeCompare(b[1].friendlyName)
+      }
+      return typeOrderA < typeOrderB ? -1 : 1
+    })
+    return integrationsArray
+  }
 </script>
 
 <Modal bind:this={internalTableModal}>
@@ -157,7 +181,7 @@
     <Layout noPadding gap="XS">
       <Body size="S">Connect to an external datasource</Body>
       <div class="item-list">
-        {#each Object.entries(integrations).filter(([key, val]) => key !== IntegrationTypes.INTERNAL && !val.custom) as [integrationType, schema]}
+        {#each sortedIntegrations.filter(([key, val]) => key !== IntegrationTypes.INTERNAL && !val.custom) as [integrationType, schema]}
           <DatasourceCard
             on:selected={evt => selectIntegration(evt.detail)}
             {schema}

packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte

@@ -64,7 +64,6 @@
       // reload
       await datasources.fetch()
       await queries.fetch()
-      await datasources.select(datasourceId)
 
       if (navigateDatasource) {
         $goto(`./datasource/${datasourceId}`)

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { datasources, queries, tables } from "stores/backend"
+  import { datasources } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -14,23 +14,10 @@
 
   async function deleteDatasource() {
     try {
-      let wasSelectedSource = $datasources.selected
+      const isSelected = datasource.selected || datasource.containsSelected
-      if (!wasSelectedSource && $queries.selected) {
-        const queryId = $queries.selected
-        wasSelectedSource = $datasources.list.find(ds =>
-          queryId.includes(ds._id)
-        )?._id
-      }
-      const wasSelectedTable = $tables.selected
       await datasources.delete(datasource)
       notifications.success("Datasource deleted")
-      // Navigate to first index page if the source you are deleting is selected
+      if (isSelected) {
-      const entities = Object.values(datasource?.entities || {})
-      if (
-        wasSelectedSource === datasource._id ||
-        (entities &&
-          entities.find(entity => entity._id === wasSelectedTable?._id))
-      ) {
         $goto("./datasource")
       }
     } catch (error) {

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte

@@ -5,23 +5,17 @@
   import { datasources, queries } from "stores/backend"
 
   export let query
-  export let onClickQuery
 
   let confirmDeleteDialog
 
   async function deleteQuery() {
     try {
-      const wasSelectedQuery = $queries.selected
+      // Go back to the datasource if we are deleting the active query
-      // need to calculate this before the query is deleted
+      if ($queries.selectedQueryId === query._id) {
-      const navigateToDatasource = wasSelectedQuery === query._id
-
-      await queries.delete(query)
-      await datasources.fetch()
-
-      if (navigateToDatasource) {
-        await datasources.select(query.datasourceId)
         $goto(`./datasource/${query.datasourceId}`)
       }
+      await queries.delete(query)
+      await datasources.fetch()
       notifications.success("Query deleted")
     } catch (error) {
       notifications.error("Error deleting query")
@@ -31,7 +25,7 @@
   async function duplicateQuery() {
     try {
       const newQuery = await queries.duplicate(query)
-      onClickQuery(newQuery)
+      $goto(`./query/${newQuery._id}`)
     } catch (error) {
       notifications.error("Error duplicating query")
     }

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -1,39 +1,18 @@
 <script>
-  import { goto } from "@roxi/routify"
   import { tables, views, database } from "stores/backend"
   import { TableNames } from "constants"
   import EditTablePopover from "./popovers/EditTablePopover.svelte"
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
+  import { goto, isActive } from "@roxi/routify"
 
   const alphabetical = (a, b) => a.name?.toLowerCase() > b.name?.toLowerCase()
 
   export let sourceId
 
-  $: selectedView = $views.selected && $views.selected.name
   $: sortedTables = $tables.list
     .filter(table => table.sourceId === sourceId)
     .sort(alphabetical)
-
-  function selectTable(table) {
-    tables.select(table)
-    $goto(`./table/${table._id}`)
-  }
-
-  function selectView(view) {
-    views.select(view)
-    $goto(`./view/${view.name}`)
-  }
-
-  function onClickView(table, viewName) {
-    if (selectedView === viewName) {
-      return
-    }
-    selectView({
-      name: viewName,
-      ...table.views[viewName],
-    })
-  }
 </script>
 
 {#if $database?._id}
@@ -44,8 +23,9 @@
         border={idx > 0}
         icon={table._id === TableNames.USERS ? "UserGroup" : "Table"}
         text={table.name}
-        selected={$tables.selected?._id === table._id}
+        selected={$isActive("./table/:tableId") &&
-        on:click={() => selectTable(table)}
+          $tables.selected?._id === table._id}
+        on:click={() => $goto(`./table/${table._id}`)}
       >
         {#if table._id !== TableNames.USERS}
           <EditTablePopover {table} />
@@ -56,8 +36,8 @@
           indentLevel={2}
           icon="Remove"
           text={viewName}
-          selected={selectedView === viewName}
+          selected={$isActive("./view") && $views.selected?.name === viewName}
-          on:click={() => onClickView(table, viewName)}
+          on:click={() => $goto(`./view/${encodeURIComponent(viewName)}`)}
         >
           <EditViewPopover
             view={{ name: viewName, ...table.views[viewName] }}

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -1,6 +1,7 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { store } from "builderStore"
+  import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
   import {
     ActionMenu,
@@ -18,7 +19,10 @@
   let editorModal
   let confirmDeleteDialog
   let error = ""
-  let originalName = table.name
+
+  let originalName
+  let updatedName
+
   let templateScreens
   let willBeDeleted
   let deleteTableName
@@ -37,17 +41,16 @@
   }
 
   async function deleteTable() {
-    const wasSelectedTable = $tables.selected
+    const isSelected = $params.tableId === table._id
     try {
       await tables.delete(table)
       await store.actions.screens.delete(templateScreens)
-      await tables.fetch()
       if (table.type === "external") {
         await datasources.fetch()
       }
       notifications.success("Table deleted")
-      if (wasSelectedTable && wasSelectedTable._id === table._id) {
+      if (isSelected) {
-        $goto("./table")
+        $goto(`./datasource/${table.datasourceId}`)
       }
     } catch (error) {
       notifications.error("Error deleting table")
@@ -59,7 +62,9 @@
   }
 
   async function save() {
-    await tables.save(table)
+    const updatedTable = cloneDeep(table)
+    updatedTable.name = updatedName
+    await tables.save(updatedTable)
     notifications.success("Table renamed successfully")
   }
 
@@ -70,6 +75,11 @@
         ? `Table with name ${tableName} already exists. Please choose another name.`
         : ""
   }
+
+  const initForm = () => {
+    originalName = table.name + ""
+    updatedName = table.name + ""
+  }
 </script>
 
 {#if allowDeletion}
@@ -84,17 +94,17 @@
   </ActionMenu>
 {/if}
 
-<Modal bind:this={editorModal}>
+<Modal bind:this={editorModal} on:show={initForm}>
   <ModalContent
     title="Edit Table"
     confirmText="Save"
     onConfirm={save}
-    disabled={table.name === originalName || error}
+    disabled={updatedName === originalName || error}
   >
     <Input
       label="Table Name"
       thin
-      bind:value={table.name}
+      bind:value={updatedName}
       on:input={checkValid}
       {error}
     />

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -1,6 +1,7 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { views } from "stores/backend"
+  import { cloneDeep } from "lodash/fp"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import {
     notifications,
@@ -15,28 +16,41 @@
   export let view
 
   let editorModal
-  let originalName = view.name
+  let originalName
+  let updatedName
   let confirmDeleteDialog
 
   async function save() {
+    const updatedView = cloneDeep(view)
+    updatedView.name = updatedName
+
     await views.save({
       originalName,
-      ...view,
+      ...updatedView,
     })
     notifications.success("View renamed successfully")
   }
 
   async function deleteView() {
     try {
+      const isSelected =
+        decodeURIComponent($params.viewName) === $views.selectedViewName
       const name = view.name
       const id = view.tableId
       await views.delete(name)
       notifications.success("View deleted")
-      $goto(`./table/${id}`)
+      if (isSelected) {
+        $goto(`./table/${id}`)
+      }
     } catch (error) {
       notifications.error("Error deleting view")
     }
   }
+
+  const initForm = () => {
+    updatedName = view.name + ""
+    originalName = view.name + ""
+  }
 </script>
 
 <ActionMenu>
@@ -46,9 +60,9 @@
   <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>
   <MenuItem icon="Delete" on:click={confirmDeleteDialog.show}>Delete</MenuItem>
 </ActionMenu>
-<Modal bind:this={editorModal}>
+<Modal bind:this={editorModal} on:show={initForm}>
   <ModalContent title="Edit View" onConfirm={save} confirmText="Save">
-    <Input label="View Name" thin bind:value={view.name} />
+    <Input label="View Name" thin bind:value={updatedName} />
   </ModalContent>
 </Modal>
 <ConfirmDialog

packages/builder/src/components/deploy/DeployModal.svelte

@@ -12,7 +12,6 @@
   import { ProgressCircle } from "@budibase/bbui"
   import CopyInput from "components/common/inputs/CopyInput.svelte"
 
-  let feedbackModal
   let publishModal
   let asyncModal
   let publishCompleteModal
@@ -23,13 +22,13 @@
 
   export let onOk
 
-  async function deployApp() {
+  async function publishApp() {
     try {
       //In Progress
       asyncModal.show()
       publishModal.hide()
 
-      published = await API.deployAppChanges()
+      published = await API.publishAppChanges($store.appId)
 
       if (typeof onOk === "function") {
         await onOk()
@@ -56,20 +55,11 @@
 </script>
 
 <Button cta on:click={publishModal.show}>Publish</Button>
-<Modal bind:this={feedbackModal}>
-  <ModalContent
-    title="Enjoying Budibase?"
-    size="L"
-    showConfirmButton={false}
-    showCancelButton={false}
-  />
-</Modal>
-
 <Modal bind:this={publishModal}>
   <ModalContent
     title="Publish to Production"
     confirmText="Publish"
-    onConfirm={deployApp}
+    onConfirm={publishApp}
     dataCy={"deploy-app-modal"}
   >
     <span

packages/builder/src/components/design/Panel.svelte

@@ -51,6 +51,7 @@
 <style>
   .panel {
     width: 260px;
+    flex: 0 0 260px;
     background: var(--background);
     display: flex;
     flex-direction: column;
@@ -66,6 +67,7 @@
   }
   .panel.wide {
     width: 420px;
+    flex: 0 0 420px;
   }
   .header {
     flex: 0 0 48px;

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/PromptUser.svelte

@@ -0,0 +1,50 @@
+<script>
+  import { Body, Label, Input } from "@budibase/bbui"
+  import { onMount } from "svelte"
+
+  export let parameters
+
+  onMount(() => {
+    if (!parameters.confirm) {
+      parameters.confirm = true
+    }
+  })
+</script>
+
+<div class="root">
+  <Body size="S">Enter the message you wish to display to the user.</Body>
+  <div class="params">
+    <Label small>Title</Label>
+    <Input placeholder="Prompt User" bind:value={parameters.customTitleText} />
+    <Label small>Message</Label>
+    <Input
+      placeholder="Are you sure you want to continue?"
+      bind:value={parameters.confirmText}
+    />
+  </div>
+</div>
+
+<style>
+  .root {
+    width: 100%;
+    max-width: 800px;
+    margin: 0 auto;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: var(--spacing-xl);
+  }
+
+  .root :global(p) {
+    line-height: 1.5;
+  }
+
+  .params {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js

@@ -16,5 +16,6 @@ export { default as ExportData } from "./ExportData.svelte"
 export { default as ContinueIf } from "./ContinueIf.svelte"
 export { default as UpdateFieldValue } from "./UpdateFieldValue.svelte"
 export { default as ShowNotification } from "./ShowNotification.svelte"
+export { default as PromptUser } from "./PromptUser.svelte"
 export { default as OpenSidePanel } from "./OpenSidePanel.svelte"
 export { default as CloseSidePanel } from "./CloseSidePanel.svelte"

packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json

@@ -117,6 +117,11 @@
       "component": "ShowNotification",
       "dependsOnFeature": "showNotificationAction"
     },
+    {
+      "name": "Prompt User",
+      "type": "application",
+      "component": "PromptUser"
+    },
     {
       "name": "Open Side Panel",
       "type": "application",

packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte

@@ -144,6 +144,11 @@
     drawer.show()
   }
 
+  const getQueryValue = queries => {
+    value = queries.find(q => q._id === value._id) || value
+    return value
+  }
+
   const saveQueryParams = () => {
     handleSelected({
       ...value,
@@ -175,7 +180,7 @@
           {/if}
           <IntegrationQueryEditor
             height={200}
-            query={value}
+            query={getQueryValue(queries)}
             schema={fetchQueryDefinition(value)}
             datasource={getQueryDatasource(value)}
             editable={false}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -25,7 +25,7 @@
   export let panel = ClientBindingPanel
   export let allowBindings = true
   export let fillWidth = false
-  export let tableId
+  export let datasource
 
   const dispatch = createEventDispatcher()
   const { OperatorOptions } = Constants
@@ -41,11 +41,7 @@
 
   $: parseFilters(filters)
   $: dispatch("change", enrichFilters(rawFilters, matchAny))
-  $: enrichedSchemaFields = getFields(
+  $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
-    schemaFields || [],
-    { allowLinks: true },
-    tableId
-  )
   $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
@@ -119,7 +115,11 @@
 
   const santizeOperator = filter => {
     // Ensure a valid operator is selected
-    const operators = getValidOperatorsForType(filter.type).map(x => x.value)
+    const operators = getValidOperatorsForType(
+      filter.type,
+      filter.field,
+      datasource
+    ).map(x => x.value)
     if (!operators.includes(filter.operator)) {
       filter.operator = operators[0] ?? OperatorOptions.Equals.value
     }
@@ -201,7 +201,11 @@
               />
               <Select
                 disabled={!filter.field}
-                options={getValidOperatorsForType(filter.type)}
+                options={getValidOperatorsForType(
+                  filter.type,
+                  filter.field,
+                  datasource
+                )}
                 bind:value={filter.operator}
                 on:change={() => onOperatorChange(filter)}
                 placeholder={null}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte

@@ -17,8 +17,8 @@
   let drawer
 
   $: tempValue = value
-  $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
+  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource)?.schema
   $: schemaFields = Object.values(schema || {})
 
   async function saveFilter() {
@@ -36,7 +36,7 @@
     filters={value}
     {bindings}
     {schemaFields}
-    tableId={dataSource.tableId}
+    {datasource}
     on:change={e => (tempValue = e.detail)}
   />
 </Drawer>

packages/builder/src/components/integration/QueryViewer.svelte

@@ -29,11 +29,12 @@
 
   export let query
 
+  const transformerDocs = "https://docs.budibase.com/docs/transformers"
+
   let fields = query?.schema ? schemaToFields(query.schema) : []
   let parameters
   let data = []
   let saveId
-  const transformerDocs = "https://docs.budibase.com/docs/transformers"
 
   $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
   $: query.schema = fieldsToSchema(fields)
@@ -94,132 +95,144 @@
     try {
       const { _id } = await queries.save(query.datasourceId, query)
       saveId = _id
-      notifications.success(`Query saved successfully.`)
+      notifications.success(`Query saved successfully`)
-      $goto(`../${_id}`)
+
+      // Go to the correct URL if we just created a new query
+      if (!query._rev) {
+        $goto(`../../${_id}`)
+      }
     } catch (error) {
-      notifications.error("Error creating query")
+      notifications.error("Error saving query")
     }
   }
 </script>
 
-<Layout gap="S" noPadding>
+<div class="wrapper">
-  <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
+  <Layout gap="S" noPadding>
-  <Divider />
+    <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
-  <Heading size="S">Config</Heading>
-  <div class="config">
-    <div class="config-field">
-      <Label>Query Name</Label>
-      <Input bind:value={query.name} />
-    </div>
-    {#if queryConfig}
-      <div class="config-field">
-        <Label>Function</Label>
-        <Select
-          bind:value={query.queryVerb}
-          on:change={resetDependentFields}
-          options={Object.keys(queryConfig)}
-          getOptionLabel={verb =>
-            queryConfig[verb]?.displayName || capitalise(verb)}
-        />
-      </div>
-      <div class="config-field">
-        <AccessLevelSelect {saveId} {query} label="Access Level" />
-      </div>
-      {#if integrationInfo?.extra && query.queryVerb}
-        <ExtraQueryConfig
-          {query}
-          {populateExtraQuery}
-          config={integrationInfo.extra}
-        />
-      {/if}
-      {#key query.parameters}
-        <BindingBuilder
-          queryBindings={query.parameters}
-          bindable={false}
-          on:change={e => {
-            query.parameters = e.detail.map(binding => {
-              return {
-                name: binding.name,
-                default: binding.value,
-              }
-            })
-          }}
-        />
-      {/key}
-    {/if}
-  </div>
-  {#if shouldShowQueryConfig}
     <Divider />
+    <Heading size="S">Config</Heading>
     <div class="config">
-      <Heading size="S">Fields</Heading>
+      <div class="config-field">
-      <Body size="S">Fill in the fields specific to this query.</Body>
+        <Label>Query Name</Label>
-      <IntegrationQueryEditor
+        <Input bind:value={query.name} />
-        {datasource}
-        {query}
-        height={200}
-        schema={queryConfig[query.queryVerb]}
-        bind:parameters
-      />
-      <Divider />
-    </div>
-    <div class="config">
-      <div class="help-heading">
-        <Heading size="S">Transformer</Heading>
-        <Icon
-          on:click={() => window.open(transformerDocs)}
-          hoverable
-          name="Help"
-          size="L"
-        />
       </div>
-      <Body size="S"
+      {#if queryConfig}
-        >Add a JavaScript function to transform the query result.</Body
+        <div class="config-field">
-      >
+          <Label>Function</Label>
-      <CodeMirrorEditor
+          <Select
-        height={200}
+            bind:value={query.queryVerb}
-        label="Transformer"
+            on:change={resetDependentFields}
-        value={query.transformer}
+            options={Object.keys(queryConfig)}
-        resize="vertical"
+            getOptionLabel={verb =>
-        on:change={e => (query.transformer = e.detail)}
+              queryConfig[verb]?.displayName || capitalise(verb)}
-      />
+          />
-      <Divider />
+        </div>
-    </div>
+        <div class="config-field">
-    <div class="viewer-controls">
+          <AccessLevelSelect {saveId} {query} label="Access Level" />
-      <Heading size="S">Results</Heading>
+        </div>
-      <ButtonGroup gap="M">
+        {#if integrationInfo?.extra && query.queryVerb}
-        <Button cta disabled={queryInvalid} on:click={saveQuery}>
+          <ExtraQueryConfig
-          Save Query
+            {query}
-        </Button>
+            {populateExtraQuery}
-        <Button secondary on:click={previewQuery}>Run Query</Button>
+            config={integrationInfo.extra}
-      </ButtonGroup>
+          />
-    </div>
+        {/if}
-    <Body size="S">
+        {#key query.parameters}
-      Below, you can preview the results from your query and change the schema.
+          <BindingBuilder
-    </Body>
+            queryBindings={query.parameters}
-    <section class="viewer">
+            bindable={false}
-      {#if data}
+            on:change={e => {
-        <Tabs selected="JSON">
+              query.parameters = e.detail.map(binding => {
-          <Tab title="JSON">
+                return {
-            <JSONPreview data={data[0]} minHeight="120" />
+                  name: binding.name,
-          </Tab>
+                  default: binding.value,
-          <Tab title="Schema">
+                }
-            <KeyValueBuilder
+              })
-              bind:object={fields}
+            }}
-              name="field"
+          />
-              headings
+        {/key}
-              options={SchemaTypeOptions}
-            />
-          </Tab>
-          <Tab title="Preview">
-            <ExternalDataSourceTable {query} {data} />
-          </Tab>
-        </Tabs>
       {/if}
-    </section>
+    </div>
-  {/if}
+    {#if shouldShowQueryConfig}
-</Layout>
+      <Divider />
+      <div class="config">
+        <Heading size="S">Fields</Heading>
+        <Body size="S">Fill in the fields specific to this query.</Body>
+        <IntegrationQueryEditor
+          {datasource}
+          {query}
+          height={200}
+          schema={queryConfig[query.queryVerb]}
+          bind:parameters
+        />
+        <Divider />
+      </div>
+      <div class="config">
+        <div class="help-heading">
+          <Heading size="S">Transformer</Heading>
+          <Icon
+            on:click={() => window.open(transformerDocs)}
+            hoverable
+            name="Help"
+            size="L"
+          />
+        </div>
+        <Body size="S"
+          >Add a JavaScript function to transform the query result.</Body
+        >
+        <CodeMirrorEditor
+          height={200}
+          label="Transformer"
+          value={query.transformer}
+          resize="vertical"
+          on:change={e => (query.transformer = e.detail)}
+        />
+        <Divider />
+      </div>
+      <div class="viewer-controls">
+        <Heading size="S">Results</Heading>
+        <ButtonGroup gap="XS">
+          <Button cta disabled={queryInvalid} on:click={saveQuery}>
+            Save Query
+          </Button>
+          <Button secondary on:click={previewQuery}>Run Query</Button>
+        </ButtonGroup>
+      </div>
+      <Body size="S">
+        Below, you can preview the results from your query and change the
+        schema.
+      </Body>
+      <section class="viewer">
+        {#if data}
+          <Tabs selected="JSON">
+            <Tab title="JSON">
+              <JSONPreview data={data[0]} minHeight="120" />
+            </Tab>
+            <Tab title="Schema">
+              <KeyValueBuilder
+                bind:object={fields}
+                name="field"
+                headings
+                options={SchemaTypeOptions}
+              />
+            </Tab>
+            <Tab title="Preview">
+              <ExternalDataSourceTable {query} {data} />
+            </Tab>
+          </Tabs>
+        {/if}
+      </section>
+    {/if}
+  </Layout>
+</div>
 
 <style>
+  .wrapper {
+    width: 640px;
+    margin: auto;
+  }
+
   .config {
     display: grid;
     grid-gap: var(--spacing-s);